sysadmin/ipxe
1
0
Fork 0
mirror of https://github.com/ipxe/ipxe synced 2025-12-08 10:20:24 +03:00
Code Issues Packages Projects Releases Wiki Activity
Files
410566cef7b760b5e59050b38e1eb4e7572f059b
ipxe/src/drivers/infiniband/hermon.c
Michael Brown 410566cef7 [hermon] Minimise reset time 
Check for reset completion by waiting for the device to respond to PCI
configuration cycles, as documented in the Programmer's Reference
Manual.  On the original ConnectX HCA, this reduces the time spent on
reset from 1000ms down to 1ms.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
2021-02-01 22:29:30 +00:00

4216 lines
122 KiB
C
Raw Blame History

/*
* Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>.
* Copyright (C) 2008 Mellanox Technologies Ltd.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
FILE_LICENCE ( GPL2_OR_LATER );
#include <stdint.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#include <errno.h>
#include <byteswap.h>
#include <ipxe/io.h>
#include <ipxe/pci.h>
#include <ipxe/pcibackup.h>
#include <ipxe/malloc.h>
#include <ipxe/umalloc.h>
#include <ipxe/iobuf.h>
#include <ipxe/netdevice.h>
#include <ipxe/infiniband.h>
#include <ipxe/ib_smc.h>
#include <ipxe/if_ether.h>
#include <ipxe/ethernet.h>
#include <ipxe/fcoe.h>
#include <ipxe/vlan.h>
#include <ipxe/bofm.h>
#include <ipxe/nvsvpd.h>
#include <ipxe/nvo.h>
#include "hermon.h"
/**
* @file
*
* Mellanox Hermon Infiniband HCA
*
*/
/***************************************************************************
*
* Queue number allocation
*
***************************************************************************
*/
/**
* Allocate offsets within usage bitmask
*
* @v bits Usage bitmask
* @v bits_len Length of usage bitmask
* @v num_bits Number of contiguous bits to allocate within bitmask
* @ret bit First free bit within bitmask, or negative error
*/
static int hermon_bitmask_alloc ( hermon_bitmask_t *bits,
unsigned int bits_len,
unsigned int num_bits ) {
unsigned int bit = 0;
hermon_bitmask_t mask = 1;
unsigned int found = 0;
/* Search bits for num_bits contiguous free bits */
while ( bit < bits_len ) {
if ( ( mask & *bits ) == 0 ) {
if ( ++found == num_bits )
goto found;
} else {
found = 0;
}
bit++;
mask = ( mask << 1 ) | ( mask >> ( 8 * sizeof ( mask ) - 1 ) );
if ( mask == 1 )
bits++;
}
return -ENFILE;
found:
/* Mark bits as in-use */
do {
*bits |= mask;
if ( mask == 1 )
bits--;
mask = ( mask >> 1 ) | ( mask << ( 8 * sizeof ( mask ) - 1 ) );
} while ( --found );
return ( bit - num_bits + 1 );
}
/**
* Free offsets within usage bitmask
*
* @v bits Usage bitmask
* @v bit Starting bit within bitmask
* @v num_bits Number of contiguous bits to free within bitmask
*/
static void hermon_bitmask_free ( hermon_bitmask_t *bits,
int bit, unsigned int num_bits ) {
hermon_bitmask_t mask;
for ( ; num_bits ; bit++, num_bits-- ) {
mask = ( 1 << ( bit % ( 8 * sizeof ( mask ) ) ) );
bits[ ( bit / ( 8 * sizeof ( mask ) ) ) ] &= ~mask;
}
}
/***************************************************************************
*
* HCA commands
*
***************************************************************************
*/
/**
* Wait for Hermon command completion
*
* @v hermon Hermon device
* @v hcr HCA command registers
* @ret rc Return status code
*/
static int hermon_cmd_wait ( struct hermon *hermon,
struct hermonprm_hca_command_register *hcr ) {
unsigned int wait;
for ( wait = ( 100 * HERMON_HCR_MAX_WAIT_MS ) ; wait ; wait-- ) {
hcr->u.dwords[6] =
readl ( hermon->config + HERMON_HCR_REG ( 6 ) );
if ( ( MLX_GET ( hcr, go ) == 0 ) &&
( MLX_GET ( hcr, t ) == hermon->toggle ) )
return 0;
udelay ( 10 );
}
return -EBUSY;
}
/**
* Issue HCA command
*
* @v hermon Hermon device
* @v command Command opcode, flags and input/output lengths
* @v op_mod Opcode modifier (0 if no modifier applicable)
* @v in Input parameters
* @v in_mod Input modifier (0 if no modifier applicable)
* @v out Output parameters
* @ret rc Return status code
*/
static int hermon_cmd ( struct hermon *hermon, unsigned long command,
unsigned int op_mod, const void *in,
unsigned int in_mod, void *out ) {
struct hermonprm_hca_command_register hcr;
unsigned int opcode = HERMON_HCR_OPCODE ( command );
size_t in_len = HERMON_HCR_IN_LEN ( command );
size_t out_len = HERMON_HCR_OUT_LEN ( command );
void *in_buffer;
void *out_buffer;
unsigned int status;
unsigned int i;
int rc;
assert ( in_len <= HERMON_MBOX_SIZE );
assert ( out_len <= HERMON_MBOX_SIZE );
DBGC2 ( hermon, "Hermon %p command %04x in %zx%s out %zx%s\n",
hermon, opcode, in_len,
( ( command & HERMON_HCR_IN_MBOX ) ? "(mbox)" : "" ), out_len,
( ( command & HERMON_HCR_OUT_MBOX ) ? "(mbox)" : "" ) );
/* Check that HCR is free */
if ( ( rc = hermon_cmd_wait ( hermon, &hcr ) ) != 0 ) {
DBGC ( hermon, "Hermon %p command interface locked\n",
hermon );
return rc;
}
/* Flip HCR toggle */
hermon->toggle = ( 1 - hermon->toggle );
/* Prepare HCR */
memset ( &hcr, 0, sizeof ( hcr ) );
in_buffer = &hcr.u.dwords[0];
if ( in_len && ( command & HERMON_HCR_IN_MBOX ) ) {
memset ( hermon->mailbox_in, 0, HERMON_MBOX_SIZE );
in_buffer = hermon->mailbox_in;
MLX_FILL_H ( &hcr, 0, in_param_h, virt_to_bus ( in_buffer ) );
MLX_FILL_1 ( &hcr, 1, in_param_l, virt_to_bus ( in_buffer ) );
}
memcpy ( in_buffer, in, in_len );
MLX_FILL_1 ( &hcr, 2, input_modifier, in_mod );
out_buffer = &hcr.u.dwords[3];
if ( out_len && ( command & HERMON_HCR_OUT_MBOX ) ) {
out_buffer = hermon->mailbox_out;
MLX_FILL_H ( &hcr, 3, out_param_h,
virt_to_bus ( out_buffer ) );
MLX_FILL_1 ( &hcr, 4, out_param_l,
virt_to_bus ( out_buffer ) );
}
MLX_FILL_4 ( &hcr, 6,
opcode, opcode,
opcode_modifier, op_mod,
go, 1,
t, hermon->toggle );
DBGC2_HDA ( hermon, virt_to_phys ( hermon->config + HERMON_HCR_BASE ),
&hcr, sizeof ( hcr ) );
if ( in_len && ( command & HERMON_HCR_IN_MBOX ) ) {
DBGC2 ( hermon, "Input mailbox:\n" );
DBGC2_HDA ( hermon, virt_to_phys ( in_buffer ), in_buffer,
( ( in_len < 512 ) ? in_len : 512 ) );
}
/* Issue command */
for ( i = 0 ; i < ( sizeof ( hcr ) / sizeof ( hcr.u.dwords[0] ) ) ;
i++ ) {
writel ( hcr.u.dwords[i],
hermon->config + HERMON_HCR_REG ( i ) );
barrier();
}
/* Wait for command completion */
if ( ( rc = hermon_cmd_wait ( hermon, &hcr ) ) != 0 ) {
DBGC ( hermon, "Hermon %p timed out waiting for command "
"%04x:\n", hermon, opcode );
DBGC_HDA ( hermon,
virt_to_phys ( hermon->config + HERMON_HCR_BASE ),
&hcr, sizeof ( hcr ) );
return rc;
}
/* Check command status */
status = MLX_GET ( &hcr, status );
if ( status != 0 ) {
DBGC ( hermon, "Hermon %p command %04x failed with status "
"%02x:\n", hermon, opcode, status );
DBGC_HDA ( hermon,
virt_to_phys ( hermon->config + HERMON_HCR_BASE ),
&hcr, sizeof ( hcr ) );
return -EIO;
}
/* Read output parameters, if any */
hcr.u.dwords[3] = readl ( hermon->config + HERMON_HCR_REG ( 3 ) );
hcr.u.dwords[4] = readl ( hermon->config + HERMON_HCR_REG ( 4 ) );
memcpy ( out, out_buffer, out_len );
if ( out_len ) {
DBGC2 ( hermon, "Output%s:\n",
( command & HERMON_HCR_OUT_MBOX ) ? " mailbox" : "" );
DBGC2_HDA ( hermon, virt_to_phys ( out_buffer ), out_buffer,
( ( out_len < 512 ) ? out_len : 512 ) );
}
return 0;
}
static inline int
hermon_cmd_query_dev_cap ( struct hermon *hermon,
struct hermonprm_query_dev_cap *dev_cap ) {
return hermon_cmd ( hermon,
HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_DEV_CAP,
1, sizeof ( *dev_cap ) ),
0, NULL, 0, dev_cap );
}
static inline int
hermon_cmd_query_fw ( struct hermon *hermon, struct hermonprm_query_fw *fw ) {
return hermon_cmd ( hermon,
HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_FW,
1, sizeof ( *fw ) ),
0, NULL, 0, fw );
}
static inline int
hermon_cmd_init_hca ( struct hermon *hermon,
const struct hermonprm_init_hca *init_hca ) {
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_INIT_HCA,
1, sizeof ( *init_hca ) ),
0, init_hca, 0, NULL );
}
static inline int
hermon_cmd_close_hca ( struct hermon *hermon ) {
return hermon_cmd ( hermon,
HERMON_HCR_VOID_CMD ( HERMON_HCR_CLOSE_HCA ),
0, NULL, 0, NULL );
}
static inline int
hermon_cmd_init_port ( struct hermon *hermon, unsigned int port ) {
return hermon_cmd ( hermon,
HERMON_HCR_VOID_CMD ( HERMON_HCR_INIT_PORT ),
0, NULL, port, NULL );
}
static inline int
hermon_cmd_close_port ( struct hermon *hermon, unsigned int port ) {
return hermon_cmd ( hermon,
HERMON_HCR_VOID_CMD ( HERMON_HCR_CLOSE_PORT ),
0, NULL, port, NULL );
}
static inline int
hermon_cmd_set_port ( struct hermon *hermon, int is_ethernet,
unsigned int port_selector,
const union hermonprm_set_port *set_port ) {
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_SET_PORT,
1, sizeof ( *set_port ) ),
is_ethernet, set_port, port_selector, NULL );
}
static inline int
hermon_cmd_sw2hw_mpt ( struct hermon *hermon, unsigned int index,
const struct hermonprm_mpt *mpt ) {
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_MPT,
1, sizeof ( *mpt ) ),
0, mpt, index, NULL );
}
static inline int
hermon_cmd_hw2sw_mpt ( struct hermon *hermon, unsigned int index ) {
return hermon_cmd ( hermon,
HERMON_HCR_VOID_CMD ( HERMON_HCR_HW2SW_MPT ),
0, NULL, index, NULL );
}
static inline int
hermon_cmd_write_mtt ( struct hermon *hermon,
const struct hermonprm_write_mtt *write_mtt ) {
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_WRITE_MTT,
1, sizeof ( *write_mtt ) ),
0, write_mtt, 1, NULL );
}
static inline int
hermon_cmd_map_eq ( struct hermon *hermon, unsigned long index_map,
const struct hermonprm_event_mask *mask ) {
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_EQ,
0, sizeof ( *mask ) ),
0, mask, index_map, NULL );
}
static inline int
hermon_cmd_sw2hw_eq ( struct hermon *hermon, unsigned int index,
const struct hermonprm_eqc *eqctx ) {
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_EQ,
1, sizeof ( *eqctx ) ),
0, eqctx, index, NULL );
}
static inline int
hermon_cmd_hw2sw_eq ( struct hermon *hermon, unsigned int index,
struct hermonprm_eqc *eqctx ) {
return hermon_cmd ( hermon,
HERMON_HCR_OUT_CMD ( HERMON_HCR_HW2SW_EQ,
1, sizeof ( *eqctx ) ),
1, NULL, index, eqctx );
}
static inline int
hermon_cmd_query_eq ( struct hermon *hermon, unsigned int index,
struct hermonprm_eqc *eqctx ) {
return hermon_cmd ( hermon,
HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_EQ,
1, sizeof ( *eqctx ) ),
0, NULL, index, eqctx );
}
static inline int
hermon_cmd_sw2hw_cq ( struct hermon *hermon, unsigned long cqn,
const struct hermonprm_completion_queue_context *cqctx ){
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_CQ,
1, sizeof ( *cqctx ) ),
0, cqctx, cqn, NULL );
}
static inline int
hermon_cmd_hw2sw_cq ( struct hermon *hermon, unsigned long cqn,
struct hermonprm_completion_queue_context *cqctx ) {
return hermon_cmd ( hermon,
HERMON_HCR_OUT_CMD ( HERMON_HCR_HW2SW_CQ,
1, sizeof ( *cqctx ) ),
0, NULL, cqn, cqctx );
}
static inline int
hermon_cmd_query_cq ( struct hermon *hermon, unsigned long cqn,
struct hermonprm_completion_queue_context *cqctx ) {
return hermon_cmd ( hermon,
HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_CQ,
1, sizeof ( *cqctx ) ),
0, NULL, cqn, cqctx );
}
static inline int
hermon_cmd_rst2init_qp ( struct hermon *hermon, unsigned long qpn,
const struct hermonprm_qp_ee_state_transitions *ctx ){
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_RST2INIT_QP,
1, sizeof ( *ctx ) ),
0, ctx, qpn, NULL );
}
static inline int
hermon_cmd_init2rtr_qp ( struct hermon *hermon, unsigned long qpn,
const struct hermonprm_qp_ee_state_transitions *ctx ){
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_INIT2RTR_QP,
1, sizeof ( *ctx ) ),
0, ctx, qpn, NULL );
}
static inline int
hermon_cmd_rtr2rts_qp ( struct hermon *hermon, unsigned long qpn,
const struct hermonprm_qp_ee_state_transitions *ctx ) {
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_RTR2RTS_QP,
1, sizeof ( *ctx ) ),
0, ctx, qpn, NULL );
}
static inline int
hermon_cmd_rts2rts_qp ( struct hermon *hermon, unsigned long qpn,
const struct hermonprm_qp_ee_state_transitions *ctx ) {
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_RTS2RTS_QP,
1, sizeof ( *ctx ) ),
0, ctx, qpn, NULL );
}
static inline int
hermon_cmd_2rst_qp ( struct hermon *hermon, unsigned long qpn ) {
return hermon_cmd ( hermon,
HERMON_HCR_VOID_CMD ( HERMON_HCR_2RST_QP ),
0x03, NULL, qpn, NULL );
}
static inline int
hermon_cmd_query_qp ( struct hermon *hermon, unsigned long qpn,
struct hermonprm_qp_ee_state_transitions *ctx ) {
return hermon_cmd ( hermon,
HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_QP,
1, sizeof ( *ctx ) ),
0, NULL, qpn, ctx );
}
static inline int
hermon_cmd_conf_special_qp ( struct hermon *hermon, unsigned int internal_qps,
unsigned long base_qpn ) {
return hermon_cmd ( hermon,
HERMON_HCR_VOID_CMD ( HERMON_HCR_CONF_SPECIAL_QP ),
internal_qps, NULL, base_qpn, NULL );
}
static inline int
hermon_cmd_mad_ifc ( struct hermon *hermon, unsigned int port,
union hermonprm_mad *mad ) {
return hermon_cmd ( hermon,
HERMON_HCR_INOUT_CMD ( HERMON_HCR_MAD_IFC,
1, sizeof ( *mad ),
1, sizeof ( *mad ) ),
0x03, mad, port, mad );
}
static inline int
hermon_cmd_read_mcg ( struct hermon *hermon, unsigned int index,
struct hermonprm_mcg_entry *mcg ) {
return hermon_cmd ( hermon,
HERMON_HCR_OUT_CMD ( HERMON_HCR_READ_MCG,
1, sizeof ( *mcg ) ),
0, NULL, index, mcg );
}
static inline int
hermon_cmd_write_mcg ( struct hermon *hermon, unsigned int index,
const struct hermonprm_mcg_entry *mcg ) {
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_WRITE_MCG,
1, sizeof ( *mcg ) ),
0, mcg, index, NULL );
}
static inline int
hermon_cmd_mgid_hash ( struct hermon *hermon, const union ib_gid *gid,
struct hermonprm_mgm_hash *hash ) {
return hermon_cmd ( hermon,
HERMON_HCR_INOUT_CMD ( HERMON_HCR_MGID_HASH,
1, sizeof ( *gid ),
0, sizeof ( *hash ) ),
0, gid, 0, hash );
}
static inline int
hermon_cmd_mod_stat_cfg ( struct hermon *hermon, unsigned int mode,
unsigned int input_mod,
struct hermonprm_scalar_parameter *portion ) {
return hermon_cmd ( hermon,
HERMON_HCR_INOUT_CMD ( HERMON_HCR_MOD_STAT_CFG,
0, sizeof ( *portion ),
0, sizeof ( *portion ) ),
mode, portion, input_mod, portion );
}
static inline int
hermon_cmd_query_port ( struct hermon *hermon, unsigned int port,
struct hermonprm_query_port_cap *query_port ) {
return hermon_cmd ( hermon,
HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_PORT,
1, sizeof ( *query_port ) ),
0, NULL, port, query_port );
}
static inline int
hermon_cmd_sense_port ( struct hermon *hermon, unsigned int port,
struct hermonprm_sense_port *port_type ) {
return hermon_cmd ( hermon,
HERMON_HCR_OUT_CMD ( HERMON_HCR_SENSE_PORT,
0, sizeof ( *port_type ) ),
0, NULL, port, port_type );
}
static inline int
hermon_cmd_run_fw ( struct hermon *hermon ) {
return hermon_cmd ( hermon,
HERMON_HCR_VOID_CMD ( HERMON_HCR_RUN_FW ),
0, NULL, 0, NULL );
}
static inline int
hermon_cmd_unmap_icm ( struct hermon *hermon, unsigned int page_count,
const struct hermonprm_scalar_parameter *offset ) {
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_UNMAP_ICM,
0, sizeof ( *offset ) ),
0, offset, page_count, NULL );
}
static inline int
hermon_cmd_map_icm ( struct hermon *hermon,
const struct hermonprm_virtual_physical_mapping *map ) {
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_ICM,
1, sizeof ( *map ) ),
0, map, 1, NULL );
}
static inline int
hermon_cmd_unmap_icm_aux ( struct hermon *hermon ) {
return hermon_cmd ( hermon,
HERMON_HCR_VOID_CMD ( HERMON_HCR_UNMAP_ICM_AUX ),
0, NULL, 0, NULL );
}
static inline int
hermon_cmd_map_icm_aux ( struct hermon *hermon,
const struct hermonprm_virtual_physical_mapping *map ) {
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_ICM_AUX,
1, sizeof ( *map ) ),
0, map, 1, NULL );
}
static inline int
hermon_cmd_set_icm_size ( struct hermon *hermon,
const struct hermonprm_scalar_parameter *icm_size,
struct hermonprm_scalar_parameter *icm_aux_size ) {
return hermon_cmd ( hermon,
HERMON_HCR_INOUT_CMD ( HERMON_HCR_SET_ICM_SIZE,
0, sizeof ( *icm_size ),
0, sizeof (*icm_aux_size) ),
0, icm_size, 0, icm_aux_size );
}
static inline int
hermon_cmd_unmap_fa ( struct hermon *hermon ) {
return hermon_cmd ( hermon,
HERMON_HCR_VOID_CMD ( HERMON_HCR_UNMAP_FA ),
0, NULL, 0, NULL );
}
static inline int
hermon_cmd_map_fa ( struct hermon *hermon,
const struct hermonprm_virtual_physical_mapping *map ) {
return hermon_cmd ( hermon,
HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_FA,
1, sizeof ( *map ) ),
0, map, 1, NULL );
}
/***************************************************************************
*
* Memory translation table operations
*
***************************************************************************
*/
/**
* Allocate MTT entries
*
* @v hermon Hermon device
* @v memory Memory to map into MTT
* @v len Length of memory to map
* @v mtt MTT descriptor to fill in
* @ret rc Return status code
*/
static int hermon_alloc_mtt ( struct hermon *hermon,
const void *memory, size_t len,
struct hermon_mtt *mtt ) {
struct hermonprm_write_mtt write_mtt;
physaddr_t start;
physaddr_t addr;
unsigned int page_offset;
unsigned int num_pages;
int mtt_offset;
unsigned int mtt_base_addr;
unsigned int i;
int rc;
/* Find available MTT entries */
start = virt_to_phys ( memory );
page_offset = ( start & ( HERMON_PAGE_SIZE - 1 ) );
start -= page_offset;
len += page_offset;
num_pages = ( ( len + HERMON_PAGE_SIZE - 1 ) / HERMON_PAGE_SIZE );
mtt_offset = hermon_bitmask_alloc ( hermon->mtt_inuse, HERMON_MAX_MTTS,
num_pages );
if ( mtt_offset < 0 ) {
rc = mtt_offset;
DBGC ( hermon, "Hermon %p could not allocate %d MTT entries: "
"%s\n", hermon, num_pages, strerror ( rc ) );
goto err_mtt_offset;
}
mtt_base_addr = ( ( hermon->cap.reserved_mtts + mtt_offset ) *
hermon->cap.mtt_entry_size );
addr = start;
/* Fill in MTT structure */
mtt->mtt_offset = mtt_offset;
mtt->num_pages = num_pages;
mtt->mtt_base_addr = mtt_base_addr;
mtt->page_offset = page_offset;
/* Construct and issue WRITE_MTT commands */
for ( i = 0 ; i < num_pages ; i++ ) {
memset ( &write_mtt, 0, sizeof ( write_mtt ) );
MLX_FILL_1 ( &write_mtt.mtt_base_addr, 1,
value, mtt_base_addr );
MLX_FILL_H ( &write_mtt.mtt, 0, ptag_h, addr );
MLX_FILL_2 ( &write_mtt.mtt, 1,
p, 1,
ptag_l, ( addr >> 3 ) );
if ( ( rc = hermon_cmd_write_mtt ( hermon,
&write_mtt ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not write MTT at %x: "
"%s\n", hermon, mtt_base_addr,
strerror ( rc ) );
goto err_write_mtt;
}
addr += HERMON_PAGE_SIZE;
mtt_base_addr += hermon->cap.mtt_entry_size;
}
DBGC ( hermon, "Hermon %p MTT entries [%#x,%#x] for "
"[%08lx,%08lx,%08lx,%08lx)\n", hermon, mtt->mtt_offset,
( mtt->mtt_offset + mtt->num_pages - 1 ), start,
( start + page_offset ), ( start + len ), addr );
return 0;
err_write_mtt:
hermon_bitmask_free ( hermon->mtt_inuse, mtt_offset, num_pages );
err_mtt_offset:
return rc;
}
/**
* Free MTT entries
*
* @v hermon Hermon device
* @v mtt MTT descriptor
*/
static void hermon_free_mtt ( struct hermon *hermon,
struct hermon_mtt *mtt ) {
DBGC ( hermon, "Hermon %p MTT entries [%#x,%#x] freed\n",
hermon, mtt->mtt_offset,
( mtt->mtt_offset + mtt->num_pages - 1 ) );
hermon_bitmask_free ( hermon->mtt_inuse, mtt->mtt_offset,
mtt->num_pages );
}
/***************************************************************************
*
* Static configuration operations
*
***************************************************************************
*/
/**
* Calculate offset within static configuration
*
* @v field Field
* @ret offset Offset
*/
#define HERMON_MOD_STAT_CFG_OFFSET( field ) \
( ( MLX_BIT_OFFSET ( struct hermonprm_mod_stat_cfg_st, field ) / 8 ) \
& ~( sizeof ( struct hermonprm_scalar_parameter ) - 1 ) )
/**
* Query or modify static configuration
*
* @v hermon Hermon device
* @v port Port
* @v mode Command mode
* @v offset Offset within static configuration
* @v stat_cfg Static configuration
* @ret rc Return status code
*/
static int hermon_mod_stat_cfg ( struct hermon *hermon, unsigned int port,
unsigned int mode, unsigned int offset,
struct hermonprm_mod_stat_cfg *stat_cfg ) {
struct hermonprm_scalar_parameter *portion =
( ( void * ) &stat_cfg->u.bytes[offset] );
struct hermonprm_mod_stat_cfg_input_mod mod;
int rc;
/* Sanity check */
assert ( ( offset % sizeof ( *portion ) ) == 0 );
/* Construct input modifier */
memset ( &mod, 0, sizeof ( mod ) );
MLX_FILL_2 ( &mod, 0,
portnum, port,
offset, offset );
/* Issue command */
if ( ( rc = hermon_cmd_mod_stat_cfg ( hermon, mode,
be32_to_cpu ( mod.u.dwords[0] ),
portion ) ) != 0 )
return rc;
return 0;
}
/***************************************************************************
*
* MAD operations
*
***************************************************************************
*/
/**
* Issue management datagram
*
* @v ibdev Infiniband device
* @v mad Management datagram
* @ret rc Return status code
*/
static int hermon_mad ( struct ib_device *ibdev, union ib_mad *mad ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
union hermonprm_mad mad_ifc;
int rc;
linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ),
mad_size_mismatch );
/* Copy in request packet */
memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) );
/* Issue MAD */
if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port,
&mad_ifc ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not issue MAD IFC: "
"%s\n", hermon, ibdev->port, strerror ( rc ) );
return rc;
}
/* Copy out reply packet */
memcpy ( mad, &mad_ifc.mad, sizeof ( *mad ) );
if ( mad->hdr.status != 0 ) {
DBGC ( hermon, "Hermon %p port %d MAD IFC status %04x\n",
hermon, ibdev->port, ntohs ( mad->hdr.status ) );
return -EIO;
}
return 0;
}
/***************************************************************************
*
* Completion queue operations
*
***************************************************************************
*/
/**
* Dump completion queue context (for debugging only)
*
* @v hermon Hermon device
* @v cq Completion queue
* @ret rc Return status code
*/
static __attribute__ (( unused )) int
hermon_dump_cqctx ( struct hermon *hermon, struct ib_completion_queue *cq ) {
struct hermonprm_completion_queue_context cqctx;
int rc;
memset ( &cqctx, 0, sizeof ( cqctx ) );
if ( ( rc = hermon_cmd_query_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) {
DBGC ( hermon, "Hermon %p CQN %#lx QUERY_CQ failed: %s\n",
hermon, cq->cqn, strerror ( rc ) );
return rc;
}
DBGC ( hermon, "Hermon %p CQN %#lx context:\n", hermon, cq->cqn );
DBGC_HDA ( hermon, 0, &cqctx, sizeof ( cqctx ) );
return 0;
}
/**
* Create completion queue
*
* @v ibdev Infiniband device
* @v cq Completion queue
* @ret rc Return status code
*/
static int hermon_create_cq ( struct ib_device *ibdev,
struct ib_completion_queue *cq ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermon_completion_queue *hermon_cq;
struct hermonprm_completion_queue_context cqctx;
int cqn_offset;
unsigned int i;
int rc;
/* Find a free completion queue number */
cqn_offset = hermon_bitmask_alloc ( hermon->cq_inuse,
HERMON_MAX_CQS, 1 );
if ( cqn_offset < 0 ) {
DBGC ( hermon, "Hermon %p out of completion queues\n",
hermon );
rc = cqn_offset;
goto err_cqn_offset;
}
cq->cqn = ( hermon->cap.reserved_cqs + cqn_offset );
/* Allocate control structures */
hermon_cq = zalloc ( sizeof ( *hermon_cq ) );
if ( ! hermon_cq ) {
DBGC ( hermon, "Hermon %p CQN %#lx could not allocate CQ\n",
hermon, cq->cqn );
rc = -ENOMEM;
goto err_hermon_cq;
}
/* Allocate doorbell */
hermon_cq->doorbell = malloc_phys ( sizeof ( hermon_cq->doorbell[0] ),
sizeof ( hermon_cq->doorbell[0] ) );
if ( ! hermon_cq->doorbell ) {
DBGC ( hermon, "Hermon %p CQN %#lx could not allocate "
"doorbell\n", hermon, cq->cqn );
rc = -ENOMEM;
goto err_doorbell;
}
memset ( hermon_cq->doorbell, 0, sizeof ( hermon_cq->doorbell[0] ) );
/* Allocate completion queue itself */
hermon_cq->cqe_size = ( cq->num_cqes * sizeof ( hermon_cq->cqe[0] ) );
hermon_cq->cqe = malloc_phys ( hermon_cq->cqe_size,
sizeof ( hermon_cq->cqe[0] ) );
if ( ! hermon_cq->cqe ) {
DBGC ( hermon, "Hermon %p CQN %#lx could not allocate CQEs\n",
hermon, cq->cqn );
rc = -ENOMEM;
goto err_cqe;
}
memset ( hermon_cq->cqe, 0, hermon_cq->cqe_size );
for ( i = 0 ; i < cq->num_cqes ; i++ ) {
MLX_FILL_1 ( &hermon_cq->cqe[i].normal, 7, owner, 1 );
}
barrier();
/* Allocate MTT entries */
if ( ( rc = hermon_alloc_mtt ( hermon, hermon_cq->cqe,
hermon_cq->cqe_size,
&hermon_cq->mtt ) ) != 0 ) {
DBGC ( hermon, "Hermon %p CQN %#lx could not allocate MTTs: "
"%s\n", hermon, cq->cqn, strerror ( rc ) );
goto err_alloc_mtt;
}
/* Hand queue over to hardware */
memset ( &cqctx, 0, sizeof ( cqctx ) );
MLX_FILL_1 ( &cqctx, 0, st, 0xa /* "Event fired" */ );
MLX_FILL_1 ( &cqctx, 2,
page_offset, ( hermon_cq->mtt.page_offset >> 5 ) );
MLX_FILL_2 ( &cqctx, 3,
usr_page, HERMON_UAR_NON_EQ_PAGE,
log_cq_size, fls ( cq->num_cqes - 1 ) );
MLX_FILL_1 ( &cqctx, 5, c_eqn, hermon->eq.eqn );
MLX_FILL_H ( &cqctx, 6, mtt_base_addr_h,
hermon_cq->mtt.mtt_base_addr );
MLX_FILL_1 ( &cqctx, 7, mtt_base_addr_l,
( hermon_cq->mtt.mtt_base_addr >> 3 ) );
MLX_FILL_H ( &cqctx, 14, db_record_addr_h,
virt_to_phys ( hermon_cq->doorbell ) );
MLX_FILL_1 ( &cqctx, 15, db_record_addr_l,
( virt_to_phys ( hermon_cq->doorbell ) >> 3 ) );
if ( ( rc = hermon_cmd_sw2hw_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) {
DBGC ( hermon, "Hermon %p CQN %#lx SW2HW_CQ failed: %s\n",
hermon, cq->cqn, strerror ( rc ) );
goto err_sw2hw_cq;
}
DBGC ( hermon, "Hermon %p CQN %#lx ring [%08lx,%08lx), doorbell "
"%08lx\n", hermon, cq->cqn, virt_to_phys ( hermon_cq->cqe ),
( virt_to_phys ( hermon_cq->cqe ) + hermon_cq->cqe_size ),
virt_to_phys ( hermon_cq->doorbell ) );
ib_cq_set_drvdata ( cq, hermon_cq );
return 0;
err_sw2hw_cq:
hermon_free_mtt ( hermon, &hermon_cq->mtt );
err_alloc_mtt:
free_phys ( hermon_cq->cqe, hermon_cq->cqe_size );
err_cqe:
free_phys ( hermon_cq->doorbell, sizeof ( hermon_cq->doorbell[0] ) );
err_doorbell:
free ( hermon_cq );
err_hermon_cq:
hermon_bitmask_free ( hermon->cq_inuse, cqn_offset, 1 );
err_cqn_offset:
return rc;
}
/**
* Destroy completion queue
*
* @v ibdev Infiniband device
* @v cq Completion queue
*/
static void hermon_destroy_cq ( struct ib_device *ibdev,
struct ib_completion_queue *cq ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermon_completion_queue *hermon_cq = ib_cq_get_drvdata ( cq );
struct hermonprm_completion_queue_context cqctx;
int cqn_offset;
int rc;
/* Take ownership back from hardware */
if ( ( rc = hermon_cmd_hw2sw_cq ( hermon, cq->cqn, &cqctx ) ) != 0 ) {
DBGC ( hermon, "Hermon %p CQN %#lx FATAL HW2SW_CQ failed: "
"%s\n", hermon, cq->cqn, strerror ( rc ) );
/* Leak memory and return; at least we avoid corruption */
return;
}
/* Free MTT entries */
hermon_free_mtt ( hermon, &hermon_cq->mtt );
/* Free memory */
free_phys ( hermon_cq->cqe, hermon_cq->cqe_size );
free_phys ( hermon_cq->doorbell, sizeof ( hermon_cq->doorbell[0] ) );
free ( hermon_cq );
/* Mark queue number as free */
cqn_offset = ( cq->cqn - hermon->cap.reserved_cqs );
hermon_bitmask_free ( hermon->cq_inuse, cqn_offset, 1 );
ib_cq_set_drvdata ( cq, NULL );
}
/***************************************************************************
*
* Queue pair operations
*
***************************************************************************
*/
/**
* Assign queue pair number
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @ret rc Return status code
*/
static int hermon_alloc_qpn ( struct ib_device *ibdev,
struct ib_queue_pair *qp ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
unsigned int port_offset;
int qpn_offset;
/* Calculate queue pair number */
port_offset = ( ibdev->port - HERMON_PORT_BASE );
switch ( qp->type ) {
case IB_QPT_SMI:
qp->qpn = ( hermon->special_qpn_base + port_offset );
return 0;
case IB_QPT_GSI:
qp->qpn = ( hermon->special_qpn_base + 2 + port_offset );
return 0;
case IB_QPT_UD:
case IB_QPT_RC:
case IB_QPT_ETH:
/* Find a free queue pair number */
qpn_offset = hermon_bitmask_alloc ( hermon->qp_inuse,
HERMON_MAX_QPS, 1 );
if ( qpn_offset < 0 ) {
DBGC ( hermon, "Hermon %p out of queue pairs\n",
hermon );
return qpn_offset;
}
qp->qpn = ( ( random() & HERMON_QPN_RANDOM_MASK ) |
( hermon->qpn_base + qpn_offset ) );
return 0;
default:
DBGC ( hermon, "Hermon %p unsupported QP type %d\n",
hermon, qp->type );
return -ENOTSUP;
}
}
/**
* Free queue pair number
*
* @v ibdev Infiniband device
* @v qp Queue pair
*/
static void hermon_free_qpn ( struct ib_device *ibdev,
struct ib_queue_pair *qp ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
int qpn_offset;
qpn_offset = ( ( qp->qpn & ~HERMON_QPN_RANDOM_MASK )
- hermon->qpn_base );
if ( qpn_offset >= 0 )
hermon_bitmask_free ( hermon->qp_inuse, qpn_offset, 1 );
}
/**
* Calculate transmission rate
*
* @v av Address vector
* @ret hermon_rate Hermon rate
*/
static unsigned int hermon_rate ( struct ib_address_vector *av ) {
return ( ( ( av->rate >= IB_RATE_2_5 ) && ( av->rate <= IB_RATE_120 ) )
? ( av->rate + 5 ) : 0 );
}
/**
* Calculate schedule queue
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @ret sched_queue Schedule queue
*/
static unsigned int hermon_sched_queue ( struct ib_device *ibdev,
struct ib_queue_pair *qp ) {
return ( ( ( qp->type == IB_QPT_SMI ) ?
HERMON_SCHED_QP0 : HERMON_SCHED_DEFAULT ) |
( ( ibdev->port - 1 ) << 6 ) );
}
/** Queue pair transport service type map */
static uint8_t hermon_qp_st[] = {
[IB_QPT_SMI] = HERMON_ST_MLX,
[IB_QPT_GSI] = HERMON_ST_MLX,
[IB_QPT_UD] = HERMON_ST_UD,
[IB_QPT_RC] = HERMON_ST_RC,
[IB_QPT_ETH] = HERMON_ST_MLX,
};
/**
* Dump queue pair context (for debugging only)
*
* @v hermon Hermon device
* @v qp Queue pair
* @ret rc Return status code
*/
static __attribute__ (( unused )) int
hermon_dump_qpctx ( struct hermon *hermon, struct ib_queue_pair *qp ) {
struct hermonprm_qp_ee_state_transitions qpctx;
int rc;
memset ( &qpctx, 0, sizeof ( qpctx ) );
if ( ( rc = hermon_cmd_query_qp ( hermon, qp->qpn, &qpctx ) ) != 0 ) {
DBGC ( hermon, "Hermon %p QPN %#lx QUERY_QP failed: %s\n",
hermon, qp->qpn, strerror ( rc ) );
return rc;
}
DBGC ( hermon, "Hermon %p QPN %#lx context:\n", hermon, qp->qpn );
DBGC_HDA ( hermon, 0, &qpctx.u.dwords[2], ( sizeof ( qpctx ) - 8 ) );
return 0;
}
/**
* Create queue pair
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @ret rc Return status code
*/
static int hermon_create_qp ( struct ib_device *ibdev,
struct ib_queue_pair *qp ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermon_queue_pair *hermon_qp;
struct hermonprm_qp_ee_state_transitions qpctx;
struct hermonprm_wqe_segment_data_ptr *data;
unsigned int i;
int rc;
/* Calculate queue pair number */
if ( ( rc = hermon_alloc_qpn ( ibdev, qp ) ) != 0 )
goto err_alloc_qpn;
/* Allocate control structures */
hermon_qp = zalloc ( sizeof ( *hermon_qp ) );
if ( ! hermon_qp ) {
DBGC ( hermon, "Hermon %p QPN %#lx could not allocate QP\n",
hermon, qp->qpn );
rc = -ENOMEM;
goto err_hermon_qp;
}
/* Allocate doorbells */
hermon_qp->recv.doorbell =
malloc_phys ( sizeof ( hermon_qp->recv.doorbell[0] ),
sizeof ( hermon_qp->recv.doorbell[0] ) );
if ( ! hermon_qp->recv.doorbell ) {
DBGC ( hermon, "Hermon %p QPN %#lx could not allocate "
"doorbell\n", hermon, qp->qpn );
rc = -ENOMEM;
goto err_recv_doorbell;
}
memset ( hermon_qp->recv.doorbell, 0,
sizeof ( hermon_qp->recv.doorbell[0] ) );
hermon_qp->send.doorbell =
( hermon->uar + HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE +
HERMON_DB_POST_SND_OFFSET );
/* Allocate work queue buffer */
hermon_qp->send.num_wqes = ( qp->send.num_wqes /* headroom */ + 1 +
( 2048 / sizeof ( hermon_qp->send.wqe[0] ) ) );
hermon_qp->send.num_wqes =
( 1 << fls ( hermon_qp->send.num_wqes - 1 ) ); /* round up */
hermon_qp->send.wqe_size = ( hermon_qp->send.num_wqes *
sizeof ( hermon_qp->send.wqe[0] ) );
hermon_qp->recv.wqe_size = ( qp->recv.num_wqes *
sizeof ( hermon_qp->recv.wqe[0] ) );
if ( ( qp->type == IB_QPT_SMI ) || ( qp->type == IB_QPT_GSI ) ||
( qp->type == IB_QPT_UD ) ) {
hermon_qp->recv.grh_size = ( qp->recv.num_wqes *
sizeof ( hermon_qp->recv.grh[0] ));
}
hermon_qp->wqe_size = ( hermon_qp->send.wqe_size +
hermon_qp->recv.wqe_size +
hermon_qp->recv.grh_size );
hermon_qp->wqe = malloc_phys ( hermon_qp->wqe_size,
sizeof ( hermon_qp->send.wqe[0] ) );
if ( ! hermon_qp->wqe ) {
DBGC ( hermon, "Hermon %p QPN %#lx could not allocate WQEs\n",
hermon, qp->qpn );
rc = -ENOMEM;
goto err_alloc_wqe;
}
hermon_qp->send.wqe = hermon_qp->wqe;
hermon_qp->recv.wqe = ( hermon_qp->wqe + hermon_qp->send.wqe_size );
if ( hermon_qp->recv.grh_size ) {
hermon_qp->recv.grh = ( hermon_qp->wqe +
hermon_qp->send.wqe_size +
hermon_qp->recv.wqe_size );
}
/* Initialise work queue entries */
memset ( hermon_qp->send.wqe, 0xff, hermon_qp->send.wqe_size );
memset ( hermon_qp->recv.wqe, 0, hermon_qp->recv.wqe_size );
data = &hermon_qp->recv.wqe[0].recv.data[0];
for ( i = 0 ; i < ( hermon_qp->recv.wqe_size / sizeof ( *data ) ); i++){
MLX_FILL_1 ( data, 1, l_key, HERMON_INVALID_LKEY );
data++;
}
/* Allocate MTT entries */
if ( ( rc = hermon_alloc_mtt ( hermon, hermon_qp->wqe,
hermon_qp->wqe_size,
&hermon_qp->mtt ) ) != 0 ) {
DBGC ( hermon, "Hermon %p QPN %#lx could not allocate MTTs: "
"%s\n", hermon, qp->qpn, strerror ( rc ) );
goto err_alloc_mtt;
}
/* Transition queue to INIT state */
memset ( &qpctx, 0, sizeof ( qpctx ) );
MLX_FILL_2 ( &qpctx, 2,
qpc_eec_data.pm_state, HERMON_PM_STATE_MIGRATED,
qpc_eec_data.st, hermon_qp_st[qp->type] );
MLX_FILL_1 ( &qpctx, 3, qpc_eec_data.pd, HERMON_GLOBAL_PD );
MLX_FILL_4 ( &qpctx, 4,
qpc_eec_data.log_rq_size, fls ( qp->recv.num_wqes - 1 ),
qpc_eec_data.log_rq_stride,
( fls ( sizeof ( hermon_qp->recv.wqe[0] ) - 1 ) - 4 ),
qpc_eec_data.log_sq_size,
fls ( hermon_qp->send.num_wqes - 1 ),
qpc_eec_data.log_sq_stride,
( fls ( sizeof ( hermon_qp->send.wqe[0] ) - 1 ) - 4 ) );
MLX_FILL_1 ( &qpctx, 5,
qpc_eec_data.usr_page, HERMON_UAR_NON_EQ_PAGE );
MLX_FILL_1 ( &qpctx, 33, qpc_eec_data.cqn_snd, qp->send.cq->cqn );
MLX_FILL_4 ( &qpctx, 38,
qpc_eec_data.rre, 1,
qpc_eec_data.rwe, 1,
qpc_eec_data.rae, 1,
qpc_eec_data.page_offset,
( hermon_qp->mtt.page_offset >> 6 ) );
MLX_FILL_1 ( &qpctx, 41, qpc_eec_data.cqn_rcv, qp->recv.cq->cqn );
MLX_FILL_H ( &qpctx, 42, qpc_eec_data.db_record_addr_h,
virt_to_phys ( hermon_qp->recv.doorbell ) );
MLX_FILL_1 ( &qpctx, 43, qpc_eec_data.db_record_addr_l,
( virt_to_phys ( hermon_qp->recv.doorbell ) >> 2 ) );
MLX_FILL_H ( &qpctx, 52, qpc_eec_data.mtt_base_addr_h,
hermon_qp->mtt.mtt_base_addr );
MLX_FILL_1 ( &qpctx, 53, qpc_eec_data.mtt_base_addr_l,
( hermon_qp->mtt.mtt_base_addr >> 3 ) );
if ( ( rc = hermon_cmd_rst2init_qp ( hermon, qp->qpn,
&qpctx ) ) != 0 ) {
DBGC ( hermon, "Hermon %p QPN %#lx RST2INIT_QP failed: %s\n",
hermon, qp->qpn, strerror ( rc ) );
goto err_rst2init_qp;
}
hermon_qp->state = HERMON_QP_ST_INIT;
DBGC ( hermon, "Hermon %p QPN %#lx send ring [%08lx,%08lx), doorbell "
"%08lx\n", hermon, qp->qpn,
virt_to_phys ( hermon_qp->send.wqe ),
( virt_to_phys ( hermon_qp->send.wqe ) +
hermon_qp->send.wqe_size ),
virt_to_phys ( hermon_qp->send.doorbell ) );
DBGC ( hermon, "Hermon %p QPN %#lx receive ring [%08lx,%08lx), "
"doorbell %08lx\n", hermon, qp->qpn,
virt_to_phys ( hermon_qp->recv.wqe ),
( virt_to_phys ( hermon_qp->recv.wqe ) +
hermon_qp->recv.wqe_size ),
virt_to_phys ( hermon_qp->recv.doorbell ) );
DBGC ( hermon, "Hermon %p QPN %#lx send CQN %#lx receive CQN %#lx\n",
hermon, qp->qpn, qp->send.cq->cqn, qp->recv.cq->cqn );
ib_qp_set_drvdata ( qp, hermon_qp );
return 0;
hermon_cmd_2rst_qp ( hermon, qp->qpn );
err_rst2init_qp:
hermon_free_mtt ( hermon, &hermon_qp->mtt );
err_alloc_mtt:
free_phys ( hermon_qp->wqe, hermon_qp->wqe_size );
err_alloc_wqe:
free_phys ( hermon_qp->recv.doorbell,
sizeof ( hermon_qp->recv.doorbell[0] ) );
err_recv_doorbell:
free ( hermon_qp );
err_hermon_qp:
hermon_free_qpn ( ibdev, qp );
err_alloc_qpn:
return rc;
}
/**
* Modify queue pair
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @ret rc Return status code
*/
static int hermon_modify_qp ( struct ib_device *ibdev,
struct ib_queue_pair *qp ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp );
struct hermonprm_qp_ee_state_transitions qpctx;
int rc;
/* Transition queue to RTR state, if applicable */
if ( hermon_qp->state < HERMON_QP_ST_RTR ) {
memset ( &qpctx, 0, sizeof ( qpctx ) );
MLX_FILL_2 ( &qpctx, 4,
qpc_eec_data.mtu,
( ( qp->type == IB_QPT_ETH ) ?
HERMON_MTU_ETH : HERMON_MTU_2048 ),
qpc_eec_data.msg_max, 31 );
MLX_FILL_1 ( &qpctx, 7,
qpc_eec_data.remote_qpn_een, qp->av.qpn );
MLX_FILL_1 ( &qpctx, 9,
qpc_eec_data.primary_address_path.rlid,
qp->av.lid );
MLX_FILL_1 ( &qpctx, 10,
qpc_eec_data.primary_address_path.max_stat_rate,
hermon_rate ( &qp->av ) );
memcpy ( &qpctx.u.dwords[12], &qp->av.gid,
sizeof ( qp->av.gid ) );
MLX_FILL_1 ( &qpctx, 16,
qpc_eec_data.primary_address_path.sched_queue,
hermon_sched_queue ( ibdev, qp ) );
MLX_FILL_1 ( &qpctx, 39,
qpc_eec_data.next_rcv_psn, qp->recv.psn );
if ( ( rc = hermon_cmd_init2rtr_qp ( hermon, qp->qpn,
&qpctx ) ) != 0 ) {
DBGC ( hermon, "Hermon %p QPN %#lx INIT2RTR_QP failed:"
" %s\n", hermon, qp->qpn, strerror ( rc ) );
return rc;
}
hermon_qp->state = HERMON_QP_ST_RTR;
}
/* Transition queue to RTS state */
if ( hermon_qp->state < HERMON_QP_ST_RTS ) {
memset ( &qpctx, 0, sizeof ( qpctx ) );
MLX_FILL_1 ( &qpctx, 10,
qpc_eec_data.primary_address_path.ack_timeout,
14 /* 4.096us * 2^(14) = 67ms */ );
MLX_FILL_2 ( &qpctx, 30,
qpc_eec_data.retry_count, HERMON_RETRY_MAX,
qpc_eec_data.rnr_retry, HERMON_RETRY_MAX );
MLX_FILL_1 ( &qpctx, 32,
qpc_eec_data.next_send_psn, qp->send.psn );
if ( ( rc = hermon_cmd_rtr2rts_qp ( hermon, qp->qpn,
&qpctx ) ) != 0 ) {
DBGC ( hermon, "Hermon %p QPN %#lx RTR2RTS_QP failed: "
"%s\n", hermon, qp->qpn, strerror ( rc ) );
return rc;
}
hermon_qp->state = HERMON_QP_ST_RTS;
}
/* Update parameters in RTS state */
memset ( &qpctx, 0, sizeof ( qpctx ) );
MLX_FILL_1 ( &qpctx, 0, opt_param_mask, HERMON_QP_OPT_PARAM_QKEY );
MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
if ( ( rc = hermon_cmd_rts2rts_qp ( hermon, qp->qpn, &qpctx ) ) != 0 ){
DBGC ( hermon, "Hermon %p QPN %#lx RTS2RTS_QP failed: %s\n",
hermon, qp->qpn, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Destroy queue pair
*
* @v ibdev Infiniband device
* @v qp Queue pair
*/
static void hermon_destroy_qp ( struct ib_device *ibdev,
struct ib_queue_pair *qp ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp );
int rc;
/* Take ownership back from hardware */
if ( ( rc = hermon_cmd_2rst_qp ( hermon, qp->qpn ) ) != 0 ) {
DBGC ( hermon, "Hermon %p QPN %#lx FATAL 2RST_QP failed: %s\n",
hermon, qp->qpn, strerror ( rc ) );
/* Leak memory and return; at least we avoid corruption */
return;
}
/* Free MTT entries */
hermon_free_mtt ( hermon, &hermon_qp->mtt );
/* Free memory */
free_phys ( hermon_qp->wqe, hermon_qp->wqe_size );
free_phys ( hermon_qp->recv.doorbell,
sizeof ( hermon_qp->recv.doorbell[0] ) );
free ( hermon_qp );
/* Mark queue number as free */
hermon_free_qpn ( ibdev, qp );
ib_qp_set_drvdata ( qp, NULL );
}
/***************************************************************************
*
* Work request operations
*
***************************************************************************
*/
/**
* Construct UD send work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v dest Destination address vector
* @v iobuf I/O buffer
* @v wqe Send work queue entry
* @ret opcode Control opcode
*/
static __attribute__ (( unused )) unsigned int
hermon_fill_nop_send_wqe ( struct ib_device *ibdev __unused,
struct ib_queue_pair *qp __unused,
struct ib_address_vector *dest __unused,
struct io_buffer *iobuf __unused,
union hermon_send_wqe *wqe ) {
MLX_FILL_1 ( &wqe->ctrl, 1, ds, ( sizeof ( wqe->ctrl ) / 16 ) );
MLX_FILL_1 ( &wqe->ctrl, 2, c, 0x03 /* generate completion */ );
return HERMON_OPCODE_NOP;
}
/**
* Construct UD send work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v dest Destination address vector
* @v iobuf I/O buffer
* @v wqe Send work queue entry
* @ret opcode Control opcode
*/
static unsigned int
hermon_fill_ud_send_wqe ( struct ib_device *ibdev,
struct ib_queue_pair *qp __unused,
struct ib_address_vector *dest,
struct io_buffer *iobuf,
union hermon_send_wqe *wqe ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
MLX_FILL_1 ( &wqe->ud.ctrl, 1, ds,
( ( offsetof ( typeof ( wqe->ud ), data[1] ) / 16 ) ) );
MLX_FILL_1 ( &wqe->ud.ctrl, 2, c, 0x03 /* generate completion */ );
MLX_FILL_2 ( &wqe->ud.ud, 0,
ud_address_vector.pd, HERMON_GLOBAL_PD,
ud_address_vector.port_number, ibdev->port );
MLX_FILL_2 ( &wqe->ud.ud, 1,
ud_address_vector.rlid, dest->lid,
ud_address_vector.g, dest->gid_present );
MLX_FILL_1 ( &wqe->ud.ud, 2,
ud_address_vector.max_stat_rate, hermon_rate ( dest ) );
MLX_FILL_1 ( &wqe->ud.ud, 3, ud_address_vector.sl, dest->sl );
memcpy ( &wqe->ud.ud.u.dwords[4], &dest->gid, sizeof ( dest->gid ) );
MLX_FILL_1 ( &wqe->ud.ud, 8, destination_qp, dest->qpn );
MLX_FILL_1 ( &wqe->ud.ud, 9, q_key, dest->qkey );
MLX_FILL_1 ( &wqe->ud.data[0], 0, byte_count, iob_len ( iobuf ) );
MLX_FILL_1 ( &wqe->ud.data[0], 1, l_key, hermon->lkey );
MLX_FILL_H ( &wqe->ud.data[0], 2,
local_address_h, virt_to_bus ( iobuf->data ) );
MLX_FILL_1 ( &wqe->ud.data[0], 3,
local_address_l, virt_to_bus ( iobuf->data ) );
return HERMON_OPCODE_SEND;
}
/**
* Construct MLX send work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v dest Destination address vector
* @v iobuf I/O buffer
* @v wqe Send work queue entry
* @ret opcode Control opcode
*/
static unsigned int
hermon_fill_mlx_send_wqe ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct ib_address_vector *dest,
struct io_buffer *iobuf,
union hermon_send_wqe *wqe ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct io_buffer headers;
/* Construct IB headers */
iob_populate ( &headers, &wqe->mlx.headers, 0,
sizeof ( wqe->mlx.headers ) );
iob_reserve ( &headers, sizeof ( wqe->mlx.headers ) );
ib_push ( ibdev, &headers, qp, iob_len ( iobuf ), dest );
/* Fill work queue entry */
MLX_FILL_1 ( &wqe->mlx.ctrl, 1, ds,
( ( offsetof ( typeof ( wqe->mlx ), data[2] ) / 16 ) ) );
MLX_FILL_5 ( &wqe->mlx.ctrl, 2,
c, 0x03 /* generate completion */,
icrc, 0 /* generate ICRC */,
max_statrate, hermon_rate ( dest ),
slr, 0,
v15, ( ( qp->ext_qpn == IB_QPN_SMI ) ? 1 : 0 ) );
MLX_FILL_1 ( &wqe->mlx.ctrl, 3, rlid, dest->lid );
MLX_FILL_1 ( &wqe->mlx.data[0], 0,
byte_count, iob_len ( &headers ) );
MLX_FILL_1 ( &wqe->mlx.data[0], 1, l_key, hermon->lkey );
MLX_FILL_H ( &wqe->mlx.data[0], 2,
local_address_h, virt_to_bus ( headers.data ) );
MLX_FILL_1 ( &wqe->mlx.data[0], 3,
local_address_l, virt_to_bus ( headers.data ) );
MLX_FILL_1 ( &wqe->mlx.data[1], 0,
byte_count, ( iob_len ( iobuf ) + 4 /* ICRC */ ) );
MLX_FILL_1 ( &wqe->mlx.data[1], 1, l_key, hermon->lkey );
MLX_FILL_H ( &wqe->mlx.data[1], 2,
local_address_h, virt_to_bus ( iobuf->data ) );
MLX_FILL_1 ( &wqe->mlx.data[1], 3,
local_address_l, virt_to_bus ( iobuf->data ) );
return HERMON_OPCODE_SEND;
}
/**
* Construct RC send work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v dest Destination address vector
* @v iobuf I/O buffer
* @v wqe Send work queue entry
* @ret opcode Control opcode
*/
static unsigned int
hermon_fill_rc_send_wqe ( struct ib_device *ibdev,
struct ib_queue_pair *qp __unused,
struct ib_address_vector *dest __unused,
struct io_buffer *iobuf,
union hermon_send_wqe *wqe ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
MLX_FILL_1 ( &wqe->rc.ctrl, 1, ds,
( ( offsetof ( typeof ( wqe->rc ), data[1] ) / 16 ) ) );
MLX_FILL_1 ( &wqe->rc.ctrl, 2, c, 0x03 /* generate completion */ );
MLX_FILL_1 ( &wqe->rc.data[0], 0, byte_count, iob_len ( iobuf ) );
MLX_FILL_1 ( &wqe->rc.data[0], 1, l_key, hermon->lkey );
MLX_FILL_H ( &wqe->rc.data[0], 2,
local_address_h, virt_to_bus ( iobuf->data ) );
MLX_FILL_1 ( &wqe->rc.data[0], 3,
local_address_l, virt_to_bus ( iobuf->data ) );
return HERMON_OPCODE_SEND;
}
/**
* Construct Ethernet send work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v dest Destination address vector
* @v iobuf I/O buffer
* @v wqe Send work queue entry
* @ret opcode Control opcode
*/
static unsigned int
hermon_fill_eth_send_wqe ( struct ib_device *ibdev,
struct ib_queue_pair *qp __unused,
struct ib_address_vector *dest __unused,
struct io_buffer *iobuf,
union hermon_send_wqe *wqe ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
/* Fill work queue entry */
MLX_FILL_1 ( &wqe->eth.ctrl, 1, ds,
( ( offsetof ( typeof ( wqe->mlx ), data[1] ) / 16 ) ) );
MLX_FILL_2 ( &wqe->eth.ctrl, 2,
c, 0x03 /* generate completion */,
s, 1 /* inhibit ICRC */ );
MLX_FILL_1 ( &wqe->eth.data[0], 0,
byte_count, iob_len ( iobuf ) );
MLX_FILL_1 ( &wqe->eth.data[0], 1, l_key, hermon->lkey );
MLX_FILL_H ( &wqe->eth.data[0], 2,
local_address_h, virt_to_bus ( iobuf->data ) );
MLX_FILL_1 ( &wqe->eth.data[0], 3,
local_address_l, virt_to_bus ( iobuf->data ) );
return HERMON_OPCODE_SEND;
}
/** Work queue entry constructors */
static unsigned int
( * hermon_fill_send_wqe[] ) ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct ib_address_vector *dest,
struct io_buffer *iobuf,
union hermon_send_wqe *wqe ) = {
[IB_QPT_SMI] = hermon_fill_mlx_send_wqe,
[IB_QPT_GSI] = hermon_fill_mlx_send_wqe,
[IB_QPT_UD] = hermon_fill_ud_send_wqe,
[IB_QPT_RC] = hermon_fill_rc_send_wqe,
[IB_QPT_ETH] = hermon_fill_eth_send_wqe,
};
/**
* Post send work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v dest Destination address vector
* @v iobuf I/O buffer
* @ret rc Return status code
*/
static int hermon_post_send ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct ib_address_vector *dest,
struct io_buffer *iobuf ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp );
struct ib_work_queue *wq = &qp->send;
struct hermon_send_work_queue *hermon_send_wq = &hermon_qp->send;
union hermon_send_wqe *wqe;
union hermonprm_doorbell_register db_reg;
unsigned long wqe_idx_mask;
unsigned long wqe_idx;
unsigned int owner;
unsigned int opcode;
/* Allocate work queue entry */
wqe_idx = ( wq->next_idx & ( hermon_send_wq->num_wqes - 1 ) );
owner = ( ( wq->next_idx & hermon_send_wq->num_wqes ) ? 1 : 0 );
wqe_idx_mask = ( wq->num_wqes - 1 );
if ( wq->iobufs[ wqe_idx & wqe_idx_mask ] ) {
DBGC ( hermon, "Hermon %p QPN %#lx send queue full",
hermon, qp->qpn );
return -ENOBUFS;
}
wq->iobufs[ wqe_idx & wqe_idx_mask ] = iobuf;
wqe = &hermon_send_wq->wqe[wqe_idx];
/* Construct work queue entry */
memset ( ( ( ( void * ) wqe ) + 4 /* avoid ctrl.owner */ ), 0,
( sizeof ( *wqe ) - 4 ) );
assert ( qp->type < ( sizeof ( hermon_fill_send_wqe ) /
sizeof ( hermon_fill_send_wqe[0] ) ) );
assert ( hermon_fill_send_wqe[qp->type] != NULL );
opcode = hermon_fill_send_wqe[qp->type] ( ibdev, qp, dest, iobuf, wqe );
barrier();
MLX_FILL_2 ( &wqe->ctrl, 0,
opcode, opcode,
owner, owner );
DBGCP ( hermon, "Hermon %p QPN %#lx posting send WQE %#lx:\n",
hermon, qp->qpn, wqe_idx );
DBGCP_HDA ( hermon, virt_to_phys ( wqe ), wqe, sizeof ( *wqe ) );
/* Ring doorbell register */
MLX_FILL_1 ( &db_reg.send, 0, qn, qp->qpn );
barrier();
writel ( db_reg.dword[0], hermon_send_wq->doorbell );
/* Update work queue's index */
wq->next_idx++;
return 0;
}
/**
* Post receive work queue entry
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v iobuf I/O buffer
* @ret rc Return status code
*/
static int hermon_post_recv ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
struct io_buffer *iobuf ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermon_queue_pair *hermon_qp = ib_qp_get_drvdata ( qp );
struct ib_work_queue *wq = &qp->recv;
struct hermon_recv_work_queue *hermon_recv_wq = &hermon_qp->recv;
struct hermonprm_recv_wqe *wqe;
struct hermonprm_wqe_segment_data_ptr *data;
struct ib_global_route_header *grh;
unsigned int wqe_idx_mask;
/* Allocate work queue entry */
wqe_idx_mask = ( wq->num_wqes - 1 );
if ( wq->iobufs[wq->next_idx & wqe_idx_mask] ) {
DBGC ( hermon, "Hermon %p QPN %#lx receive queue full",
hermon, qp->qpn );
return -ENOBUFS;
}
wq->iobufs[wq->next_idx & wqe_idx_mask] = iobuf;
wqe = &hermon_recv_wq->wqe[wq->next_idx & wqe_idx_mask].recv;
/* Construct work queue entry */
data = &wqe->data[0];
if ( hermon_qp->recv.grh ) {
grh = &hermon_qp->recv.grh[wq->next_idx & wqe_idx_mask];
MLX_FILL_1 ( data, 0, byte_count, sizeof ( *grh ) );
MLX_FILL_1 ( data, 1, l_key, hermon->lkey );
MLX_FILL_H ( data, 2, local_address_h, virt_to_bus ( grh ) );
MLX_FILL_1 ( data, 3, local_address_l, virt_to_bus ( grh ) );
data++;
}
MLX_FILL_1 ( data, 0, byte_count, iob_tailroom ( iobuf ) );
MLX_FILL_1 ( data, 1, l_key, hermon->lkey );
MLX_FILL_H ( data, 2, local_address_h, virt_to_bus ( iobuf->data ) );
MLX_FILL_1 ( data, 3, local_address_l, virt_to_bus ( iobuf->data ) );
/* Update work queue's index */
wq->next_idx++;
/* Update doorbell record */
barrier();
MLX_FILL_1 ( hermon_recv_wq->doorbell, 0, receive_wqe_counter,
( wq->next_idx & 0xffff ) );
return 0;
}
/**
* Handle completion
*
* @v ibdev Infiniband device
* @v cq Completion queue
* @v cqe Hardware completion queue entry
* @ret rc Return status code
*/
static int hermon_complete ( struct ib_device *ibdev,
struct ib_completion_queue *cq,
union hermonprm_completion_entry *cqe ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermon_queue_pair *hermon_qp;
struct ib_work_queue *wq;
struct ib_queue_pair *qp;
struct io_buffer *iobuf;
struct ib_address_vector recv_dest;
struct ib_address_vector recv_source;
struct ib_global_route_header *grh;
struct ib_address_vector *source;
unsigned int opcode;
unsigned long qpn;
int is_send;
unsigned long wqe_idx;
unsigned long wqe_idx_mask;
size_t len;
int rc = 0;
/* Parse completion */
qpn = MLX_GET ( &cqe->normal, qpn );
is_send = MLX_GET ( &cqe->normal, s_r );
opcode = MLX_GET ( &cqe->normal, opcode );
if ( opcode >= HERMON_OPCODE_RECV_ERROR ) {
/* "s" field is not valid for error opcodes */
is_send = ( opcode == HERMON_OPCODE_SEND_ERROR );
DBGC ( hermon, "Hermon %p CQN %#lx syndrome %x vendor %x\n",
hermon, cq->cqn, MLX_GET ( &cqe->error, syndrome ),
MLX_GET ( &cqe->error, vendor_error_syndrome ) );
rc = -EIO;
/* Don't return immediately; propagate error to completer */
}
/* Identify work queue */
wq = ib_find_wq ( cq, qpn, is_send );
if ( ! wq ) {
DBGC ( hermon, "Hermon %p CQN %#lx unknown %s QPN %#lx\n",
hermon, cq->cqn, ( is_send ? "send" : "recv" ), qpn );
return -EIO;
}
qp = wq->qp;
hermon_qp = ib_qp_get_drvdata ( qp );
/* Identify work queue entry */
wqe_idx = MLX_GET ( &cqe->normal, wqe_counter );
wqe_idx_mask = ( wq->num_wqes - 1 );
DBGCP ( hermon, "Hermon %p CQN %#lx QPN %#lx %s WQE %#lx completed:\n",
hermon, cq->cqn, qp->qpn, ( is_send ? "send" : "recv" ),
wqe_idx );
DBGCP_HDA ( hermon, virt_to_phys ( cqe ), cqe, sizeof ( *cqe ) );
/* Identify I/O buffer */
iobuf = wq->iobufs[ wqe_idx & wqe_idx_mask ];
if ( ! iobuf ) {
DBGC ( hermon, "Hermon %p CQN %#lx QPN %#lx empty %s WQE "
"%#lx\n", hermon, cq->cqn, qp->qpn,
( is_send ? "send" : "recv" ), wqe_idx );
return -EIO;
}
wq->iobufs[ wqe_idx & wqe_idx_mask ] = NULL;
if ( is_send ) {
/* Hand off to completion handler */
ib_complete_send ( ibdev, qp, iobuf, rc );
} else {
/* Set received length */
len = MLX_GET ( &cqe->normal, byte_cnt );
memset ( &recv_dest, 0, sizeof ( recv_dest ) );
recv_dest.qpn = qpn;
memset ( &recv_source, 0, sizeof ( recv_source ) );
switch ( qp->type ) {
case IB_QPT_SMI:
case IB_QPT_GSI:
case IB_QPT_UD:
/* Locate corresponding GRH */
assert ( hermon_qp->recv.grh != NULL );
grh = &hermon_qp->recv.grh[ wqe_idx & wqe_idx_mask ];
len -= sizeof ( *grh );
/* Construct address vector */
source = &recv_source;
source->qpn = MLX_GET ( &cqe->normal, srq_rqpn );
source->lid = MLX_GET ( &cqe->normal, slid_smac47_32 );
source->sl = MLX_GET ( &cqe->normal, sl );
recv_dest.gid_present = source->gid_present =
MLX_GET ( &cqe->normal, g );
memcpy ( &recv_dest.gid, &grh->dgid,
sizeof ( recv_dest.gid ) );
memcpy ( &source->gid, &grh->sgid,
sizeof ( source->gid ) );
break;
case IB_QPT_RC:
source = &qp->av;
break;
case IB_QPT_ETH:
/* Construct address vector */
source = &recv_source;
source->vlan_present = MLX_GET ( &cqe->normal, vlan );
source->vlan = MLX_GET ( &cqe->normal, vid );
break;
default:
assert ( 0 );
return -EINVAL;
}
assert ( len <= iob_tailroom ( iobuf ) );
iob_put ( iobuf, len );
/* Hand off to completion handler */
ib_complete_recv ( ibdev, qp, &recv_dest, source, iobuf, rc );
}
return rc;
}
/**
* Poll completion queue
*
* @v ibdev Infiniband device
* @v cq Completion queue
*/
static void hermon_poll_cq ( struct ib_device *ibdev,
struct ib_completion_queue *cq ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermon_completion_queue *hermon_cq = ib_cq_get_drvdata ( cq );
union hermonprm_completion_entry *cqe;
unsigned int cqe_idx_mask;
int rc;
while ( 1 ) {
/* Look for completion entry */
cqe_idx_mask = ( cq->num_cqes - 1 );
cqe = &hermon_cq->cqe[cq->next_idx & cqe_idx_mask];
if ( MLX_GET ( &cqe->normal, owner ) ^
( ( cq->next_idx & cq->num_cqes ) ? 1 : 0 ) ) {
/* Entry still owned by hardware; end of poll */
break;
}
/* Handle completion */
if ( ( rc = hermon_complete ( ibdev, cq, cqe ) ) != 0 ) {
DBGC ( hermon, "Hermon %p CQN %#lx failed to complete:"
" %s\n", hermon, cq->cqn, strerror ( rc ) );
DBGC_HDA ( hermon, virt_to_phys ( cqe ),
cqe, sizeof ( *cqe ) );
}
/* Update completion queue's index */
cq->next_idx++;
/* Update doorbell record */
MLX_FILL_1 ( hermon_cq->doorbell, 0, update_ci,
( cq->next_idx & 0x00ffffffUL ) );
}
}
/***************************************************************************
*
* Event queues
*
***************************************************************************
*/
/**
* Dump event queue context (for debugging only)
*
* @v hermon Hermon device
* @v hermon_eq Event queue
* @ret rc Return status code
*/
static __attribute__ (( unused )) int
hermon_dump_eqctx ( struct hermon *hermon,
struct hermon_event_queue *hermon_eq ) {
struct hermonprm_eqc eqctx;
int rc;
memset ( &eqctx, 0, sizeof ( eqctx ) );
if ( ( rc = hermon_cmd_query_eq ( hermon, hermon_eq->eqn,
&eqctx ) ) != 0 ) {
DBGC ( hermon, "Hermon %p EQN %#lx QUERY_EQ failed: %s\n",
hermon, hermon_eq->eqn, strerror ( rc ) );
return rc;
}
DBGC ( hermon, "Hermon %p EQN %#lx context:\n",
hermon, hermon_eq->eqn );
DBGC_HDA ( hermon, 0, &eqctx, sizeof ( eqctx ) );
return 0;
}
/**
* Dump unconsumed event queue entries (for debugging only)
*
* @v hermon Hermon device
* @v hermon_eq Event queue
* @ret rc Return status code
*/
static __attribute__ (( unused )) int
hermon_dump_eqes ( struct hermon *hermon,
struct hermon_event_queue *hermon_eq ) {
struct hermonprm_eqc eqctx;
union hermonprm_event_entry *eqe;
unsigned int mask;
unsigned int prod;
unsigned int cons;
unsigned int idx;
int rc;
memset ( &eqctx, 0, sizeof ( eqctx ) );
if ( ( rc = hermon_cmd_query_eq ( hermon, hermon_eq->eqn,
&eqctx ) ) != 0 ) {
DBGC ( hermon, "Hermon %p EQN %#lx QUERY_EQ failed: %s\n",
hermon, hermon_eq->eqn, strerror ( rc ) );
return rc;
}
mask = ( HERMON_NUM_EQES - 1 );
prod = MLX_GET ( &eqctx, producer_counter ) & mask;
cons = MLX_GET ( &eqctx, consumer_counter ) & mask;
idx = hermon_eq->next_idx;
if ( ( idx & mask ) != ( cons & mask ) ) {
DBGC ( hermon, "Hermon %p EQN %#lx mismatch: SW %#x != HW "
"%#x\n", hermon, hermon_eq->eqn, idx, cons );
}
for ( ; ( idx & mask ) != ( prod & mask ) ; idx++ ) {
eqe = &hermon_eq->eqe[idx & mask];
DBGC ( hermon, "Hermon %p EQN %#lx event %#x owner %d type "
"%#02x:%#02x\n", hermon, hermon_eq->eqn, idx,
MLX_GET ( &eqe->generic, owner ),
MLX_GET ( &eqe->generic, event_type ),
MLX_GET ( &eqe->generic, event_sub_type ) );
DBGC_HDA ( hermon, 0, eqe, sizeof ( *eqe ) );
}
return 0;
}
/**
* Create event queue
*
* @v hermon Hermon device
* @ret rc Return status code
*/
static int hermon_create_eq ( struct hermon *hermon ) {
struct hermon_event_queue *hermon_eq = &hermon->eq;
struct hermonprm_eqc eqctx;
struct hermonprm_event_mask mask;
unsigned int i;
int rc;
/* Select event queue number */
hermon_eq->eqn = ( 4 * hermon->cap.reserved_uars );
if ( hermon_eq->eqn < hermon->cap.reserved_eqs )
hermon_eq->eqn = hermon->cap.reserved_eqs;
/* Calculate doorbell address */
hermon_eq->doorbell =
( hermon->uar + HERMON_DB_EQ_OFFSET ( hermon_eq->eqn ) );
/* Allocate event queue itself */
hermon_eq->eqe_size =
( HERMON_NUM_EQES * sizeof ( hermon_eq->eqe[0] ) );
hermon_eq->eqe = malloc_phys ( hermon_eq->eqe_size,
sizeof ( hermon_eq->eqe[0] ) );
if ( ! hermon_eq->eqe ) {
DBGC ( hermon, "Hermon %p EQN %#lx could not allocate EQEs\n",
hermon, hermon_eq->eqn );
rc = -ENOMEM;
goto err_eqe;
}
memset ( hermon_eq->eqe, 0, hermon_eq->eqe_size );
for ( i = 0 ; i < HERMON_NUM_EQES ; i++ ) {
MLX_FILL_1 ( &hermon_eq->eqe[i].generic, 7, owner, 1 );
}
barrier();
/* Allocate MTT entries */
if ( ( rc = hermon_alloc_mtt ( hermon, hermon_eq->eqe,
hermon_eq->eqe_size,
&hermon_eq->mtt ) ) != 0 ) {
DBGC ( hermon, "Hermon %p EQN %#lx could not allocate MTTs: "
"%s\n", hermon, hermon_eq->eqn, strerror ( rc ) );
goto err_alloc_mtt;
}
/* Hand queue over to hardware */
memset ( &eqctx, 0, sizeof ( eqctx ) );
MLX_FILL_2 ( &eqctx, 0,
st, 0xa /* "Fired" */,
oi, 1 );
MLX_FILL_1 ( &eqctx, 2,
page_offset, ( hermon_eq->mtt.page_offset >> 5 ) );
MLX_FILL_1 ( &eqctx, 3, log_eq_size, fls ( HERMON_NUM_EQES - 1 ) );
MLX_FILL_H ( &eqctx, 6, mtt_base_addr_h,
hermon_eq->mtt.mtt_base_addr );
MLX_FILL_1 ( &eqctx, 7, mtt_base_addr_l,
( hermon_eq->mtt.mtt_base_addr >> 3 ) );
if ( ( rc = hermon_cmd_sw2hw_eq ( hermon, hermon_eq->eqn,
&eqctx ) ) != 0 ) {
DBGC ( hermon, "Hermon %p EQN %#lx SW2HW_EQ failed: %s\n",
hermon, hermon_eq->eqn, strerror ( rc ) );
goto err_sw2hw_eq;
}
/* Map all events to this event queue */
memset ( &mask, 0xff, sizeof ( mask ) );
if ( ( rc = hermon_cmd_map_eq ( hermon,
( HERMON_MAP_EQ | hermon_eq->eqn ),
&mask ) ) != 0 ) {
DBGC ( hermon, "Hermon %p EQN %#lx MAP_EQ failed: %s\n",
hermon, hermon_eq->eqn, strerror ( rc ) );
goto err_map_eq;
}
DBGC ( hermon, "Hermon %p EQN %#lx ring [%08lx,%08lx), doorbell "
"%08lx\n", hermon, hermon_eq->eqn,
virt_to_phys ( hermon_eq->eqe ),
( virt_to_phys ( hermon_eq->eqe ) + hermon_eq->eqe_size ),
virt_to_phys ( hermon_eq->doorbell ) );
return 0;
err_map_eq:
hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn, &eqctx );
err_sw2hw_eq:
hermon_free_mtt ( hermon, &hermon_eq->mtt );
err_alloc_mtt:
free_phys ( hermon_eq->eqe, hermon_eq->eqe_size );
err_eqe:
memset ( hermon_eq, 0, sizeof ( *hermon_eq ) );
return rc;
}
/**
* Destroy event queue
*
* @v hermon Hermon device
*/
static void hermon_destroy_eq ( struct hermon *hermon ) {
struct hermon_event_queue *hermon_eq = &hermon->eq;
struct hermonprm_eqc eqctx;
struct hermonprm_event_mask mask;
int rc;
/* Unmap events from event queue */
memset ( &mask, 0xff, sizeof ( mask ) );
if ( ( rc = hermon_cmd_map_eq ( hermon,
( HERMON_UNMAP_EQ | hermon_eq->eqn ),
&mask ) ) != 0 ) {
DBGC ( hermon, "Hermon %p EQN %#lx FATAL MAP_EQ failed to "
"unmap: %s\n", hermon, hermon_eq->eqn, strerror ( rc ) );
/* Continue; HCA may die but system should survive */
}
/* Take ownership back from hardware */
if ( ( rc = hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn,
&eqctx ) ) != 0 ) {
DBGC ( hermon, "Hermon %p EQN %#lx FATAL HW2SW_EQ failed: %s\n",
hermon, hermon_eq->eqn, strerror ( rc ) );
/* Leak memory and return; at least we avoid corruption */
return;
}
/* Free MTT entries */
hermon_free_mtt ( hermon, &hermon_eq->mtt );
/* Free memory */
free_phys ( hermon_eq->eqe, hermon_eq->eqe_size );
memset ( hermon_eq, 0, sizeof ( *hermon_eq ) );
}
/**
* Handle port state event
*
* @v hermon Hermon device
* @v eqe Port state change event queue entry
*/
static void hermon_event_port_state_change ( struct hermon *hermon,
union hermonprm_event_entry *eqe){
unsigned int port;
int link_up;
/* Get port and link status */
port = ( MLX_GET ( &eqe->port_state_change, data.p ) - 1 );
link_up = ( MLX_GET ( &eqe->generic, event_sub_type ) & 0x04 );
DBGC ( hermon, "Hermon %p port %d link %s\n", hermon, ( port + 1 ),
( link_up ? "up" : "down" ) );
/* Sanity check */
if ( port >= hermon->cap.num_ports ) {
DBGC ( hermon, "Hermon %p port %d does not exist!\n",
hermon, ( port + 1 ) );
return;
}
/* Notify device of port state change */
hermon->port[port].type->state_change ( hermon, &hermon->port[port],
link_up );
}
/**
* Handle port management event
*
* @v hermon Hermon device
* @v eqe Port management change event queue entry
*/
static void hermon_event_port_mgmnt_change ( struct hermon *hermon,
union hermonprm_event_entry *eqe){
unsigned int port;
/* Get port */
port = ( MLX_GET ( &eqe->port_mgmnt_change, port ) - 1 );
DBGC ( hermon, "Hermon %p port %d management change\n",
hermon, ( port + 1 ) );
/* Sanity check */
if ( port >= hermon->cap.num_ports ) {
DBGC ( hermon, "Hermon %p port %d does not exist!\n",
hermon, ( port + 1 ) );
return;
}
/* Update MAD parameters */
ib_smc_update ( hermon->port[port].ibdev, hermon_mad );
}
/**
* Poll event queue
*
* @v ibdev Infiniband device
*/
static void hermon_poll_eq ( struct ib_device *ibdev ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermon_event_queue *hermon_eq = &hermon->eq;
union hermonprm_event_entry *eqe;
union hermonprm_doorbell_register db_reg;
unsigned long now;
unsigned long elapsed;
unsigned int eqe_idx_mask;
unsigned int event_type;
/* No event is generated upon reaching INIT, so we must poll
* separately for link state changes while we remain DOWN.
*/
if ( ib_is_open ( ibdev ) &&
( ibdev->port_state == IB_PORT_STATE_DOWN ) ) {
now = currticks();
elapsed = ( now - hermon->last_poll );
if ( elapsed >= HERMON_LINK_POLL_INTERVAL ) {
hermon->last_poll = now;
ib_smc_update ( ibdev, hermon_mad );
}
}
/* Poll event queue */
while ( 1 ) {
/* Look for event entry */
eqe_idx_mask = ( HERMON_NUM_EQES - 1 );
eqe = &hermon_eq->eqe[hermon_eq->next_idx & eqe_idx_mask];
if ( MLX_GET ( &eqe->generic, owner ) ^
( ( hermon_eq->next_idx & HERMON_NUM_EQES ) ? 1 : 0 ) ) {
/* Entry still owned by hardware; end of poll */
break;
}
DBGCP ( hermon, "Hermon %p EQN %#lx event:\n",
hermon, hermon_eq->eqn );
DBGCP_HDA ( hermon, virt_to_phys ( eqe ),
eqe, sizeof ( *eqe ) );
/* Handle event */
event_type = MLX_GET ( &eqe->generic, event_type );
switch ( event_type ) {
case HERMON_EV_PORT_STATE_CHANGE:
hermon_event_port_state_change ( hermon, eqe );
break;
case HERMON_EV_PORT_MGMNT_CHANGE:
hermon_event_port_mgmnt_change ( hermon, eqe );
break;
default:
DBGC ( hermon, "Hermon %p EQN %#lx unrecognised event "
"type %#02x:%#02x\n",
hermon, hermon_eq->eqn, event_type,
MLX_GET ( &eqe->generic, event_sub_type ) );
DBGC_HDA ( hermon, virt_to_phys ( eqe ),
eqe, sizeof ( *eqe ) );
break;
}
/* Update event queue's index */
hermon_eq->next_idx++;
/* Ring doorbell */
MLX_FILL_1 ( &db_reg.event, 0,
ci, ( hermon_eq->next_idx & 0x00ffffffUL ) );
writel ( db_reg.dword[0], hermon_eq->doorbell );
}
}
/***************************************************************************
*
* Firmware control
*
***************************************************************************
*/
/**
* Map virtual to physical address for firmware usage
*
* @v hermon Hermon device
* @v map Mapping function
* @v va Virtual address
* @v pa Physical address
* @v len Length of region
* @ret rc Return status code
*/
static int hermon_map_vpm ( struct hermon *hermon,
int ( *map ) ( struct hermon *hermon,
const struct hermonprm_virtual_physical_mapping* ),
uint64_t va, physaddr_t pa, size_t len ) {
struct hermonprm_virtual_physical_mapping mapping;
physaddr_t start;
physaddr_t low;
physaddr_t high;
physaddr_t end;
size_t size;
int rc;
/* Sanity checks */
assert ( ( va & ( HERMON_PAGE_SIZE - 1 ) ) == 0 );
assert ( ( pa & ( HERMON_PAGE_SIZE - 1 ) ) == 0 );
assert ( ( len & ( HERMON_PAGE_SIZE - 1 ) ) == 0 );
assert ( len != 0 );
/* Calculate starting points */
start = pa;
end = ( start + len );
size = ( 1UL << ( fls ( start ^ end ) - 1 ) );
low = high = ( end & ~( size - 1 ) );
assert ( start < low );
assert ( high <= end );
/* These mappings tend to generate huge volumes of
* uninteresting debug data, which basically makes it
* impossible to use debugging otherwise.
*/
DBG_DISABLE ( DBGLVL_LOG | DBGLVL_EXTRA );
/* Map blocks in descending order of size */
while ( size >= HERMON_PAGE_SIZE ) {
/* Find the next candidate block */
if ( ( low - size ) >= start ) {
low -= size;
pa = low;
} else if ( high <= ( end - size ) ) {
pa = high;
high += size;
} else {
size >>= 1;
continue;
}
assert ( ( va & ( size - 1 ) ) == 0 );
assert ( ( pa & ( size - 1 ) ) == 0 );
/* Map this block */
memset ( &mapping, 0, sizeof ( mapping ) );
MLX_FILL_1 ( &mapping, 0, va_h, ( va >> 32 ) );
MLX_FILL_1 ( &mapping, 1, va_l, ( va >> 12 ) );
MLX_FILL_H ( &mapping, 2, pa_h, pa );
MLX_FILL_2 ( &mapping, 3,
log2size, ( ( fls ( size ) - 1 ) - 12 ),
pa_l, ( pa >> 12 ) );
if ( ( rc = map ( hermon, &mapping ) ) != 0 ) {
DBG_ENABLE ( DBGLVL_LOG | DBGLVL_EXTRA );
DBGC ( hermon, "Hermon %p could not map %08llx+%zx to "
"%08lx: %s\n",
hermon, va, size, pa, strerror ( rc ) );
return rc;
}
va += size;
}
assert ( low == start );
assert ( high == end );
DBG_ENABLE ( DBGLVL_LOG | DBGLVL_EXTRA );
return 0;
}
/**
* Start firmware running
*
* @v hermon Hermon device
* @ret rc Return status code
*/
static int hermon_start_firmware ( struct hermon *hermon ) {
struct hermonprm_query_fw fw;
unsigned int fw_pages;
size_t fw_len;
physaddr_t fw_base;
int rc;
/* Get firmware parameters */
if ( ( rc = hermon_cmd_query_fw ( hermon, &fw ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not query firmware: %s\n",
hermon, strerror ( rc ) );
goto err_query_fw;
}
DBGC ( hermon, "Hermon %p firmware version %d.%d.%d\n", hermon,
MLX_GET ( &fw, fw_rev_major ), MLX_GET ( &fw, fw_rev_minor ),
MLX_GET ( &fw, fw_rev_subminor ) );
fw_pages = MLX_GET ( &fw, fw_pages );
DBGC ( hermon, "Hermon %p requires %d pages (%d kB) for firmware\n",
hermon, fw_pages, ( fw_pages * 4 ) );
/* Allocate firmware pages and map firmware area */
fw_len = ( fw_pages * HERMON_PAGE_SIZE );
if ( ! hermon->firmware_area ) {
hermon->firmware_len = fw_len;
hermon->firmware_area = umalloc ( hermon->firmware_len );
if ( ! hermon->firmware_area ) {
DBGC ( hermon, "Hermon %p could not allocate firmware "
"area\n", hermon );
rc = -ENOMEM;
goto err_alloc_fa;
}
} else {
assert ( hermon->firmware_len == fw_len );
}
fw_base = user_to_phys ( hermon->firmware_area, 0 );
DBGC ( hermon, "Hermon %p firmware area at physical [%08lx,%08lx)\n",
hermon, fw_base, ( fw_base + fw_len ) );
if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_fa,
0, fw_base, fw_len ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not map firmware: %s\n",
hermon, strerror ( rc ) );
goto err_map_fa;
}
/* Start firmware */
if ( ( rc = hermon_cmd_run_fw ( hermon ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not run firmware: %s\n",
hermon, strerror ( rc ) );
goto err_run_fw;
}
DBGC ( hermon, "Hermon %p firmware started\n", hermon );
return 0;
err_run_fw:
err_map_fa:
hermon_cmd_unmap_fa ( hermon );
err_alloc_fa:
err_query_fw:
return rc;
}
/**
* Stop firmware running
*
* @v hermon Hermon device
*/
static void hermon_stop_firmware ( struct hermon *hermon ) {
int rc;
if ( ( rc = hermon_cmd_unmap_fa ( hermon ) ) != 0 ) {
DBGC ( hermon, "Hermon %p FATAL could not stop firmware: %s\n",
hermon, strerror ( rc ) );
/* Leak memory and return; at least we avoid corruption */
hermon->firmware_area = UNULL;
return;
}
}
/***************************************************************************
*
* Infinihost Context Memory management
*
***************************************************************************
*/
/**
* Get device limits
*
* @v hermon Hermon device
* @ret rc Return status code
*/
static int hermon_get_cap ( struct hermon *hermon ) {
struct hermonprm_query_dev_cap dev_cap;
int rc;
if ( ( rc = hermon_cmd_query_dev_cap ( hermon, &dev_cap ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not get device limits: %s\n",
hermon, strerror ( rc ) );
return rc;
}
hermon->cap.cmpt_entry_size = MLX_GET ( &dev_cap, c_mpt_entry_sz );
hermon->cap.reserved_qps =
( 1 << MLX_GET ( &dev_cap, log2_rsvd_qps ) );
hermon->cap.qpc_entry_size = MLX_GET ( &dev_cap, qpc_entry_sz );
hermon->cap.altc_entry_size = MLX_GET ( &dev_cap, altc_entry_sz );
hermon->cap.auxc_entry_size = MLX_GET ( &dev_cap, aux_entry_sz );
hermon->cap.reserved_srqs =
( 1 << MLX_GET ( &dev_cap, log2_rsvd_srqs ) );
hermon->cap.srqc_entry_size = MLX_GET ( &dev_cap, srq_entry_sz );
hermon->cap.reserved_cqs =
( 1 << MLX_GET ( &dev_cap, log2_rsvd_cqs ) );
hermon->cap.cqc_entry_size = MLX_GET ( &dev_cap, cqc_entry_sz );
hermon->cap.reserved_eqs = MLX_GET ( &dev_cap, num_rsvd_eqs );
if ( hermon->cap.reserved_eqs == 0 ) {
/* Backward compatibility */
hermon->cap.reserved_eqs =
( 1 << MLX_GET ( &dev_cap, log2_rsvd_eqs ) );
}
hermon->cap.eqc_entry_size = MLX_GET ( &dev_cap, eqc_entry_sz );
hermon->cap.reserved_mtts =
( 1 << MLX_GET ( &dev_cap, log2_rsvd_mtts ) );
hermon->cap.mtt_entry_size = MLX_GET ( &dev_cap, mtt_entry_sz );
hermon->cap.reserved_mrws =
( 1 << MLX_GET ( &dev_cap, log2_rsvd_mrws ) );
hermon->cap.dmpt_entry_size = MLX_GET ( &dev_cap, d_mpt_entry_sz );
hermon->cap.reserved_uars = MLX_GET ( &dev_cap, num_rsvd_uars );
hermon->cap.num_ports = MLX_GET ( &dev_cap, num_ports );
hermon->cap.dpdp = MLX_GET ( &dev_cap, dpdp );
/* Sanity check */
if ( hermon->cap.num_ports > HERMON_MAX_PORTS ) {
DBGC ( hermon, "Hermon %p has %d ports (only %d supported)\n",
hermon, hermon->cap.num_ports, HERMON_MAX_PORTS );
hermon->cap.num_ports = HERMON_MAX_PORTS;
}
return 0;
}
/**
* Align ICM table
*
* @v icm_offset Current ICM offset
* @v len ICM table length
* @ret icm_offset ICM offset
*/
static uint64_t icm_align ( uint64_t icm_offset, size_t len ) {
/* Round up to a multiple of the table size */
assert ( len == ( 1UL << ( fls ( len ) - 1 ) ) );
return ( ( icm_offset + len - 1 ) & ~( ( ( uint64_t ) len ) - 1 ) );
}
/**
* Map ICM (allocating if necessary)
*
* @v hermon Hermon device
* @v init_hca INIT_HCA structure to fill in
* @ret rc Return status code
*/
static int hermon_map_icm ( struct hermon *hermon,
struct hermonprm_init_hca *init_hca ) {
struct hermonprm_scalar_parameter icm_size;
struct hermonprm_scalar_parameter icm_aux_size;
uint64_t icm_offset = 0;
unsigned int log_num_qps, log_num_srqs, log_num_cqs, log_num_eqs;
unsigned int log_num_mtts, log_num_mpts, log_num_mcs;
size_t cmpt_max_len;
size_t icm_len, icm_aux_len;
size_t len;
physaddr_t icm_phys;
int i;
int rc;
/*
* Start by carving up the ICM virtual address space
*
*/
/* Calculate number of each object type within ICM */
log_num_qps = fls ( hermon->cap.reserved_qps +
HERMON_RSVD_SPECIAL_QPS + HERMON_MAX_QPS - 1 );
log_num_srqs = fls ( hermon->cap.reserved_srqs - 1 );
log_num_cqs = fls ( hermon->cap.reserved_cqs + HERMON_MAX_CQS - 1 );
log_num_eqs = fls ( hermon->cap.reserved_eqs + HERMON_MAX_EQS - 1 );
log_num_mtts = fls ( hermon->cap.reserved_mtts + HERMON_MAX_MTTS - 1 );
log_num_mpts = fls ( hermon->cap.reserved_mrws + 1 - 1 );
log_num_mcs = HERMON_LOG_MULTICAST_HASH_SIZE;
/* ICM starts with the cMPT tables, which are sparse */
cmpt_max_len = ( HERMON_CMPT_MAX_ENTRIES *
( ( uint64_t ) hermon->cap.cmpt_entry_size ) );
len = ( ( ( ( 1 << log_num_qps ) * hermon->cap.cmpt_entry_size ) +
HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
hermon->icm_map[HERMON_ICM_QP_CMPT].offset = icm_offset;
hermon->icm_map[HERMON_ICM_QP_CMPT].len = len;
icm_offset += cmpt_max_len;
len = ( ( ( ( 1 << log_num_srqs ) * hermon->cap.cmpt_entry_size ) +
HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
hermon->icm_map[HERMON_ICM_SRQ_CMPT].offset = icm_offset;
hermon->icm_map[HERMON_ICM_SRQ_CMPT].len = len;
icm_offset += cmpt_max_len;
len = ( ( ( ( 1 << log_num_cqs ) * hermon->cap.cmpt_entry_size ) +
HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
hermon->icm_map[HERMON_ICM_CQ_CMPT].offset = icm_offset;
hermon->icm_map[HERMON_ICM_CQ_CMPT].len = len;
icm_offset += cmpt_max_len;
len = ( ( ( ( 1 << log_num_eqs ) * hermon->cap.cmpt_entry_size ) +
HERMON_PAGE_SIZE - 1 ) & ~( HERMON_PAGE_SIZE - 1 ) );
hermon->icm_map[HERMON_ICM_EQ_CMPT].offset = icm_offset;
hermon->icm_map[HERMON_ICM_EQ_CMPT].len = len;
icm_offset += cmpt_max_len;
hermon->icm_map[HERMON_ICM_OTHER].offset = icm_offset;
/* Queue pair contexts */
len = ( ( 1 << log_num_qps ) * hermon->cap.qpc_entry_size );
icm_offset = icm_align ( icm_offset, len );
MLX_FILL_1 ( init_hca, 12,
qpc_eec_cqc_eqc_rdb_parameters.qpc_base_addr_h,
( icm_offset >> 32 ) );
MLX_FILL_2 ( init_hca, 13,
qpc_eec_cqc_eqc_rdb_parameters.qpc_base_addr_l,
( icm_offset >> 5 ),
qpc_eec_cqc_eqc_rdb_parameters.log_num_of_qp,
log_num_qps );
DBGC ( hermon, "Hermon %p ICM QPC is %d x %#zx at [%08llx,%08llx)\n",
hermon, ( 1 << log_num_qps ), hermon->cap.qpc_entry_size,
icm_offset, ( icm_offset + len ) );
icm_offset += len;
/* Extended alternate path contexts */
len = ( ( 1 << log_num_qps ) * hermon->cap.altc_entry_size );
icm_offset = icm_align ( icm_offset, len );
MLX_FILL_1 ( init_hca, 24,
qpc_eec_cqc_eqc_rdb_parameters.altc_base_addr_h,
( icm_offset >> 32 ) );
MLX_FILL_1 ( init_hca, 25,
qpc_eec_cqc_eqc_rdb_parameters.altc_base_addr_l,
icm_offset );
DBGC ( hermon, "Hermon %p ICM ALTC is %d x %#zx at [%08llx,%08llx)\n",
hermon, ( 1 << log_num_qps ), hermon->cap.altc_entry_size,
icm_offset, ( icm_offset + len ) );
icm_offset += len;
/* Extended auxiliary contexts */
len = ( ( 1 << log_num_qps ) * hermon->cap.auxc_entry_size );
icm_offset = icm_align ( icm_offset, len );
MLX_FILL_1 ( init_hca, 28,
qpc_eec_cqc_eqc_rdb_parameters.auxc_base_addr_h,
( icm_offset >> 32 ) );
MLX_FILL_1 ( init_hca, 29,
qpc_eec_cqc_eqc_rdb_parameters.auxc_base_addr_l,
icm_offset );
DBGC ( hermon, "Hermon %p ICM AUXC is %d x %#zx at [%08llx,%08llx)\n",
hermon, ( 1 << log_num_qps ), hermon->cap.auxc_entry_size,
icm_offset, ( icm_offset + len ) );
icm_offset += len;
/* Shared receive queue contexts */
len = ( ( 1 << log_num_srqs ) * hermon->cap.srqc_entry_size );
icm_offset = icm_align ( icm_offset, len );
MLX_FILL_1 ( init_hca, 18,
qpc_eec_cqc_eqc_rdb_parameters.srqc_base_addr_h,
( icm_offset >> 32 ) );
MLX_FILL_2 ( init_hca, 19,
qpc_eec_cqc_eqc_rdb_parameters.srqc_base_addr_l,
( icm_offset >> 5 ),
qpc_eec_cqc_eqc_rdb_parameters.log_num_of_srq,
log_num_srqs );
DBGC ( hermon, "Hermon %p ICM SRQC is %d x %#zx at [%08llx,%08llx)\n",
hermon, ( 1 << log_num_srqs ), hermon->cap.srqc_entry_size,
icm_offset, ( icm_offset + len ) );
icm_offset += len;
/* Completion queue contexts */
len = ( ( 1 << log_num_cqs ) * hermon->cap.cqc_entry_size );
icm_offset = icm_align ( icm_offset, len );
MLX_FILL_1 ( init_hca, 20,
qpc_eec_cqc_eqc_rdb_parameters.cqc_base_addr_h,
( icm_offset >> 32 ) );
MLX_FILL_2 ( init_hca, 21,
qpc_eec_cqc_eqc_rdb_parameters.cqc_base_addr_l,
( icm_offset >> 5 ),
qpc_eec_cqc_eqc_rdb_parameters.log_num_of_cq,
log_num_cqs );
DBGC ( hermon, "Hermon %p ICM CQC is %d x %#zx at [%08llx,%08llx)\n",
hermon, ( 1 << log_num_cqs ), hermon->cap.cqc_entry_size,
icm_offset, ( icm_offset + len ) );
icm_offset += len;
/* Event queue contexts */
len = ( ( 1 << log_num_eqs ) * hermon->cap.eqc_entry_size );
icm_offset = icm_align ( icm_offset, len );
MLX_FILL_1 ( init_hca, 32,
qpc_eec_cqc_eqc_rdb_parameters.eqc_base_addr_h,
( icm_offset >> 32 ) );
MLX_FILL_2 ( init_hca, 33,
qpc_eec_cqc_eqc_rdb_parameters.eqc_base_addr_l,
( icm_offset >> 5 ),
qpc_eec_cqc_eqc_rdb_parameters.log_num_of_eq,
log_num_eqs );
DBGC ( hermon, "Hermon %p ICM EQC is %d x %#zx at [%08llx,%08llx)\n",
hermon, ( 1 << log_num_eqs ), hermon->cap.eqc_entry_size,
icm_offset, ( icm_offset + len ) );
icm_offset += len;
/* Memory translation table */
len = ( ( 1 << log_num_mtts ) * hermon->cap.mtt_entry_size );
icm_offset = icm_align ( icm_offset, len );
MLX_FILL_1 ( init_hca, 64,
tpt_parameters.mtt_base_addr_h, ( icm_offset >> 32 ) );
MLX_FILL_1 ( init_hca, 65,
tpt_parameters.mtt_base_addr_l, icm_offset );
DBGC ( hermon, "Hermon %p ICM MTT is %d x %#zx at [%08llx,%08llx)\n",
hermon, ( 1 << log_num_mtts ), hermon->cap.mtt_entry_size,
icm_offset, ( icm_offset + len ) );
icm_offset += len;
/* Memory protection table */
len = ( ( 1 << log_num_mpts ) * hermon->cap.dmpt_entry_size );
icm_offset = icm_align ( icm_offset, len );
MLX_FILL_1 ( init_hca, 60,
tpt_parameters.dmpt_base_adr_h, ( icm_offset >> 32 ) );
MLX_FILL_1 ( init_hca, 61,
tpt_parameters.dmpt_base_adr_l, icm_offset );
MLX_FILL_1 ( init_hca, 62,
tpt_parameters.log_dmpt_sz, log_num_mpts );
DBGC ( hermon, "Hermon %p ICM DMPT is %d x %#zx at [%08llx,%08llx)\n",
hermon, ( 1 << log_num_mpts ), hermon->cap.dmpt_entry_size,
icm_offset, ( icm_offset + len ) );
icm_offset += len;
/* Multicast table */
len = ( ( 1 << log_num_mcs ) * sizeof ( struct hermonprm_mcg_entry ) );
icm_offset = icm_align ( icm_offset, len );
MLX_FILL_1 ( init_hca, 48,
multicast_parameters.mc_base_addr_h,
( icm_offset >> 32 ) );
MLX_FILL_1 ( init_hca, 49,
multicast_parameters.mc_base_addr_l, icm_offset );
MLX_FILL_1 ( init_hca, 52,
multicast_parameters.log_mc_table_entry_sz,
fls ( sizeof ( struct hermonprm_mcg_entry ) - 1 ) );
MLX_FILL_1 ( init_hca, 53,
multicast_parameters.log_mc_table_hash_sz, log_num_mcs );
MLX_FILL_1 ( init_hca, 54,
multicast_parameters.log_mc_table_sz, log_num_mcs );
DBGC ( hermon, "Hermon %p ICM MC is %d x %#zx at [%08llx,%08llx)\n",
hermon, ( 1 << log_num_mcs ),
sizeof ( struct hermonprm_mcg_entry ),
icm_offset, ( icm_offset + len ) );
icm_offset += len;
hermon->icm_map[HERMON_ICM_OTHER].len =
( icm_offset - hermon->icm_map[HERMON_ICM_OTHER].offset );
/*
* Allocate and map physical memory for (portions of) ICM
*
* Map is:
* ICM AUX area (aligned to its own size)
* cMPT areas
* Other areas
*/
/* Calculate physical memory required for ICM */
icm_len = 0;
for ( i = 0 ; i < HERMON_ICM_NUM_REGIONS ; i++ ) {
icm_len += hermon->icm_map[i].len;
}
/* Get ICM auxiliary area size */
memset ( &icm_size, 0, sizeof ( icm_size ) );
MLX_FILL_1 ( &icm_size, 0, value_hi, ( icm_offset >> 32 ) );
MLX_FILL_1 ( &icm_size, 1, value, icm_offset );
if ( ( rc = hermon_cmd_set_icm_size ( hermon, &icm_size,
&icm_aux_size ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not set ICM size: %s\n",
hermon, strerror ( rc ) );
goto err_set_icm_size;
}
icm_aux_len = ( MLX_GET ( &icm_aux_size, value ) * HERMON_PAGE_SIZE );
/* Allocate ICM data and auxiliary area */
DBGC ( hermon, "Hermon %p requires %zd kB ICM and %zd kB AUX ICM\n",
hermon, ( icm_len / 1024 ), ( icm_aux_len / 1024 ) );
if ( ! hermon->icm ) {
hermon->icm_len = icm_len;
hermon->icm_aux_len = icm_aux_len;
hermon->icm = umalloc ( hermon->icm_aux_len + hermon->icm_len );
if ( ! hermon->icm ) {
DBGC ( hermon, "Hermon %p could not allocate ICM\n",
hermon );
rc = -ENOMEM;
goto err_alloc;
}
} else {
assert ( hermon->icm_len == icm_len );
assert ( hermon->icm_aux_len == icm_aux_len );
}
icm_phys = user_to_phys ( hermon->icm, 0 );
/* Map ICM auxiliary area */
DBGC ( hermon, "Hermon %p mapping ICM AUX => %08lx\n",
hermon, icm_phys );
if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_icm_aux,
0, icm_phys, icm_aux_len ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not map AUX ICM: %s\n",
hermon, strerror ( rc ) );
goto err_map_icm_aux;
}
icm_phys += icm_aux_len;
/* MAP ICM area */
for ( i = 0 ; i < HERMON_ICM_NUM_REGIONS ; i++ ) {
DBGC ( hermon, "Hermon %p mapping ICM %llx+%zx => %08lx\n",
hermon, hermon->icm_map[i].offset,
hermon->icm_map[i].len, icm_phys );
if ( ( rc = hermon_map_vpm ( hermon, hermon_cmd_map_icm,
hermon->icm_map[i].offset,
icm_phys,
hermon->icm_map[i].len ) ) != 0 ){
DBGC ( hermon, "Hermon %p could not map ICM: %s\n",
hermon, strerror ( rc ) );
goto err_map_icm;
}
icm_phys += hermon->icm_map[i].len;
}
return 0;
err_map_icm:
assert ( i == 0 ); /* We don't handle partial failure at present */
err_map_icm_aux:
hermon_cmd_unmap_icm_aux ( hermon );
err_alloc:
err_set_icm_size:
return rc;
}
/**
* Unmap ICM
*
* @v hermon Hermon device
*/
static void hermon_unmap_icm ( struct hermon *hermon ) {
struct hermonprm_scalar_parameter unmap_icm;
int i;
for ( i = ( HERMON_ICM_NUM_REGIONS - 1 ) ; i >= 0 ; i-- ) {
memset ( &unmap_icm, 0, sizeof ( unmap_icm ) );
MLX_FILL_1 ( &unmap_icm, 0, value_hi,
( hermon->icm_map[i].offset >> 32 ) );
MLX_FILL_1 ( &unmap_icm, 1, value,
hermon->icm_map[i].offset );
hermon_cmd_unmap_icm ( hermon,
( 1 << fls ( ( hermon->icm_map[i].len /
HERMON_PAGE_SIZE ) - 1)),
&unmap_icm );
}
hermon_cmd_unmap_icm_aux ( hermon );
}
/***************************************************************************
*
* Initialisation and teardown
*
***************************************************************************
*/
/**
* Reset device
*
* @v hermon Hermon device
* @ret rc Return status code
*/
static int hermon_reset ( struct hermon *hermon ) {
struct pci_device *pci = hermon->pci;
struct pci_config_backup backup;
static const uint8_t backup_exclude[] =
PCI_CONFIG_BACKUP_EXCLUDE ( 0x58, 0x5c );
uint16_t vendor;
unsigned int i;
/* Reset command interface toggle */
hermon->toggle = 0;
/* Perform device reset and preserve PCI configuration */
pci_backup ( pci, &backup, backup_exclude );
writel ( HERMON_RESET_MAGIC,
( hermon->config + HERMON_RESET_OFFSET ) );
/* Wait until device starts responding to configuration cycles */
for ( i = 0 ; i < HERMON_RESET_MAX_WAIT_MS ; i++ ) {
/* Read PCI vendor ID */
pci_read_config_word ( pci, PCI_VENDOR_ID, &vendor );
if ( vendor == pci->vendor ) {
/* Restore PCI configuration */
pci_restore ( pci, &backup, backup_exclude );
DBGC ( hermon, "Hermon %p reset after %dms\n",
hermon, i );
return 0;
}
/* Delay */
mdelay ( 1 );
}
DBGC ( hermon, "Hermon %p timed out waiting for reset\n", hermon );
return -ETIMEDOUT;
}
/**
* Set up memory protection table
*
* @v hermon Hermon device
* @ret rc Return status code
*/
static int hermon_setup_mpt ( struct hermon *hermon ) {
struct hermonprm_mpt mpt;
uint32_t key;
int rc;
/* Derive key */
key = ( hermon->cap.reserved_mrws | HERMON_MKEY_PREFIX );
hermon->lkey = ( ( key << 8 ) | ( key >> 24 ) );
/* Initialise memory protection table */
memset ( &mpt, 0, sizeof ( mpt ) );
MLX_FILL_7 ( &mpt, 0,
atomic, 1,
rw, 1,
rr, 1,
lw, 1,
lr, 1,
pa, 1,
r_w, 1 );
MLX_FILL_1 ( &mpt, 2, mem_key, key );
MLX_FILL_1 ( &mpt, 3,
pd, HERMON_GLOBAL_PD );
MLX_FILL_1 ( &mpt, 10, len64, 1 );
if ( ( rc = hermon_cmd_sw2hw_mpt ( hermon,
hermon->cap.reserved_mrws,
&mpt ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not set up MPT: %s\n",
hermon, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Unmap memory protection table
*
* @v hermon Hermon device
* @ret rc Return status code
*/
static int hermon_unmap_mpt ( struct hermon *hermon ) {
int rc;
if ( ( rc = hermon_cmd_hw2sw_mpt ( hermon,
hermon->cap.reserved_mrws ) ) != 0 ){
DBGC ( hermon, "Hermon %p could not unmap MPT: %s\n",
hermon, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Configure special queue pairs
*
* @v hermon Hermon device
* @ret rc Return status code
*/
static int hermon_configure_special_qps ( struct hermon *hermon ) {
int rc;
/* Special QP block must be aligned on its own size */
hermon->special_qpn_base = ( ( hermon->cap.reserved_qps +
HERMON_NUM_SPECIAL_QPS - 1 )
& ~( HERMON_NUM_SPECIAL_QPS - 1 ) );
hermon->qpn_base = ( hermon->special_qpn_base +
HERMON_NUM_SPECIAL_QPS );
DBGC ( hermon, "Hermon %p special QPs at [%lx,%lx]\n", hermon,
hermon->special_qpn_base, ( hermon->qpn_base - 1 ) );
/* Issue command to configure special QPs */
if ( ( rc = hermon_cmd_conf_special_qp ( hermon, 0x00,
hermon->special_qpn_base ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not configure special QPs: "
"%s\n", hermon, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Start Hermon device
*
* @v hermon Hermon device
* @v running Firmware is already running
* @ret rc Return status code
*/
static int hermon_start ( struct hermon *hermon, int running ) {
struct hermonprm_init_hca init_hca;
unsigned int i;
int rc;
/* Start firmware if not already running */
if ( ! running ) {
if ( ( rc = hermon_start_firmware ( hermon ) ) != 0 )
goto err_start_firmware;
}
/* Allocate and map ICM */
memset ( &init_hca, 0, sizeof ( init_hca ) );
if ( ( rc = hermon_map_icm ( hermon, &init_hca ) ) != 0 )
goto err_map_icm;
/* Initialise HCA */
MLX_FILL_1 ( &init_hca, 0, version, 0x02 /* "Must be 0x02" */ );
MLX_FILL_1 ( &init_hca, 5, udp, 1 );
MLX_FILL_1 ( &init_hca, 74, uar_parameters.log_max_uars, 8 );
if ( ( rc = hermon_cmd_init_hca ( hermon, &init_hca ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not initialise HCA: %s\n",
hermon, strerror ( rc ) );
goto err_init_hca;
}
/* Set up memory protection */
if ( ( rc = hermon_setup_mpt ( hermon ) ) != 0 )
goto err_setup_mpt;
for ( i = 0 ; i < hermon->cap.num_ports ; i++ )
hermon->port[i].ibdev->rdma_key = hermon->lkey;
/* Set up event queue */
if ( ( rc = hermon_create_eq ( hermon ) ) != 0 )
goto err_create_eq;
/* Configure special QPs */
if ( ( rc = hermon_configure_special_qps ( hermon ) ) != 0 )
goto err_conf_special_qps;
DBGC ( hermon, "Hermon %p device started\n", hermon );
return 0;
err_conf_special_qps:
hermon_destroy_eq ( hermon );
err_create_eq:
hermon_unmap_mpt ( hermon );
err_setup_mpt:
hermon_cmd_close_hca ( hermon );
err_init_hca:
hermon_unmap_icm ( hermon );
err_map_icm:
hermon_stop_firmware ( hermon );
err_start_firmware:
return rc;
}
/**
* Stop Hermon device
*
* @v hermon Hermon device
*/
static void hermon_stop ( struct hermon *hermon ) {
hermon_destroy_eq ( hermon );
hermon_unmap_mpt ( hermon );
hermon_cmd_close_hca ( hermon );
hermon_unmap_icm ( hermon );
hermon_stop_firmware ( hermon );
hermon_reset ( hermon );
}
/**
* Open Hermon device
*
* @v hermon Hermon device
* @ret rc Return status code
*/
static int hermon_open ( struct hermon *hermon ) {
int rc;
/* Start device if applicable */
if ( hermon->open_count == 0 ) {
if ( ( rc = hermon_start ( hermon, 0 ) ) != 0 )
return rc;
}
/* Increment open counter */
hermon->open_count++;
return 0;
}
/**
* Close Hermon device
*
* @v hermon Hermon device
*/
static void hermon_close ( struct hermon *hermon ) {
/* Decrement open counter */
assert ( hermon->open_count != 0 );
hermon->open_count--;
/* Stop device if applicable */
if ( hermon->open_count == 0 )
hermon_stop ( hermon );
}
/***************************************************************************
*
* Infiniband link-layer operations
*
***************************************************************************
*/
/**
* Initialise Infiniband link
*
* @v ibdev Infiniband device
* @ret rc Return status code
*/
static int hermon_ib_open ( struct ib_device *ibdev ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
union hermonprm_set_port set_port;
int rc;
/* Open hardware */
if ( ( rc = hermon_open ( hermon ) ) != 0 )
goto err_open;
/* Set port parameters */
memset ( &set_port, 0, sizeof ( set_port ) );
MLX_FILL_8 ( &set_port.ib, 0,
mmc, 1,
mvc, 1,
mp, 1,
mg, 1,
mtu_cap, IB_MTU_2048,
vl_cap, IB_VL_0,
rcm, 1,
lss, 1 );
MLX_FILL_2 ( &set_port.ib, 10,
max_pkey, 1,
max_gid, 1 );
MLX_FILL_1 ( &set_port.ib, 28,
link_speed_supported, 1 );
if ( ( rc = hermon_cmd_set_port ( hermon, 0, ibdev->port,
&set_port ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not set port: %s\n",
hermon, ibdev->port, strerror ( rc ) );
goto err_set_port;
}
/* Initialise port */
if ( ( rc = hermon_cmd_init_port ( hermon, ibdev->port ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not initialise port: "
"%s\n", hermon, ibdev->port, strerror ( rc ) );
goto err_init_port;
}
/* Update MAD parameters */
ib_smc_update ( ibdev, hermon_mad );
return 0;
err_init_port:
err_set_port:
hermon_close ( hermon );
err_open:
return rc;
}
/**
* Close Infiniband link
*
* @v ibdev Infiniband device
*/
static void hermon_ib_close ( struct ib_device *ibdev ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
int rc;
/* Close port */
if ( ( rc = hermon_cmd_close_port ( hermon, ibdev->port ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not close port: %s\n",
hermon, ibdev->port, strerror ( rc ) );
/* Nothing we can do about this */
}
/* Close hardware */
hermon_close ( hermon );
}
/**
* Inform embedded subnet management agent of a received MAD
*
* @v ibdev Infiniband device
* @v mad MAD
* @ret rc Return status code
*/
static int hermon_inform_sma ( struct ib_device *ibdev,
union ib_mad *mad ) {
int rc;
/* Send the MAD to the embedded SMA */
if ( ( rc = hermon_mad ( ibdev, mad ) ) != 0 )
return rc;
/* Update parameters held in software */
ib_smc_update ( ibdev, hermon_mad );
return 0;
}
/***************************************************************************
*
* Multicast group operations
*
***************************************************************************
*/
/**
* Attach to multicast group
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v gid Multicast GID
* @ret rc Return status code
*/
static int hermon_mcast_attach ( struct ib_device *ibdev,
struct ib_queue_pair *qp,
union ib_gid *gid ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermonprm_mgm_hash hash;
struct hermonprm_mcg_entry mcg;
unsigned int index;
int rc;
/* Generate hash table index */
if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not hash GID: %s\n",
hermon, strerror ( rc ) );
return rc;
}
index = MLX_GET ( &hash, hash );
/* Check for existing hash table entry */
if ( ( rc = hermon_cmd_read_mcg ( hermon, index, &mcg ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not read MCG %#x: %s\n",
hermon, index, strerror ( rc ) );
return rc;
}
if ( MLX_GET ( &mcg, hdr.members_count ) != 0 ) {
/* FIXME: this implementation allows only a single QP
* per multicast group, and doesn't handle hash
* collisions. Sufficient for IPoIB but may need to
* be extended in future.
*/
DBGC ( hermon, "Hermon %p MGID index %#x already in use\n",
hermon, index );
return -EBUSY;
}
/* Update hash table entry */
MLX_FILL_1 ( &mcg, 1, hdr.members_count, 1 );
MLX_FILL_1 ( &mcg, 8, qp[0].qpn, qp->qpn );
memcpy ( &mcg.u.dwords[4], gid, sizeof ( *gid ) );
if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n",
hermon, index, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Detach from multicast group
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v gid Multicast GID
*/
static void hermon_mcast_detach ( struct ib_device *ibdev,
struct ib_queue_pair *qp __unused,
union ib_gid *gid ) {
struct hermon *hermon = ib_get_drvdata ( ibdev );
struct hermonprm_mgm_hash hash;
struct hermonprm_mcg_entry mcg;
unsigned int index;
int rc;
/* Generate hash table index */
if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not hash GID: %s\n",
hermon, strerror ( rc ) );
return;
}
index = MLX_GET ( &hash, hash );
/* Clear hash table entry */
memset ( &mcg, 0, sizeof ( mcg ) );
if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n",
hermon, index, strerror ( rc ) );
return;
}
}
/** Hermon Infiniband operations */
static struct ib_device_operations hermon_ib_operations = {
.create_cq = hermon_create_cq,
.destroy_cq = hermon_destroy_cq,
.create_qp = hermon_create_qp,
.modify_qp = hermon_modify_qp,
.destroy_qp = hermon_destroy_qp,
.post_send = hermon_post_send,
.post_recv = hermon_post_recv,
.poll_cq = hermon_poll_cq,
.poll_eq = hermon_poll_eq,
.open = hermon_ib_open,
.close = hermon_ib_close,
.mcast_attach = hermon_mcast_attach,
.mcast_detach = hermon_mcast_detach,
.set_port_info = hermon_inform_sma,
.set_pkey_table = hermon_inform_sma,
};
/**
* Register Hermon Infiniband device
*
* @v hermon Hermon device
* @v port Hermon port
* @ret rc Return status code
*/
static int hermon_register_ibdev ( struct hermon *hermon,
struct hermon_port *port ) {
struct ib_device *ibdev = port->ibdev;
int rc;
/* Use Ethernet MAC as eIPoIB local EMAC */
memcpy ( ibdev->lemac, port->eth_mac.raw, ETH_ALEN );
/* Initialise parameters using SMC */
ib_smc_init ( ibdev, hermon_mad );
/* Register Infiniband device */
if ( ( rc = register_ibdev ( ibdev ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not register IB "
"device: %s\n", hermon, ibdev->port, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Handle Hermon Infiniband device port state change
*
* @v hermon Hermon device
* @v port Hermon port
* @v link_up Link is up
*/
static void hermon_state_change_ibdev ( struct hermon *hermon __unused,
struct hermon_port *port,
int link_up __unused ) {
struct ib_device *ibdev = port->ibdev;
/* Update MAD parameters */
ib_smc_update ( ibdev, hermon_mad );
}
/**
* Unregister Hermon Infiniband device
*
* @v hermon Hermon device
* @v port Hermon port
*/
static void hermon_unregister_ibdev ( struct hermon *hermon __unused,
struct hermon_port *port ) {
struct ib_device *ibdev = port->ibdev;
unregister_ibdev ( ibdev );
}
/** Hermon Infiniband port type */
static struct hermon_port_type hermon_port_type_ib = {
.register_dev = hermon_register_ibdev,
.state_change = hermon_state_change_ibdev,
.unregister_dev = hermon_unregister_ibdev,
};
/***************************************************************************
*
* Ethernet operation
*
***************************************************************************
*/
/** Number of Hermon Ethernet send work queue entries */
#define HERMON_ETH_NUM_SEND_WQES 16
/** Number of Hermon Ethernet receive work queue entries */
#define HERMON_ETH_NUM_RECV_WQES 8
/** Number of Hermon Ethernet completion entries */
#define HERMON_ETH_NUM_CQES 32
/**
* Transmit packet via Hermon Ethernet device
*
* @v netdev Network device
* @v iobuf I/O buffer
* @ret rc Return status code
*/
static int hermon_eth_transmit ( struct net_device *netdev,
struct io_buffer *iobuf ) {
struct hermon_port *port = netdev->priv;
struct ib_device *ibdev = port->ibdev;
struct hermon *hermon = ib_get_drvdata ( ibdev );
int rc;
/* Transmit packet */
if ( ( rc = ib_post_send ( ibdev, port->eth_qp, NULL,
iobuf ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not transmit: %s\n",
hermon, ibdev->port, strerror ( rc ) );
return rc;
}
return 0;
}
/** Hermon Ethernet queue pair operations */
static struct ib_queue_pair_operations hermon_eth_qp_op = {
.alloc_iob = alloc_iob,
};
/**
* Handle Hermon Ethernet device send completion
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v iobuf I/O buffer
* @v rc Completion status code
*/
static void hermon_eth_complete_send ( struct ib_device *ibdev __unused,
struct ib_queue_pair *qp,
struct io_buffer *iobuf, int rc ) {
struct net_device *netdev = ib_qp_get_ownerdata ( qp );
netdev_tx_complete_err ( netdev, iobuf, rc );
}
/**
* Handle Hermon Ethernet device receive completion
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v dest Destination address vector, or NULL
* @v source Source address vector, or NULL
* @v iobuf I/O buffer
* @v rc Completion status code
*/
static void hermon_eth_complete_recv ( struct ib_device *ibdev __unused,
struct ib_queue_pair *qp,
struct ib_address_vector *dest __unused,
struct ib_address_vector *source,
struct io_buffer *iobuf, int rc ) {
struct net_device *netdev = ib_qp_get_ownerdata ( qp );
unsigned int tag;
/* Identify VLAN tag, if applicable */
tag = ( source->vlan_present ? source->vlan : 0 );
/* Hand off to network layer */
if ( rc == 0 ) {
vlan_netdev_rx ( netdev, tag, iobuf );
} else {
vlan_netdev_rx_err ( netdev, tag, iobuf, rc );
}
}
/** Hermon Ethernet device completion operations */
static struct ib_completion_queue_operations hermon_eth_cq_op = {
.complete_send = hermon_eth_complete_send,
.complete_recv = hermon_eth_complete_recv,
};
/**
* Poll Hermon Ethernet device
*
* @v netdev Network device
*/
static void hermon_eth_poll ( struct net_device *netdev ) {
struct hermon_port *port = netdev->priv;
struct ib_device *ibdev = port->ibdev;
ib_poll_eq ( ibdev );
}
/**
* Open Hermon Ethernet device
*
* @v netdev Network device
* @ret rc Return status code
*/
static int hermon_eth_open ( struct net_device *netdev ) {
struct hermon_port *port = netdev->priv;
struct ib_device *ibdev = port->ibdev;
struct hermon *hermon = ib_get_drvdata ( ibdev );
union hermonprm_set_port set_port;
int rc;
/* Open hardware */
if ( ( rc = hermon_open ( hermon ) ) != 0 )
goto err_open;
/* Allocate completion queue */
if ( ( rc = ib_create_cq ( ibdev, HERMON_ETH_NUM_CQES,
&hermon_eth_cq_op, &port->eth_cq ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not create completion "
"queue: %s\n", hermon, ibdev->port, strerror ( rc ) );
goto err_create_cq;
}
/* Allocate queue pair */
if ( ( rc = ib_create_qp ( ibdev, IB_QPT_ETH, HERMON_ETH_NUM_SEND_WQES,
port->eth_cq, HERMON_ETH_NUM_RECV_WQES,
port->eth_cq, &hermon_eth_qp_op,
netdev->name, &port->eth_qp ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not create queue "
"pair: %s\n", hermon, ibdev->port, strerror ( rc ) );
goto err_create_qp;
}
ib_qp_set_ownerdata ( port->eth_qp, netdev );
/* Activate queue pair */
if ( ( rc = ib_modify_qp ( ibdev, port->eth_qp ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not modify queue "
"pair: %s\n", hermon, ibdev->port, strerror ( rc ) );
goto err_modify_qp;
}
/* Fill receive rings */
ib_refill_recv ( ibdev, port->eth_qp );
/* Set port general parameters */
memset ( &set_port, 0, sizeof ( set_port ) );
MLX_FILL_3 ( &set_port.general, 0,
v_mtu, 1,
v_pprx, 1,
v_pptx, 1 );
MLX_FILL_1 ( &set_port.general, 1,
mtu, ( ETH_FRAME_LEN + 40 /* Used by card */ ) );
MLX_FILL_1 ( &set_port.general, 2,
pfctx, ( 1 << FCOE_VLAN_PRIORITY ) );
MLX_FILL_1 ( &set_port.general, 3,
pfcrx, ( 1 << FCOE_VLAN_PRIORITY ) );
if ( ( rc = hermon_cmd_set_port ( hermon, 1,
( HERMON_SET_PORT_GENERAL_PARAM |
ibdev->port ),
&set_port ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not set port general "
"parameters: %s\n",
hermon, ibdev->port, strerror ( rc ) );
goto err_set_port_general_params;
}
/* Set port receive QP */
memset ( &set_port, 0, sizeof ( set_port ) );
MLX_FILL_1 ( &set_port.rqp_calc, 0, base_qpn, port->eth_qp->qpn );
MLX_FILL_1 ( &set_port.rqp_calc, 2,
mac_miss_index, 128 /* MAC misses go to promisc QP */ );
MLX_FILL_2 ( &set_port.rqp_calc, 3,
vlan_miss_index, 127 /* VLAN misses go to promisc QP */,
no_vlan_index, 126 /* VLAN-free go to promisc QP */ );
MLX_FILL_2 ( &set_port.rqp_calc, 5,
promisc_qpn, port->eth_qp->qpn,
en_uc_promisc, 1 );
MLX_FILL_2 ( &set_port.rqp_calc, 6,
def_mcast_qpn, port->eth_qp->qpn,
mc_promisc_mode, 2 /* Receive all multicasts */ );
if ( ( rc = hermon_cmd_set_port ( hermon, 1,
( HERMON_SET_PORT_RECEIVE_QP |
ibdev->port ),
&set_port ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not set port receive "
"QP: %s\n", hermon, ibdev->port, strerror ( rc ) );
goto err_set_port_receive_qp;
}
/* Initialise port */
if ( ( rc = hermon_cmd_init_port ( hermon, ibdev->port ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not initialise port: "
"%s\n", hermon, ibdev->port, strerror ( rc ) );
goto err_init_port;
}
return 0;
err_init_port:
err_set_port_receive_qp:
err_set_port_general_params:
err_modify_qp:
ib_destroy_qp ( ibdev, port->eth_qp );
err_create_qp:
ib_destroy_cq ( ibdev, port->eth_cq );
err_create_cq:
hermon_close ( hermon );
err_open:
return rc;
}
/**
* Close Hermon Ethernet device
*
* @v netdev Network device
*/
static void hermon_eth_close ( struct net_device *netdev ) {
struct hermon_port *port = netdev->priv;
struct ib_device *ibdev = port->ibdev;
struct hermon *hermon = ib_get_drvdata ( ibdev );
int rc;
/* Close port */
if ( ( rc = hermon_cmd_close_port ( hermon, ibdev->port ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not close port: %s\n",
hermon, ibdev->port, strerror ( rc ) );
/* Nothing we can do about this */
}
/* Tear down the queues */
ib_destroy_qp ( ibdev, port->eth_qp );
ib_destroy_cq ( ibdev, port->eth_cq );
/* Close hardware */
hermon_close ( hermon );
}
/** Hermon Ethernet network device operations */
static struct net_device_operations hermon_eth_operations = {
.open = hermon_eth_open,
.close = hermon_eth_close,
.transmit = hermon_eth_transmit,
.poll = hermon_eth_poll,
};
/**
* Register Hermon Ethernet device
*
* @v hermon Hermon device
* @v port Hermon port
* @ret rc Return status code
*/
static int hermon_register_netdev ( struct hermon *hermon,
struct hermon_port *port ) {
struct net_device *netdev = port->netdev;
struct ib_device *ibdev = port->ibdev;
int rc;
/* Set MAC address */
memcpy ( netdev->hw_addr, port->eth_mac.raw, ETH_ALEN );
/* Register network device */
if ( ( rc = register_netdev ( netdev ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not register network "
"device: %s\n", hermon, ibdev->port, strerror ( rc ) );
goto err_register_netdev;
}
/* Register non-volatile options */
if ( ( rc = register_nvo ( &port->nvo,
netdev_settings ( netdev ) ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not register non-"
"volatile options: %s\n",
hermon, ibdev->port, strerror ( rc ) );
goto err_register_nvo;
}
return 0;
unregister_nvo ( &port->nvo );
err_register_nvo:
unregister_netdev ( netdev );
err_register_netdev:
return rc;
}
/**
* Handle Hermon Ethernet device port state change
*
* @v hermon Hermon device
* @v port Hermon port
* @v link_up Link is up
*/
static void hermon_state_change_netdev ( struct hermon *hermon __unused,
struct hermon_port *port,
int link_up ) {
struct net_device *netdev = port->netdev;
if ( link_up ) {
netdev_link_up ( netdev );
} else {
netdev_link_down ( netdev );
}
}
/**
* Unregister Hermon Ethernet device
*
* @v hermon Hermon device
* @v port Hermon port
*/
static void hermon_unregister_netdev ( struct hermon *hermon __unused,
struct hermon_port *port ) {
struct net_device *netdev = port->netdev;
unregister_nvo ( &port->nvo );
unregister_netdev ( netdev );
}
/** Hermon Ethernet port type */
static struct hermon_port_type hermon_port_type_eth = {
.register_dev = hermon_register_netdev,
.state_change = hermon_state_change_netdev,
.unregister_dev = hermon_unregister_netdev,
};
/***************************************************************************
*
* Port type detection
*
***************************************************************************
*/
/** Timeout for port sensing */
#define HERMON_SENSE_PORT_TIMEOUT ( TICKS_PER_SEC / 2 )
/**
* Name port type
*
* @v port_type Port type
* @v port_type_name Port type name
*/
static inline const char * hermon_name_port_type ( unsigned int port_type ) {
switch ( port_type ) {
case HERMON_PORT_TYPE_UNKNOWN: return "unknown";
case HERMON_PORT_TYPE_IB: return "Infiniband";
case HERMON_PORT_TYPE_ETH: return "Ethernet";
default: return "INVALID";
}
}
/**
* Sense port type
*
* @v hermon Hermon device
* @v port Hermon port
* @ret port_type Port type, or negative error
*/
static int hermon_sense_port_type ( struct hermon *hermon,
struct hermon_port *port ) {
struct ib_device *ibdev = port->ibdev;
struct hermonprm_sense_port sense_port;
int port_type;
int rc;
/* If DPDP is not supported, always assume Infiniband */
if ( ! hermon->cap.dpdp ) {
port_type = HERMON_PORT_TYPE_IB;
DBGC ( hermon, "Hermon %p port %d does not support DPDP; "
"assuming an %s network\n", hermon, ibdev->port,
hermon_name_port_type ( port_type ) );
return port_type;
}
/* Sense the port type */
if ( ( rc = hermon_cmd_sense_port ( hermon, ibdev->port,
&sense_port ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d sense failed: %s\n",
hermon, ibdev->port, strerror ( rc ) );
return rc;
}
port_type = MLX_GET ( &sense_port, port_type );
DBGC ( hermon, "Hermon %p port %d sensed an %s network\n",
hermon, ibdev->port, hermon_name_port_type ( port_type ) );
return port_type;
}
/**
* Set port type
*
* @v hermon Hermon device
* @v port Hermon port
* @ret rc Return status code
*/
static int hermon_set_port_type ( struct hermon *hermon,
struct hermon_port *port ) {
struct ib_device *ibdev = port->ibdev;
struct hermonprm_query_port_cap query_port;
int ib_supported;
int eth_supported;
int port_type;
unsigned long start;
unsigned long elapsed;
int rc;
/* Check to see which types are supported */
if ( ( rc = hermon_cmd_query_port ( hermon, ibdev->port,
&query_port ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not query port: %s\n",
hermon, ibdev->port, strerror ( rc ) );
return rc;
}
ib_supported = MLX_GET ( &query_port, ib );
eth_supported = MLX_GET ( &query_port, eth );
DBGC ( hermon, "Hermon %p port %d supports%s%s%s\n",
hermon, ibdev->port, ( ib_supported ? " Infiniband" : "" ),
( ( ib_supported && eth_supported ) ? " and" : "" ),
( eth_supported ? " Ethernet" : "" ) );
/* Record Ethernet MAC address */
port->eth_mac.part.h = htons ( MLX_GET ( &query_port, mac_47_32 ) );
port->eth_mac.part.l = htonl ( MLX_GET ( &query_port, mac_31_0 ) );
/* Sense network, if applicable */
if ( ib_supported && eth_supported ) {
/* Both types are supported; try sensing network */
start = currticks();
do {
/* Try sensing port */
port_type = hermon_sense_port_type ( hermon, port );
if ( port_type < 0 ) {
rc = port_type;
return rc;
}
/* Avoid spamming debug output */
mdelay ( 50 );
} while ( ( port_type == HERMON_PORT_TYPE_UNKNOWN ) &&
( ( elapsed = ( currticks() - start ) ) <
HERMON_SENSE_PORT_TIMEOUT ) );
/* Set port type based on sensed network, defaulting
* to Infiniband if nothing was sensed.
*/
switch ( port_type ) {
case HERMON_PORT_TYPE_ETH:
port->type = &hermon_port_type_eth;
break;
case HERMON_PORT_TYPE_IB:
case HERMON_PORT_TYPE_UNKNOWN:
port->type = &hermon_port_type_ib;
break;
default:
return -EINVAL;
}
} else if ( eth_supported ) {
port->type = &hermon_port_type_eth;
} else {
port->type = &hermon_port_type_ib;
}
assert ( port->type != NULL );
return 0;
}
/***************************************************************************
*
* BOFM interface
*
***************************************************************************
*/
/**
* Harvest Ethernet MAC for BOFM
*
* @v bofm BOFM device
* @v mport Multi-port index
* @v mac MAC to fill in
* @ret rc Return status code
*/
static int hermon_bofm_harvest ( struct bofm_device *bofm, unsigned int mport,
uint8_t *mac ) {
struct hermon *hermon = container_of ( bofm, struct hermon, bofm );
struct hermonprm_mod_stat_cfg stat_cfg;
union {
uint8_t bytes[8];
uint32_t dwords[2];
} buf;
int rc;
/* Query static configuration */
if ( ( rc = hermon_mod_stat_cfg ( hermon, mport,
HERMON_MOD_STAT_CFG_QUERY,
HERMON_MOD_STAT_CFG_OFFSET ( mac_m ),
&stat_cfg ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not query "
"configuration: %s\n", hermon, mport, strerror ( rc ) );
return rc;
}
/* Retrieve MAC address */
buf.dwords[0] = htonl ( MLX_GET ( &stat_cfg, mac_high ) );
buf.dwords[1] = htonl ( MLX_GET ( &stat_cfg, mac_low ) );
memcpy ( mac, &buf.bytes[ sizeof ( buf.bytes ) - ETH_ALEN ],
ETH_ALEN );
DBGC ( hermon, "Hermon %p port %d harvested MAC address %s\n",
hermon, mport, eth_ntoa ( mac ) );
return 0;
}
/**
* Update Ethernet MAC for BOFM
*
* @v bofm BOFM device
* @v mport Multi-port index
* @v mac MAC to fill in
* @ret rc Return status code
*/
static int hermon_bofm_update ( struct bofm_device *bofm, unsigned int mport,
const uint8_t *mac ) {
struct hermon *hermon = container_of ( bofm, struct hermon, bofm );
struct hermonprm_mod_stat_cfg stat_cfg;
union {
uint8_t bytes[8];
uint32_t dwords[2];
} buf;
int rc;
/* Prepare MAC address */
memset ( &buf, 0, sizeof ( buf ) );
memcpy ( &buf.bytes[ sizeof ( buf.bytes ) - ETH_ALEN ], mac,
ETH_ALEN );
/* Modify static configuration */
memset ( &stat_cfg, 0, sizeof ( stat_cfg ) );
MLX_FILL_2 ( &stat_cfg, 36,
mac_m, 1,
mac_high, ntohl ( buf.dwords[0] ) );
MLX_FILL_1 ( &stat_cfg, 37, mac_low, ntohl ( buf.dwords[1] ) );
if ( ( rc = hermon_mod_stat_cfg ( hermon, mport,
HERMON_MOD_STAT_CFG_SET,
HERMON_MOD_STAT_CFG_OFFSET ( mac_m ),
&stat_cfg ) ) != 0 ) {
DBGC ( hermon, "Hermon %p port %d could not modify "
"configuration: %s\n", hermon, mport, strerror ( rc ) );
return rc;
}
DBGC ( hermon, "Hermon %p port %d updated MAC address to %s\n",
hermon, mport, eth_ntoa ( mac ) );
return 0;
}
/** Hermon BOFM operations */
static struct bofm_operations hermon_bofm_operations = {
.harvest = hermon_bofm_harvest,
.update = hermon_bofm_update,
};
/***************************************************************************
*
* PCI interface
*
***************************************************************************
*/
/**
* Allocate Hermon device
*
* @v pci PCI device
* @v id PCI ID
* @ret rc Return status code
*/
static struct hermon * hermon_alloc ( void ) {
struct hermon *hermon;
/* Allocate Hermon device */
hermon = zalloc ( sizeof ( *hermon ) );
if ( ! hermon )
goto err_hermon;
/* Allocate space for mailboxes */
hermon->mailbox_in = malloc_phys ( HERMON_MBOX_SIZE,
HERMON_MBOX_ALIGN );
if ( ! hermon->mailbox_in )
goto err_mailbox_in;
hermon->mailbox_out = malloc_phys ( HERMON_MBOX_SIZE,
HERMON_MBOX_ALIGN );
if ( ! hermon->mailbox_out )
goto err_mailbox_out;
return hermon;
free_phys ( hermon->mailbox_out, HERMON_MBOX_SIZE );
err_mailbox_out:
free_phys ( hermon->mailbox_in, HERMON_MBOX_SIZE );
err_mailbox_in:
free ( hermon );
err_hermon:
return NULL;
}
/**
* Free Hermon device
*
* @v hermon Hermon device
*/
static void hermon_free ( struct hermon *hermon ) {
ufree ( hermon->icm );
ufree ( hermon->firmware_area );
free_phys ( hermon->mailbox_out, HERMON_MBOX_SIZE );
free_phys ( hermon->mailbox_in, HERMON_MBOX_SIZE );
free ( hermon );
}
/**
* Probe PCI device
*
* @v pci PCI device
* @v id PCI ID
* @ret rc Return status code
*/
static int hermon_probe ( struct pci_device *pci ) {
struct hermon *hermon;
struct ib_device *ibdev;
struct net_device *netdev;
struct hermon_port *port;
unsigned long config;
unsigned long uar;
unsigned int i;
int rc;
/* Allocate Hermon device */
hermon = hermon_alloc();
if ( ! hermon ) {
rc = -ENOMEM;
goto err_alloc;
}
pci_set_drvdata ( pci, hermon );
hermon->pci = pci;
/* Fix up PCI device */
adjust_pci_device ( pci );
/* Map PCI BARs */
config = pci_bar_start ( pci, HERMON_PCI_CONFIG_BAR );
hermon->config = pci_ioremap ( pci, config,
HERMON_PCI_CONFIG_BAR_SIZE );
uar = pci_bar_start ( pci, HERMON_PCI_UAR_BAR );
hermon->uar = pci_ioremap ( pci, uar,
HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE );
/* Reset device */
if ( ( rc = hermon_reset ( hermon ) ) != 0 )
goto err_reset;
/* Start firmware */
if ( ( rc = hermon_start_firmware ( hermon ) ) != 0 )
goto err_start_firmware;
/* Get device limits */
if ( ( rc = hermon_get_cap ( hermon ) ) != 0 )
goto err_get_cap;
/* Allocate Infiniband devices */
for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
ibdev = alloc_ibdev ( 0 );
if ( ! ibdev ) {
rc = -ENOMEM;
goto err_alloc_ibdev;
}
hermon->port[i].ibdev = ibdev;
ibdev->op = &hermon_ib_operations;
ibdev->dev = &pci->dev;
ibdev->port = ( HERMON_PORT_BASE + i );
ibdev->ports = hermon->cap.num_ports;
ib_set_drvdata ( ibdev, hermon );
}
/* Allocate network devices */
for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
netdev = alloc_etherdev ( 0 );
if ( ! netdev ) {
rc = -ENOMEM;
goto err_alloc_netdev;
}
hermon->port[i].netdev = netdev;
netdev_init ( netdev, &hermon_eth_operations );
netdev->dev = &pci->dev;
netdev->priv = &hermon->port[i];
}
/* Start device */
if ( ( rc = hermon_start ( hermon, 1 ) ) != 0 )
goto err_start;
/* Determine port types */
for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
port = &hermon->port[i];
if ( ( rc = hermon_set_port_type ( hermon, port ) ) != 0 )
goto err_set_port_type;
}
/* Initialise non-volatile storage */
nvs_vpd_init ( &hermon->nvsvpd, pci );
for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
port = &hermon->port[i];
nvs_vpd_nvo_init ( &hermon->nvsvpd,
HERMON_VPD_FIELD ( port->ibdev->port ),
&port->nvo, NULL );
}
/* Register devices */
for ( i = 0 ; i < hermon->cap.num_ports ; i++ ) {
port = &hermon->port[i];
if ( ( rc = port->type->register_dev ( hermon, port ) ) != 0 )
goto err_register;
}
/* Leave device quiescent until opened */
if ( hermon->open_count == 0 )
hermon_stop ( hermon );
return 0;
i = hermon->cap.num_ports;
err_register:
for ( i-- ; ( signed int ) i >= 0 ; i-- ) {
port = &hermon->port[i];
port->type->unregister_dev ( hermon, port );
}
err_set_port_type:
hermon_stop ( hermon );
err_start:
i = hermon->cap.num_ports;
err_alloc_netdev:
for ( i-- ; ( signed int ) i >= 0 ; i-- ) {
netdev_nullify ( hermon->port[i].netdev );
netdev_put ( hermon->port[i].netdev );
}
i = hermon->cap.num_ports;
err_alloc_ibdev:
for ( i-- ; ( signed int ) i >= 0 ; i-- )
ibdev_put ( hermon->port[i].ibdev );
err_get_cap:
hermon_stop_firmware ( hermon );
err_start_firmware:
err_reset:
iounmap ( hermon->uar );
iounmap ( hermon->config );
hermon_free ( hermon );
err_alloc:
return rc;
}
/**
* Remove PCI device
*
* @v pci PCI device
*/
static void hermon_remove ( struct pci_device *pci ) {
struct hermon *hermon = pci_get_drvdata ( pci );
struct hermon_port *port;
int i;
for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- ) {
port = &hermon->port[i];
port->type->unregister_dev ( hermon, port );
}
for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- ) {
netdev_nullify ( hermon->port[i].netdev );
netdev_put ( hermon->port[i].netdev );
}
for ( i = ( hermon->cap.num_ports - 1 ) ; i >= 0 ; i-- )
ibdev_put ( hermon->port[i].ibdev );
iounmap ( hermon->uar );
iounmap ( hermon->config );
hermon_free ( hermon );
}
/**
* Probe PCI device for BOFM
*
* @v pci PCI device
* @v id PCI ID
* @ret rc Return status code
*/
static int hermon_bofm_probe ( struct pci_device *pci ) {
struct hermon *hermon;
unsigned long config;
int rc;
/* Allocate Hermon device */
hermon = hermon_alloc();
if ( ! hermon ) {
rc = -ENOMEM;
goto err_alloc;
}
pci_set_drvdata ( pci, hermon );
hermon->pci = pci;
/* Fix up PCI device */
adjust_pci_device ( pci );
/* Map PCI BAR */
config = pci_bar_start ( pci, HERMON_PCI_CONFIG_BAR );
hermon->config = pci_ioremap ( pci, config,
HERMON_PCI_CONFIG_BAR_SIZE );
/* Initialise BOFM device */
bofm_init ( &hermon->bofm, pci, &hermon_bofm_operations );
/* Register BOFM device */
if ( ( rc = bofm_register ( &hermon->bofm ) ) != 0 ) {
DBGC ( hermon, "Hermon %p could not register BOFM device: "
"%s\n", hermon, strerror ( rc ) );
goto err_bofm_register;
}
return 0;
err_bofm_register:
iounmap ( hermon->config );
hermon_free ( hermon );
err_alloc:
return rc;
}
/**
* Remove PCI device for BOFM
*
* @v pci PCI device
*/
static void hermon_bofm_remove ( struct pci_device *pci ) {
struct hermon *hermon = pci_get_drvdata ( pci );
bofm_unregister ( &hermon->bofm );
iounmap ( hermon->config );
hermon_free ( hermon );
}
static struct pci_device_id hermon_nics[] = {
/* Mellanox ConnectX VPI (ethernet + infiniband) */
PCI_ROM ( 0x15b3, 0x6340, "mt25408", "MT25408 HCA driver", 0 ),
PCI_ROM ( 0x15b3, 0x634a, "mt25418", "MT25418 HCA driver", 0 ),
/* Mellanox ConnectX EN (ethernet only) */
PCI_ROM ( 0x15b3, 0x6368, "mt25448", "MT25448 HCA driver", 0 ),
PCI_ROM ( 0x15b3, 0x6372, "mt25458", "MT25458 HCA driver", 0 ),
/* Mellanox ConnectX-2 VPI (ethernet + infiniband) */
PCI_ROM ( 0x15b3, 0x6732, "mt26418", "MT26418 HCA driver", 0 ),
PCI_ROM ( 0x15b3, 0x673c, "mt26428", "MT26428 HCA driver", 0 ),
PCI_ROM ( 0x15b3, 0x6746, "mt26438", "MT26438 HCA driver", 0 ),
PCI_ROM ( 0x15b3, 0x6778, "mt26488", "MT26488 HCA driver", 0 ),
/* Mellanox ConnectX-2 EN (ethernet only) */
PCI_ROM ( 0x15b3, 0x6750, "mt26448", "MT26448 HCA driver", 0 ),
PCI_ROM ( 0x15b3, 0x675a, "mt26458", "MT26458 HCA driver", 0 ),
PCI_ROM ( 0x15b3, 0x6764, "mt26468", "MT26468 HCA driver", 0 ),
PCI_ROM ( 0x15b3, 0x676e, "mt26478", "MT26478 HCA driver", 0 ),
};
struct pci_driver hermon_driver __pci_driver = {
.ids = hermon_nics,
.id_count = ( sizeof ( hermon_nics ) / sizeof ( hermon_nics[0] ) ),
.probe = hermon_probe,
.remove = hermon_remove,
};
struct pci_driver hermon_bofm_driver __bofm_driver = {
.ids = hermon_nics,
.id_count = ( sizeof ( hermon_nics ) / sizeof ( hermon_nics[0] ) ),
.probe = hermon_bofm_probe,
.remove = hermon_bofm_remove,
};
Reference in New Issue View Git Blame Copy Permalink