mirror of https://github.com/ipxe/ipxe.git
612 lines
15 KiB
C
612 lines
15 KiB
C
#ifndef _GPXE_INFINIBAND_H
|
|
#define _GPXE_INFINIBAND_H
|
|
|
|
/** @file
|
|
*
|
|
* Infiniband protocol
|
|
*
|
|
*/
|
|
|
|
FILE_LICENCE ( GPL2_OR_LATER );
|
|
|
|
#include <stdint.h>
|
|
#include <gpxe/refcnt.h>
|
|
#include <gpxe/device.h>
|
|
#include <gpxe/ib_packet.h>
|
|
#include <gpxe/ib_mad.h>
|
|
#include <gpxe/ib_gma.h>
|
|
|
|
/** Subnet management QPN */
|
|
#define IB_QPN_SMA 0
|
|
|
|
/** Subnet management queue key */
|
|
#define IB_QKEY_SMA 0
|
|
|
|
/** General management QPN */
|
|
#define IB_QPN_GMA 1
|
|
|
|
/** General management queue key */
|
|
#define IB_QKEY_GMA 0x80010000UL
|
|
|
|
/** Broadcast QPN */
|
|
#define IB_QPN_BROADCAST 0xffffffUL
|
|
|
|
/** Default Infiniband partition key */
|
|
#define IB_PKEY_NONE 0xffff
|
|
|
|
/**
|
|
* Maximum payload size
|
|
*
|
|
* This is currently hard-coded in various places (drivers, subnet
|
|
* management agent, etc.) to 2048.
|
|
*/
|
|
#define IB_MAX_PAYLOAD_SIZE 2048
|
|
|
|
struct ib_device;
|
|
struct ib_queue_pair;
|
|
struct ib_address_vector;
|
|
struct ib_completion_queue;
|
|
|
|
/** An Infiniband Work Queue */
|
|
struct ib_work_queue {
|
|
/** Containing queue pair */
|
|
struct ib_queue_pair *qp;
|
|
/** "Is a send queue" flag */
|
|
int is_send;
|
|
/** Associated completion queue */
|
|
struct ib_completion_queue *cq;
|
|
/** List of work queues on this completion queue */
|
|
struct list_head list;
|
|
/** Number of work queue entries */
|
|
unsigned int num_wqes;
|
|
/** Number of occupied work queue entries */
|
|
unsigned int fill;
|
|
/** Next work queue entry index
|
|
*
|
|
* This is the index of the next entry to be filled (i.e. the
|
|
* first empty entry). This value is not bounded by num_wqes;
|
|
* users must logical-AND with (num_wqes-1) to generate an
|
|
* array index.
|
|
*/
|
|
unsigned long next_idx;
|
|
/** I/O buffers assigned to work queue */
|
|
struct io_buffer **iobufs;
|
|
/** Driver private data */
|
|
void *drv_priv;
|
|
};
|
|
|
|
/** An Infiniband multicast GID */
|
|
struct ib_multicast_gid {
|
|
/** List of multicast GIDs on this QP */
|
|
struct list_head list;
|
|
/** Multicast GID */
|
|
struct ib_gid gid;
|
|
};
|
|
|
|
/** An Infiniband Queue Pair */
|
|
struct ib_queue_pair {
|
|
/** Containing Infiniband device */
|
|
struct ib_device *ibdev;
|
|
/** List of queue pairs on this Infiniband device */
|
|
struct list_head list;
|
|
/** Queue Pair Number */
|
|
unsigned long qpn;
|
|
/** Queue key */
|
|
unsigned long qkey;
|
|
/** Send queue */
|
|
struct ib_work_queue send;
|
|
/** Receive queue */
|
|
struct ib_work_queue recv;
|
|
/** List of multicast GIDs */
|
|
struct list_head mgids;
|
|
/** Driver private data */
|
|
void *drv_priv;
|
|
/** Queue owner private data */
|
|
void *owner_priv;
|
|
};
|
|
|
|
/** Infiniband queue pair modification flags */
|
|
enum ib_queue_pair_mods {
|
|
IB_MODIFY_QKEY = 0x0001,
|
|
};
|
|
|
|
/** An Infiniband Address Vector */
|
|
struct ib_address_vector {
|
|
/** Queue Pair Number */
|
|
unsigned long qpn;
|
|
/** Queue key
|
|
*
|
|
* Not specified for received packets.
|
|
*/
|
|
unsigned long qkey;
|
|
/** Local ID */
|
|
unsigned int lid;
|
|
/** Rate
|
|
*
|
|
* Not specified for received packets.
|
|
*/
|
|
unsigned int rate;
|
|
/** Service level */
|
|
unsigned int sl;
|
|
/** GID is present */
|
|
unsigned int gid_present;
|
|
/** GID, if present */
|
|
struct ib_gid gid;
|
|
};
|
|
|
|
/** Infiniband transmission rates */
|
|
enum ib_rate {
|
|
IB_RATE_2_5 = 2,
|
|
IB_RATE_10 = 3,
|
|
IB_RATE_30 = 4,
|
|
IB_RATE_5 = 5,
|
|
IB_RATE_20 = 6,
|
|
IB_RATE_40 = 7,
|
|
IB_RATE_60 = 8,
|
|
IB_RATE_80 = 9,
|
|
IB_RATE_120 = 10,
|
|
};
|
|
|
|
/** Infiniband completion queue operations */
|
|
struct ib_completion_queue_operations {
|
|
/**
|
|
* Complete Send WQE
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v qp Queue pair
|
|
* @v iobuf I/O buffer
|
|
* @v rc Completion status code
|
|
*/
|
|
void ( * complete_send ) ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp,
|
|
struct io_buffer *iobuf, int rc );
|
|
/**
|
|
* Complete Receive WQE
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v qp Queue pair
|
|
* @v av Address vector, or NULL
|
|
* @v iobuf I/O buffer
|
|
* @v rc Completion status code
|
|
*/
|
|
void ( * complete_recv ) ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp,
|
|
struct ib_address_vector *av,
|
|
struct io_buffer *iobuf, int rc );
|
|
};
|
|
|
|
/** An Infiniband Completion Queue */
|
|
struct ib_completion_queue {
|
|
/** Containing Infiniband device */
|
|
struct ib_device *ibdev;
|
|
/** List of completion queues on this Infiniband device */
|
|
struct list_head list;
|
|
/** Completion queue number */
|
|
unsigned long cqn;
|
|
/** Number of completion queue entries */
|
|
unsigned int num_cqes;
|
|
/** Next completion queue entry index
|
|
*
|
|
* This is the index of the next entry to be filled (i.e. the
|
|
* first empty entry). This value is not bounded by num_wqes;
|
|
* users must logical-AND with (num_wqes-1) to generate an
|
|
* array index.
|
|
*/
|
|
unsigned long next_idx;
|
|
/** List of work queues completing to this queue */
|
|
struct list_head work_queues;
|
|
/** Completion queue operations */
|
|
struct ib_completion_queue_operations *op;
|
|
/** Driver private data */
|
|
void *drv_priv;
|
|
};
|
|
|
|
/**
|
|
* Infiniband device operations
|
|
*
|
|
* These represent a subset of the Infiniband Verbs.
|
|
*/
|
|
struct ib_device_operations {
|
|
/** Create completion queue
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v cq Completion queue
|
|
* @ret rc Return status code
|
|
*/
|
|
int ( * create_cq ) ( struct ib_device *ibdev,
|
|
struct ib_completion_queue *cq );
|
|
/** Destroy completion queue
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v cq Completion queue
|
|
*/
|
|
void ( * destroy_cq ) ( struct ib_device *ibdev,
|
|
struct ib_completion_queue *cq );
|
|
/** Create queue pair
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v qp Queue pair
|
|
* @ret rc Return status code
|
|
*/
|
|
int ( * create_qp ) ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp );
|
|
/** Modify queue pair
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v qp Queue pair
|
|
* @v mod_list Modification list
|
|
* @ret rc Return status code
|
|
*/
|
|
int ( * modify_qp ) ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp,
|
|
unsigned long mod_list );
|
|
/** Destroy queue pair
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v qp Queue pair
|
|
*/
|
|
void ( * destroy_qp ) ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp );
|
|
/** Post send work queue entry
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v qp Queue pair
|
|
* @v av Address vector
|
|
* @v iobuf I/O buffer
|
|
* @ret rc Return status code
|
|
*
|
|
* If this method returns success, the I/O buffer remains
|
|
* owned by the queue pair. If this method returns failure,
|
|
* the I/O buffer is immediately released; the failure is
|
|
* interpreted as "failure to enqueue buffer".
|
|
*/
|
|
int ( * post_send ) ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp,
|
|
struct ib_address_vector *av,
|
|
struct io_buffer *iobuf );
|
|
/** Post receive work queue entry
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v qp Queue pair
|
|
* @v iobuf I/O buffer
|
|
* @ret rc Return status code
|
|
*
|
|
* If this method returns success, the I/O buffer remains
|
|
* owned by the queue pair. If this method returns failure,
|
|
* the I/O buffer is immediately released; the failure is
|
|
* interpreted as "failure to enqueue buffer".
|
|
*/
|
|
int ( * post_recv ) ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp,
|
|
struct io_buffer *iobuf );
|
|
/** Poll completion queue
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v cq Completion queue
|
|
*
|
|
* The relevant completion handler (specified at completion
|
|
* queue creation time) takes ownership of the I/O buffer.
|
|
*/
|
|
void ( * poll_cq ) ( struct ib_device *ibdev,
|
|
struct ib_completion_queue *cq );
|
|
/**
|
|
* Poll event queue
|
|
*
|
|
* @v ibdev Infiniband device
|
|
*/
|
|
void ( * poll_eq ) ( struct ib_device *ibdev );
|
|
/**
|
|
* Open port
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @ret rc Return status code
|
|
*/
|
|
int ( * open ) ( struct ib_device *ibdev );
|
|
/**
|
|
* Close port
|
|
*
|
|
* @v ibdev Infiniband device
|
|
*/
|
|
void ( * close ) ( struct ib_device *ibdev );
|
|
/** Attach to multicast group
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v qp Queue pair
|
|
* @v gid Multicast GID
|
|
* @ret rc Return status code
|
|
*/
|
|
int ( * mcast_attach ) ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp,
|
|
struct ib_gid *gid );
|
|
/** Detach from multicast group
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v qp Queue pair
|
|
* @v gid Multicast GID
|
|
*/
|
|
void ( * mcast_detach ) ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp,
|
|
struct ib_gid *gid );
|
|
/** Set port information
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v port_info New port information
|
|
*
|
|
* This method is required only by adapters that do not have
|
|
* an embedded SMA.
|
|
*/
|
|
int ( * set_port_info ) ( struct ib_device *ibdev,
|
|
const struct ib_port_info *port_info );
|
|
};
|
|
|
|
/** An Infiniband device */
|
|
struct ib_device {
|
|
/** Reference counter */
|
|
struct refcnt refcnt;
|
|
/** List of Infiniband devices */
|
|
struct list_head list;
|
|
/** Underlying device */
|
|
struct device *dev;
|
|
/** List of completion queues */
|
|
struct list_head cqs;
|
|
/** List of queue pairs */
|
|
struct list_head qps;
|
|
/** Infiniband operations */
|
|
struct ib_device_operations *op;
|
|
/** Port number */
|
|
unsigned int port;
|
|
/** Port open request counter */
|
|
unsigned int open_count;
|
|
|
|
/** Port state */
|
|
uint8_t port_state;
|
|
/** Link width */
|
|
uint8_t link_width;
|
|
/** Link speed */
|
|
uint8_t link_speed;
|
|
/** Port GID */
|
|
struct ib_gid gid;
|
|
/** Port LID */
|
|
uint16_t lid;
|
|
/** Subnet manager LID */
|
|
uint16_t sm_lid;
|
|
/** Subnet manager SL */
|
|
uint8_t sm_sl;
|
|
/** Partition key */
|
|
uint16_t pkey;
|
|
|
|
/** Outbound packet sequence number */
|
|
uint32_t psn;
|
|
|
|
/** General management agent */
|
|
struct ib_gma gma;
|
|
|
|
/** Driver private data */
|
|
void *drv_priv;
|
|
/** Owner private data */
|
|
void *owner_priv;
|
|
};
|
|
|
|
extern struct ib_completion_queue *
|
|
ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
|
|
struct ib_completion_queue_operations *op );
|
|
extern void ib_destroy_cq ( struct ib_device *ibdev,
|
|
struct ib_completion_queue *cq );
|
|
extern void ib_poll_cq ( struct ib_device *ibdev,
|
|
struct ib_completion_queue *cq );
|
|
extern struct ib_queue_pair *
|
|
ib_create_qp ( struct ib_device *ibdev, unsigned int num_send_wqes,
|
|
struct ib_completion_queue *send_cq, unsigned int num_recv_wqes,
|
|
struct ib_completion_queue *recv_cq, unsigned long qkey );
|
|
extern int ib_modify_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
|
unsigned long mod_list, unsigned long qkey );
|
|
extern void ib_destroy_qp ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp );
|
|
extern struct ib_queue_pair * ib_find_qp_qpn ( struct ib_device *ibdev,
|
|
unsigned long qpn );
|
|
extern struct ib_queue_pair * ib_find_qp_mgid ( struct ib_device *ibdev,
|
|
struct ib_gid *gid );
|
|
extern struct ib_work_queue * ib_find_wq ( struct ib_completion_queue *cq,
|
|
unsigned long qpn, int is_send );
|
|
extern int ib_post_send ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
|
struct ib_address_vector *av,
|
|
struct io_buffer *iobuf );
|
|
extern int ib_post_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
|
struct io_buffer *iobuf );
|
|
extern void ib_complete_send ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp,
|
|
struct io_buffer *iobuf, int rc );
|
|
extern void ib_complete_recv ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp,
|
|
struct ib_address_vector *av,
|
|
struct io_buffer *iobuf, int rc );
|
|
extern void ib_refill_recv ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp );
|
|
extern int ib_open ( struct ib_device *ibdev );
|
|
extern void ib_close ( struct ib_device *ibdev );
|
|
extern int ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
|
|
struct ib_gid *gid );
|
|
extern void ib_mcast_detach ( struct ib_device *ibdev,
|
|
struct ib_queue_pair *qp, struct ib_gid *gid );
|
|
extern int ib_get_hca_info ( struct ib_device *ibdev,
|
|
struct ib_gid_half *hca_guid );
|
|
extern int ib_set_port_info ( struct ib_device *ibdev,
|
|
const struct ib_port_info *port_info );
|
|
extern struct ib_device * alloc_ibdev ( size_t priv_size );
|
|
extern int register_ibdev ( struct ib_device *ibdev );
|
|
extern void unregister_ibdev ( struct ib_device *ibdev );
|
|
extern void ib_link_state_changed ( struct ib_device *ibdev );
|
|
extern void ib_poll_eq ( struct ib_device *ibdev );
|
|
extern struct list_head ib_devices;
|
|
|
|
/** Iterate over all network devices */
|
|
#define for_each_ibdev( ibdev ) \
|
|
list_for_each_entry ( (ibdev), &ib_devices, list )
|
|
|
|
/**
|
|
* Check link state
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @ret link_up Link is up
|
|
*/
|
|
static inline __always_inline int
|
|
ib_link_ok ( struct ib_device *ibdev ) {
|
|
return ( ibdev->port_state == IB_PORT_STATE_ACTIVE );
|
|
}
|
|
|
|
/**
|
|
* Get reference to Infiniband device
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @ret ibdev Infiniband device
|
|
*/
|
|
static inline __always_inline struct ib_device *
|
|
ibdev_get ( struct ib_device *ibdev ) {
|
|
ref_get ( &ibdev->refcnt );
|
|
return ibdev;
|
|
}
|
|
|
|
/**
|
|
* Drop reference to Infiniband device
|
|
*
|
|
* @v ibdev Infiniband device
|
|
*/
|
|
static inline __always_inline void
|
|
ibdev_put ( struct ib_device *ibdev ) {
|
|
ref_put ( &ibdev->refcnt );
|
|
}
|
|
|
|
/**
|
|
* Set Infiniband work queue driver-private data
|
|
*
|
|
* @v wq Work queue
|
|
* @v priv Private data
|
|
*/
|
|
static inline __always_inline void
|
|
ib_wq_set_drvdata ( struct ib_work_queue *wq, void *priv ) {
|
|
wq->drv_priv = priv;
|
|
}
|
|
|
|
/**
|
|
* Get Infiniband work queue driver-private data
|
|
*
|
|
* @v wq Work queue
|
|
* @ret priv Private data
|
|
*/
|
|
static inline __always_inline void *
|
|
ib_wq_get_drvdata ( struct ib_work_queue *wq ) {
|
|
return wq->drv_priv;
|
|
}
|
|
|
|
/**
|
|
* Set Infiniband queue pair driver-private data
|
|
*
|
|
* @v qp Queue pair
|
|
* @v priv Private data
|
|
*/
|
|
static inline __always_inline void
|
|
ib_qp_set_drvdata ( struct ib_queue_pair *qp, void *priv ) {
|
|
qp->drv_priv = priv;
|
|
}
|
|
|
|
/**
|
|
* Get Infiniband queue pair driver-private data
|
|
*
|
|
* @v qp Queue pair
|
|
* @ret priv Private data
|
|
*/
|
|
static inline __always_inline void *
|
|
ib_qp_get_drvdata ( struct ib_queue_pair *qp ) {
|
|
return qp->drv_priv;
|
|
}
|
|
|
|
/**
|
|
* Set Infiniband queue pair owner-private data
|
|
*
|
|
* @v qp Queue pair
|
|
* @v priv Private data
|
|
*/
|
|
static inline __always_inline void
|
|
ib_qp_set_ownerdata ( struct ib_queue_pair *qp, void *priv ) {
|
|
qp->owner_priv = priv;
|
|
}
|
|
|
|
/**
|
|
* Get Infiniband queue pair owner-private data
|
|
*
|
|
* @v qp Queue pair
|
|
* @ret priv Private data
|
|
*/
|
|
static inline __always_inline void *
|
|
ib_qp_get_ownerdata ( struct ib_queue_pair *qp ) {
|
|
return qp->owner_priv;
|
|
}
|
|
|
|
/**
|
|
* Set Infiniband completion queue driver-private data
|
|
*
|
|
* @v cq Completion queue
|
|
* @v priv Private data
|
|
*/
|
|
static inline __always_inline void
|
|
ib_cq_set_drvdata ( struct ib_completion_queue *cq, void *priv ) {
|
|
cq->drv_priv = priv;
|
|
}
|
|
|
|
/**
|
|
* Get Infiniband completion queue driver-private data
|
|
*
|
|
* @v cq Completion queue
|
|
* @ret priv Private data
|
|
*/
|
|
static inline __always_inline void *
|
|
ib_cq_get_drvdata ( struct ib_completion_queue *cq ) {
|
|
return cq->drv_priv;
|
|
}
|
|
|
|
/**
|
|
* Set Infiniband device driver-private data
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v priv Private data
|
|
*/
|
|
static inline __always_inline void
|
|
ib_set_drvdata ( struct ib_device *ibdev, void *priv ) {
|
|
ibdev->drv_priv = priv;
|
|
}
|
|
|
|
/**
|
|
* Get Infiniband device driver-private data
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @ret priv Private data
|
|
*/
|
|
static inline __always_inline void *
|
|
ib_get_drvdata ( struct ib_device *ibdev ) {
|
|
return ibdev->drv_priv;
|
|
}
|
|
|
|
/**
|
|
* Set Infiniband device owner-private data
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @v priv Private data
|
|
*/
|
|
static inline __always_inline void
|
|
ib_set_ownerdata ( struct ib_device *ibdev, void *priv ) {
|
|
ibdev->owner_priv = priv;
|
|
}
|
|
|
|
/**
|
|
* Get Infiniband device owner-private data
|
|
*
|
|
* @v ibdev Infiniband device
|
|
* @ret priv Private data
|
|
*/
|
|
static inline __always_inline void *
|
|
ib_get_ownerdata ( struct ib_device *ibdev ) {
|
|
return ibdev->owner_priv;
|
|
}
|
|
|
|
#endif /* _GPXE_INFINIBAND_H */
|