Logo Search packages:      
Sourcecode: dapl version File versions  Download package

dapl_ib_dto.h

/*
 * This Software is licensed under one of the following licenses:
 *
 * 1) under the terms of the "Common Public License 1.0" a copy of which is
 *    available from the Open Source Initiative, see
 *    http://www.opensource.org/licenses/cpl.php.
 *
 * 2) under the terms of the "The BSD License" a copy of which is
 *    available from the Open Source Initiative, see
 *    http://www.opensource.org/licenses/bsd-license.php.
 *
 * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
 *    copy of which is available from the Open Source Initiative, see
 *    http://www.opensource.org/licenses/gpl-license.php.
 *
 * Licensee has the right to choose one of the above licenses.
 *
 * Redistributions of source code must retain the above copyright
 * notice and one of the license notices.
 *
 * Redistributions in binary form must reproduce both the above copyright
 * notice, one of the license notices in the documentation
 * and/or other materials provided with the distribution.
 */

/***************************************************************************
 *
 *   Module:             uDAPL
 *
 *   Filename:           dapl_ib_dto.h
 *
 *   Author:             Arlin Davis
 *
 *   Created:            3/10/2005
 *
 *   Description: 
 *
 *   The OpenIB uCMA provider - DTO operations and CQE macros 
 *
 ****************************************************************************
 *             Source Control System Information
 *
 *    $Id: $
 *
 *    Copyright (c) 2005 Intel Corporation.  All rights reserved.
 *
 **************************************************************************/
#ifndef _DAPL_IB_DTO_H_
#define _DAPL_IB_DTO_H_

#include "dapl_ib_util.h"

#ifdef DAT_EXTENSIONS
#include <dat2/dat_ib_extensions.h>
#endif

#define     DEFAULT_DS_ENTRIES      8

STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p);

#define CQE_WR_TYPE_UD(id) \
      (((DAPL_COOKIE *)(uintptr_t)id)->ep->qp_handle->qp_type == IBV_QPT_UD)

/*
 * dapls_ib_post_recv
 *
 * Provider specific Post RECV function
 */
STATIC _INLINE_ DAT_RETURN 
dapls_ib_post_recv (
      IN  DAPL_EP       *ep_ptr,
      IN  DAPL_COOKIE         *cookie,
      IN  DAT_COUNT           segments,
      IN  DAT_LMR_TRIPLET     *local_iov )
{
      struct ibv_recv_wr wr;
      struct ibv_recv_wr *bad_wr;
      ib_data_segment_t *ds = (ib_data_segment_t *)local_iov;
      DAT_COUNT i, total_len;
      int ret;
      
      dapl_dbg_log(DAPL_DBG_TYPE_EP,
                 " post_rcv: ep %p cookie %p segs %d l_iov %p\n",
                 ep_ptr, cookie, segments, local_iov);

      /* setup work request */
      total_len = 0;
      wr.next = 0;
      wr.num_sge = segments;
      wr.wr_id = (uint64_t)(uintptr_t)cookie;
      wr.sg_list = ds;

      if (cookie != NULL) {
            for (i = 0; i < segments; i++) {
                  dapl_dbg_log(DAPL_DBG_TYPE_EP, 
                             " post_rcv: l_key 0x%x va %p len %d\n",
                             ds->lkey, ds->addr, ds->length );
                  total_len += ds->length;
                  ds++;
            }
            cookie->val.dto.size = total_len;
      }

      ret = ibv_post_recv(ep_ptr->qp_handle, &wr, &bad_wr);
      
      if (ret)
            return(dapl_convert_errno(errno,"ibv_recv"));

      DAPL_CNTR(ep_ptr, DCNT_EP_POST_RECV);
      DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_RECV_DATA, total_len);

      return DAT_SUCCESS;
}

/*
 * dapls_ib_post_send
 *
 * Provider specific Post SEND function
 */
STATIC _INLINE_ DAT_RETURN 
dapls_ib_post_send (
      IN  DAPL_EP             *ep_ptr,
      IN  ib_send_op_type_t         op_type,
      IN  DAPL_COOKIE               *cookie,
      IN  DAT_COUNT                 segments,
      IN  DAT_LMR_TRIPLET           *local_iov,
      IN  const DAT_RMR_TRIPLET     *remote_iov,
      IN  DAT_COMPLETION_FLAGS      completion_flags)
{
      struct ibv_send_wr wr;
      struct ibv_send_wr *bad_wr;
      ib_data_segment_t *ds = (ib_data_segment_t *)local_iov;
      ib_hca_transport_t *ibt_ptr = 
            &ep_ptr->header.owner_ia->hca_ptr->ib_trans;
      DAT_COUNT i, total_len;
      int ret;
      
      dapl_dbg_log(DAPL_DBG_TYPE_EP,
                 " post_snd: ep %p op %d ck %p sgs",
                 "%d l_iov %p r_iov %p f %d\n",
                 ep_ptr, op_type, cookie, segments, local_iov, 
                 remote_iov, completion_flags);

#ifdef DAT_EXTENSIONS   
      if (ep_ptr->qp_handle->qp_type != IBV_QPT_RC)
            return(DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP));
#endif
      /* setup the work request */
      wr.next = 0;
      wr.opcode = op_type;
      wr.num_sge = segments;
      wr.send_flags = 0;
      wr.wr_id = (uint64_t)(uintptr_t)cookie;
      wr.sg_list = ds;
      total_len = 0;

      if (cookie != NULL) {
            for (i = 0; i < segments; i++ ) {
                  dapl_dbg_log(DAPL_DBG_TYPE_EP, 
                             " post_snd: lkey 0x%x va %p len %d\n",
                             ds->lkey, ds->addr, ds->length );
                  total_len += ds->length;
                  ds++;
            }
            cookie->val.dto.size = total_len;
      }

      if (wr.num_sge && 
          (op_type == OP_RDMA_WRITE || op_type == OP_RDMA_READ)) {
            wr.wr.rdma.remote_addr = remote_iov->virtual_address;
            wr.wr.rdma.rkey = remote_iov->rmr_context;
            dapl_dbg_log(DAPL_DBG_TYPE_EP, 
                       " post_snd_rdma: rkey 0x%x va %#016Lx\n",
                       wr.wr.rdma.rkey, wr.wr.rdma.remote_addr);
      }


      /* inline data for send or write ops */
      if ((total_len <= ibt_ptr->max_inline_send) && 
         ((op_type == OP_SEND) || (op_type == OP_RDMA_WRITE))) 
            wr.send_flags |= IBV_SEND_INLINE;
      
      /* set completion flags in work request */
      wr.send_flags |= (DAT_COMPLETION_SUPPRESS_FLAG & 
                        completion_flags) ? 0 : IBV_SEND_SIGNALED;
      wr.send_flags |= (DAT_COMPLETION_BARRIER_FENCE_FLAG & 
                        completion_flags) ? IBV_SEND_FENCE : 0;
      wr.send_flags |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG & 
                        completion_flags) ? IBV_SEND_SOLICITED : 0;

      dapl_dbg_log(DAPL_DBG_TYPE_EP, 
                 " post_snd: op 0x%x flags 0x%x sglist %p, %d\n", 
                 wr.opcode, wr.send_flags, wr.sg_list, wr.num_sge);

      ret = ibv_post_send(ep_ptr->qp_handle, &wr, &bad_wr);

      if (ret)
            return(dapl_convert_errno(errno,"ibv_send"));

#ifdef DAPL_COUNTERS
      switch (op_type) {
      case OP_SEND:
            DAPL_CNTR(ep_ptr, DCNT_EP_POST_SEND);
            DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_SEND_DATA,total_len);
            break;
      case OP_RDMA_WRITE:
            DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE);
            DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_WRITE_DATA,total_len);
            break;      
      case OP_RDMA_READ:
            DAPL_CNTR(ep_ptr, DCNT_EP_POST_READ);
            DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_READ_DATA,total_len);
            break;
      default:
            break;
      }
#endif /* DAPL_COUNTERS */

      dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n");
      return DAT_SUCCESS;
}

/* map Work Completions to DAPL WR operations */
STATIC _INLINE_ DAT_DTOS dapls_cqe_dtos_opcode(ib_work_completion_t *cqe_p)
{
      switch (cqe_p->opcode) {

      case IBV_WC_SEND:
#ifdef DAT_EXTENSIONS
            if (CQE_WR_TYPE_UD(cqe_p->wr_id))
                  return (DAT_IB_DTO_SEND_UD);
            else
#endif                  
            return (DAT_DTO_SEND);
      case IBV_WC_RDMA_READ:
            return (DAT_DTO_RDMA_READ);
      case IBV_WC_BIND_MW:
            return (DAT_DTO_BIND_MW);
#ifdef DAT_EXTENSIONS
      case IBV_WC_RDMA_WRITE:
            if (cqe_p->wc_flags & IBV_WC_WITH_IMM)
                  return (DAT_IB_DTO_RDMA_WRITE_IMMED);
            else
                  return (DAT_DTO_RDMA_WRITE);
      case IBV_WC_COMP_SWAP:
            return (DAT_IB_DTO_CMP_SWAP);
      case IBV_WC_FETCH_ADD:
            return (DAT_IB_DTO_FETCH_ADD);
      case IBV_WC_RECV_RDMA_WITH_IMM:
            return (DAT_IB_DTO_RECV_IMMED);
#else
      case IBV_WC_RDMA_WRITE:
            return (DAT_DTO_RDMA_WRITE);
#endif
      case IBV_WC_RECV:
#ifdef DAT_EXTENSIONS
            if (CQE_WR_TYPE_UD(cqe_p->wr_id)) 
                  return (DAT_IB_DTO_RECV_UD);
            else if (cqe_p->wc_flags & IBV_WC_WITH_IMM)
                  return (DAT_IB_DTO_RECV_MSG_IMMED);
            else
#endif      
            return (DAT_DTO_RECEIVE);
      default:
            return (0xff);
      }
}
#define DAPL_GET_CQE_DTOS_OPTYPE(cqe_p) dapls_cqe_dtos_opcode(cqe_p)


#ifdef DAT_EXTENSIONS
/*
 * dapls_ib_post_ext_send
 *
 * Provider specific extended Post SEND function for atomics
 *    OP_COMP_AND_SWAP and OP_FETCH_AND_ADD
 */
STATIC _INLINE_ DAT_RETURN 
dapls_ib_post_ext_send (
      IN  DAPL_EP             *ep_ptr,
      IN  ib_send_op_type_t         op_type,
      IN  DAPL_COOKIE               *cookie,
      IN  DAT_COUNT                 segments,
      IN  DAT_LMR_TRIPLET           *local_iov,
      IN  const DAT_RMR_TRIPLET     *remote_iov,
      IN  DAT_UINT32                immed_data,
      IN  DAT_UINT64                compare_add,
      IN  DAT_UINT64                swap,
      IN  DAT_COMPLETION_FLAGS      completion_flags,
      IN  DAT_IB_ADDR_HANDLE        *remote_ah)
{
      struct ibv_send_wr wr;
      struct ibv_send_wr *bad_wr;
      ib_data_segment_t *ds = (ib_data_segment_t *)local_iov;
      DAT_COUNT i, total_len;
      int ret;
      
      dapl_dbg_log(DAPL_DBG_TYPE_EP,
                 " post_ext_snd: ep %p op %d ck %p sgs",
                 "%d l_iov %p r_iov %p f %d\n",
                 ep_ptr, op_type, cookie, segments, local_iov, 
                 remote_iov, completion_flags, remote_ah);

      /* setup the work request */
      wr.next = 0;
      wr.opcode = op_type;
      wr.num_sge = segments;
      wr.send_flags = 0;
      wr.wr_id = (uint64_t)(uintptr_t)cookie;
      wr.sg_list = ds;
      total_len = 0;

      if (cookie != NULL) {
            for (i = 0; i < segments; i++ ) {
                  dapl_dbg_log(DAPL_DBG_TYPE_EP, 
                             " post_snd: lkey 0x%x va %p len %d\n",
                             ds->lkey, ds->addr, ds->length );
                  total_len += ds->length;
                  ds++;
            }
            cookie->val.dto.size = total_len;
      }

      switch (op_type) {
      case OP_RDMA_WRITE_IMM:
            /* OP_RDMA_WRITE)IMMED has direct IB wr_type mapping */
            dapl_dbg_log(DAPL_DBG_TYPE_EP, 
                       " post_ext: rkey 0x%x va %#016Lx immed=0x%x\n",
                       remote_iov?remote_iov->rmr_context:0, 
                       remote_iov?remote_iov->virtual_address:0,
                       immed_data);

            wr.imm_data = immed_data;
              if (wr.num_sge) {
                  wr.wr.rdma.remote_addr = remote_iov->virtual_address;
                  wr.wr.rdma.rkey = remote_iov->rmr_context;
            }
            break;
      case OP_COMP_AND_SWAP:
            /* OP_COMP_AND_SWAP has direct IB wr_type mapping */
            dapl_dbg_log(DAPL_DBG_TYPE_EP, 
                       " post_ext: OP_COMP_AND_SWAP=%lx,"
                       "%lx rkey 0x%x va %#016Lx\n",
                       compare_add, swap, remote_iov->rmr_context,
                       remote_iov->virtual_address);
            
            wr.wr.atomic.compare_add = compare_add;
            wr.wr.atomic.swap = swap;
            wr.wr.atomic.remote_addr = remote_iov->virtual_address;
            wr.wr.atomic.rkey = remote_iov->rmr_context;
            break;
      case OP_FETCH_AND_ADD:
            /* OP_FETCH_AND_ADD has direct IB wr_type mapping */
            dapl_dbg_log(DAPL_DBG_TYPE_EP, 
                       " post_ext: OP_FETCH_AND_ADD=%lx,"
                       "%lx rkey 0x%x va %#016Lx\n",
                       compare_add, remote_iov->rmr_context,
                       remote_iov->virtual_address);

            wr.wr.atomic.compare_add = compare_add;
            wr.wr.atomic.remote_addr = remote_iov->virtual_address;
            wr.wr.atomic.rkey = remote_iov->rmr_context;
            break;
      case OP_SEND_UD:
            /* post must be on EP with service_type of UD */
            if (ep_ptr->qp_handle->qp_type != IBV_QPT_UD)
                  return(DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP));

            dapl_dbg_log(DAPL_DBG_TYPE_EP, 
                       " post_ext: OP_SEND_UD ah=%p"
                       " qp_num=0x%x\n",
                       remote_ah, remote_ah->qpn);
            
            wr.opcode = OP_SEND;
            wr.wr.ud.ah = remote_ah->ah;
            wr.wr.ud.remote_qpn = remote_ah->qpn;
            wr.wr.ud.remote_qkey = SCM_UD_QKEY;
            break;
      default:
            break;
      }

      /* set completion flags in work request */
      wr.send_flags |= (DAT_COMPLETION_SUPPRESS_FLAG & 
                        completion_flags) ? 0 : IBV_SEND_SIGNALED;
      wr.send_flags |= (DAT_COMPLETION_BARRIER_FENCE_FLAG & 
                        completion_flags) ? IBV_SEND_FENCE : 0;
      wr.send_flags |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG & 
                        completion_flags) ? IBV_SEND_SOLICITED : 0;

      dapl_dbg_log(DAPL_DBG_TYPE_EP, 
                 " post_snd: op 0x%x flags 0x%x sglist %p, %d\n", 
                 wr.opcode, wr.send_flags, wr.sg_list, wr.num_sge);

      ret = ibv_post_send(ep_ptr->qp_handle, &wr, &bad_wr);

      if (ret)
            return( dapl_convert_errno(errno,"ibv_send") );
      
#ifdef DAPL_COUNTERS
      switch (op_type) {
      case OP_RDMA_WRITE_IMM:
            DAPL_CNTR(ep_ptr, DCNT_EP_POST_WRITE_IMM);
            DAPL_CNTR_DATA(ep_ptr, 
                         DCNT_EP_POST_WRITE_IMM_DATA, total_len);
            break;
      case OP_COMP_AND_SWAP:
            DAPL_CNTR(ep_ptr, DCNT_EP_POST_CMP_SWAP);
            break;      
      case OP_FETCH_AND_ADD:
            DAPL_CNTR(ep_ptr, DCNT_EP_POST_FETCH_ADD);
            break;
      case OP_SEND_UD:
            DAPL_CNTR(ep_ptr, DCNT_EP_POST_SEND_UD);
            DAPL_CNTR_DATA(ep_ptr, DCNT_EP_POST_SEND_UD_DATA, total_len);
            break;
      default:
            break;
      }
#endif /* DAPL_COUNTERS */

      dapl_dbg_log(DAPL_DBG_TYPE_EP," post_snd: returned\n");
        return DAT_SUCCESS;
}
#endif

STATIC _INLINE_ DAT_RETURN 
dapls_ib_optional_prv_dat(
      IN  DAPL_CR       *cr_ptr,
      IN  const void          *event_data,
      OUT   DAPL_CR           **cr_pp)
{
    return DAT_SUCCESS;
}


/* map Work Completions to DAPL WR operations */
STATIC _INLINE_ int dapls_cqe_opcode(ib_work_completion_t *cqe_p)
{
#ifdef DAPL_COUNTERS
      DAPL_COOKIE *cookie = (DAPL_COOKIE *)(uintptr_t)cqe_p->wr_id;
#endif /* DAPL_COUNTERS */

      switch (cqe_p->opcode) {
      case IBV_WC_SEND:
            if (CQE_WR_TYPE_UD(cqe_p->wr_id))
                  return(OP_SEND_UD);
            else
                  return (OP_SEND);
      case IBV_WC_RDMA_WRITE:
            if (cqe_p->wc_flags & IBV_WC_WITH_IMM)
                  return (OP_RDMA_WRITE_IMM);
            else
                  return (OP_RDMA_WRITE);
      case IBV_WC_RDMA_READ:
            return (OP_RDMA_READ);
      case IBV_WC_COMP_SWAP:
            return (OP_COMP_AND_SWAP);
      case IBV_WC_FETCH_ADD:
            return (OP_FETCH_AND_ADD);
      case IBV_WC_BIND_MW:
            return (OP_BIND_MW);
      case IBV_WC_RECV:
            if (CQE_WR_TYPE_UD(cqe_p->wr_id)) {
                  DAPL_CNTR(cookie->ep, DCNT_EP_RECV_UD);
                  DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_UD_DATA, 
                               cqe_p->byte_len);
                  return (OP_RECV_UD);
            }
            else if (cqe_p->wc_flags & IBV_WC_WITH_IMM) {
                  DAPL_CNTR(cookie->ep, DCNT_EP_RECV_IMM);
                  DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_IMM_DATA, 
                               cqe_p->byte_len);
                  return (OP_RECEIVE_IMM);
            } else {
                  DAPL_CNTR(cookie->ep, DCNT_EP_RECV);
                  DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_DATA, 
                               cqe_p->byte_len);
                  return (OP_RECEIVE);
            }
      case IBV_WC_RECV_RDMA_WITH_IMM:
            DAPL_CNTR(cookie->ep, DCNT_EP_RECV_RDMA_IMM);
            DAPL_CNTR_DATA(cookie->ep, DCNT_EP_RECV_RDMA_IMM_DATA, 
                         cqe_p->byte_len);
            return (OP_RECEIVE_IMM);
      default:
            return (OP_INVALID);
      }
}

#define DAPL_GET_CQE_OPTYPE(cqe_p) dapls_cqe_opcode(cqe_p)
#define DAPL_GET_CQE_WRID(cqe_p) ((ib_work_completion_t*)cqe_p)->wr_id
#define DAPL_GET_CQE_STATUS(cqe_p) ((ib_work_completion_t*)cqe_p)->status
#define DAPL_GET_CQE_VENDOR_ERR(cqe_p) ((ib_work_completion_t*)cqe_p)->vendor_err
#define DAPL_GET_CQE_BYTESNUM(cqe_p) ((ib_work_completion_t*)cqe_p)->byte_len
#define DAPL_GET_CQE_IMMED_DATA(cqe_p) ((ib_work_completion_t*)cqe_p)->imm_data

STATIC _INLINE_ char * dapls_dto_op_str(int op)
{
    static char *optable[] =
    {
        "OP_RDMA_WRITE",
        "OP_RDMA_WRITE_IMM",
        "OP_SEND",
        "OP_SEND_IMM",
        "OP_RDMA_READ",
        "OP_COMP_AND_SWAP",
        "OP_FETCH_AND_ADD",
        "OP_RECEIVE",
        "OP_RECEIVE_MSG_IMM",
      "OP_RECEIVE_RDMA_IMM",
        "OP_BIND_MW"
      "OP_SEND_UD"
      "OP_RECV_UD"
    };
    return ((op < 0 || op > 12) ? "Invalid CQE OP?" : optable[op]);
}

static _INLINE_ char *
dapls_cqe_op_str(IN ib_work_completion_t *cqe_ptr)
{
    return dapls_dto_op_str(DAPL_GET_CQE_OPTYPE(cqe_ptr));
}

#define DAPL_GET_CQE_OP_STR(cqe) dapls_cqe_op_str(cqe)

#endif      /*  _DAPL_IB_DTO_H_ */

Generated by  Doxygen 1.6.0   Back to index