Logo Search packages:      
Sourcecode: dapl version File versions  Download package

dapl_ib_util.c

/*
 * This Software is licensed under one of the following licenses:
 *
 * 1) under the terms of the "Common Public License 1.0" a copy of which is
 *    available from the Open Source Initiative, see
 *    http://www.opensource.org/licenses/cpl.php.
 *
 * 2) under the terms of the "The BSD License" a copy of which is
 *    available from the Open Source Initiative, see
 *    http://www.opensource.org/licenses/bsd-license.php.
 *
 * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
 *    copy of which is available from the Open Source Initiative, see
 *    http://www.opensource.org/licenses/gpl-license.php.
 *
 * Licensee has the right to choose one of the above licenses.
 *
 * Redistributions of source code must retain the above copyright
 * notice and one of the license notices.
 *
 * Redistributions in binary form must reproduce both the above copyright
 * notice, one of the license notices in the documentation
 * and/or other materials provided with the distribution.
 */

/***************************************************************************
 *
 *   Module:             uDAPL
 *
 *   Filename:           dapl_ib_util.c
 *
 *   Author:             Arlin Davis
 *
 *   Created:            3/10/2005
 *
 *   Description: 
 *
 *   The uDAPL openib provider - init, open, close, utilities
 *
 ****************************************************************************
 *             Source Control System Information
 *
 *    $Id: $
 *
 *    Copyright (c) 2005 Intel Corporation.  All rights reserved.
 *
 **************************************************************************/
#ifdef RCSID
static const char rcsid[] = "$Id:  $";
#endif

#include "openib_osd.h"
#include "dapl.h"
#include "dapl_adapter_util.h"
#include "dapl_ib_util.h"
#include "dapl_osd.h"

#include <stdlib.h>

int g_dapl_loopback_connection = 0;
DAPL_SOCKET g_scm[2];

enum ibv_mtu dapl_ib_mtu(int mtu)
{
      switch (mtu) {
      case 256:
            return IBV_MTU_256;
      case 512:
            return IBV_MTU_512;
      case 1024:
            return IBV_MTU_1024;
      case 2048:
            return IBV_MTU_2048;
      case 4096:
            return IBV_MTU_4096;
      default:
            return IBV_MTU_1024;
      }
}

char *dapl_ib_mtu_str(enum ibv_mtu mtu)
{
      switch (mtu) {
      case IBV_MTU_256:
            return "256";
      case IBV_MTU_512:
            return "512";
      case IBV_MTU_1024:
            return "1024";
      case IBV_MTU_2048:
            return "2048";
      case IBV_MTU_4096:
            return "4096";
      default:
            return "1024";
      }
}

static DAT_RETURN getlocalipaddr(DAT_SOCK_ADDR * addr, int addr_len)
{
      struct sockaddr_in *sin;
      struct addrinfo *res, hint, *ai;
      int ret;
      char hostname[256];

      if (addr_len < sizeof(*sin)) {
            return DAT_INTERNAL_ERROR;
      }

      ret = gethostname(hostname, 256);
      if (ret)
            return dapl_convert_errno(ret, "gethostname");

      memset(&hint, 0, sizeof hint);
      hint.ai_flags = AI_PASSIVE;
      hint.ai_family = AF_INET;
      hint.ai_socktype = SOCK_STREAM;
      hint.ai_protocol = IPPROTO_TCP;

      ret = getaddrinfo(hostname, NULL, &hint, &res);
      if (ret) {
            dapl_log(DAPL_DBG_TYPE_ERR,
                   " getaddrinfo ERR: %d %s\n", ret, gai_strerror(ret));
            return DAT_INVALID_ADDRESS;
      }

      ret = DAT_INVALID_ADDRESS;
      for (ai = res; ai; ai = ai->ai_next) {
            sin = (struct sockaddr_in *)ai->ai_addr;
            if (*((uint32_t *) & sin->sin_addr) != htonl(0x7f000001)) {
                  *((struct sockaddr_in *)addr) = *sin;
                  ret = DAT_SUCCESS;
                  break;
            }
      }

      freeaddrinfo(res);
      return ret;
}

/*
 * dapls_ib_init, dapls_ib_release
 *
 * Initialize Verb related items for device open
 *
 * Input:
 *    none
 *
 * Output:
 *    none
 *
 * Returns:
 *    0 success, -1 error
 *
 */
int32_t dapls_ib_init(void)
{
      DAPL_SOCKET listen_socket;
      struct sockaddr_in addr;
      socklen_t addrlen = sizeof(addr);
      int ret;

      listen_socket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
      if (listen_socket == DAPL_INVALID_SOCKET)
            return 1;

      memset(&addr, 0, sizeof addr);
      addr.sin_family = AF_INET;
      addr.sin_addr.s_addr = htonl(0x7f000001);
      ret = bind(listen_socket, (struct sockaddr *)&addr, sizeof addr);
      if (ret)
            goto err1;

      ret = getsockname(listen_socket, (struct sockaddr *)&addr, &addrlen);
      if (ret)
            goto err1;

      ret = listen(listen_socket, 0);
      if (ret)
            goto err1;

      g_scm[1] = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
      if (g_scm[1] == DAPL_INVALID_SOCKET)
            goto err1;

      ret = connect(g_scm[1], (struct sockaddr *)&addr, sizeof(addr));
      if (ret)
            goto err2;

      g_scm[0] = accept(listen_socket, NULL, NULL);
      if (g_scm[0] == DAPL_INVALID_SOCKET)
            goto err2;

      closesocket(listen_socket);
      return 0;

      err2:
      closesocket(g_scm[1]);
      err1:
      closesocket(listen_socket);
      return 1;
}

int32_t dapls_ib_release(void)
{
      closesocket(g_scm[0]);
      closesocket(g_scm[1]);
      return 0;
}

#if defined(_WIN64) || defined(_WIN32)
int dapls_config_comp_channel(struct ibv_comp_channel *channel)
{
      return 0;
}
#else                   // _WIN64 || WIN32
int dapls_config_comp_channel(struct ibv_comp_channel *channel)
{
      int opts;

      opts = fcntl(channel->fd, F_GETFL); /* uCQ */
      if (opts < 0 || fcntl(channel->fd, F_SETFL, opts | O_NONBLOCK) < 0) {
            dapl_log(DAPL_DBG_TYPE_ERR,
                   " dapls_create_comp_channel: fcntl on ib_cq->fd %d ERR %d %s\n",
                   channel->fd, opts, strerror(errno));
            return errno;
      }

      return 0;
}
#endif

/*
 * dapls_ib_open_hca
 *
 * Open HCA
 *
 * Input:
 *      *hca_name         pointer to provider device name
 *      *ib_hca_handle_p  pointer to provide HCA handle
 *
 * Output:
 *      none
 *
 * Return:
 *      DAT_SUCCESS
 *      dapl_convert_errno
 *
 */
DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA * hca_ptr)
{
      struct ibv_device **dev_list;
      struct ibv_port_attr port_attr;
      int i;
      DAT_RETURN dat_status;

      dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
                 " open_hca: %s - %p\n", hca_name, hca_ptr);

      /* get the IP address of the device */
      dat_status = getlocalipaddr((DAT_SOCK_ADDR *) & hca_ptr->hca_address,
                            sizeof(DAT_SOCK_ADDR6));
      if (dat_status != DAT_SUCCESS)
            return dat_status;

      /* Get list of all IB devices, find match, open */
      dev_list = ibv_get_device_list(NULL);
      if (!dev_list) {
            dapl_dbg_log(DAPL_DBG_TYPE_ERR,
                       " open_hca: ibv_get_device_list() failed\n",
                       hca_name);
            return DAT_INTERNAL_ERROR;
      }

      for (i = 0; dev_list[i]; ++i) {
            hca_ptr->ib_trans.ib_dev = dev_list[i];
            if (!strcmp(ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
                      hca_name))
                  goto found;
      }

      dapl_log(DAPL_DBG_TYPE_ERR,
             " open_hca: device %s not found\n", hca_name);
      goto err;

      found:
      dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " open_hca: Found dev %s %016llx\n",
                 ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
                 (unsigned long long)
                 ntohll(ibv_get_device_guid(hca_ptr->ib_trans.ib_dev)));

      hca_ptr->ib_hca_handle = ibv_open_device(hca_ptr->ib_trans.ib_dev);
      if (!hca_ptr->ib_hca_handle) {
            dapl_log(DAPL_DBG_TYPE_ERR,
                   " open_hca: dev open failed for %s, err=%s\n",
                   ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
                   strerror(errno));
            goto err;
      }

      /* get lid for this hca-port, network order */
      if (ibv_query_port(hca_ptr->ib_hca_handle,
                     (uint8_t) hca_ptr->port_num, &port_attr)) {
            dapl_log(DAPL_DBG_TYPE_ERR,
                   " open_hca: get lid ERR for %s, err=%s\n",
                   ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
                   strerror(errno));
            goto err;
      } else {
            hca_ptr->ib_trans.lid = htons(port_attr.lid);
      }

      /* get gid for this hca-port, network order */
      if (ibv_query_gid(hca_ptr->ib_hca_handle,
                    (uint8_t) hca_ptr->port_num,
                    0, &hca_ptr->ib_trans.gid)) {
            dapl_log(DAPL_DBG_TYPE_ERR,
                   " open_hca: query GID ERR for %s, err=%s\n",
                   ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
                   strerror(errno));
            goto err;
      }

      /* set RC tunables via enviroment or default */
      hca_ptr->ib_trans.max_inline_send =
          dapl_os_get_env_val("DAPL_MAX_INLINE", INLINE_SEND_DEFAULT);
      hca_ptr->ib_trans.ack_retry =
          dapl_os_get_env_val("DAPL_ACK_RETRY", SCM_ACK_RETRY);
      hca_ptr->ib_trans.ack_timer =
          dapl_os_get_env_val("DAPL_ACK_TIMER", SCM_ACK_TIMER);
      hca_ptr->ib_trans.rnr_retry =
          dapl_os_get_env_val("DAPL_RNR_RETRY", SCM_RNR_RETRY);
      hca_ptr->ib_trans.rnr_timer =
          dapl_os_get_env_val("DAPL_RNR_TIMER", SCM_RNR_TIMER);
      hca_ptr->ib_trans.global =
          dapl_os_get_env_val("DAPL_GLOBAL_ROUTING", SCM_GLOBAL);
      hca_ptr->ib_trans.hop_limit =
          dapl_os_get_env_val("DAPL_HOP_LIMIT", SCM_HOP_LIMIT);
      hca_ptr->ib_trans.tclass =
          dapl_os_get_env_val("DAPL_TCLASS", SCM_TCLASS);
      hca_ptr->ib_trans.mtu =
          dapl_ib_mtu(dapl_os_get_env_val("DAPL_IB_MTU", SCM_IB_MTU));

#ifndef CQ_WAIT_OBJECT
      /* initialize cq_lock */
      dat_status = dapl_os_lock_init(&hca_ptr->ib_trans.cq_lock);
      if (dat_status != DAT_SUCCESS) {
            dapl_log(DAPL_DBG_TYPE_ERR,
                   " open_hca: failed to init cq_lock\n");
            goto bail;
      }
      /* EVD events without direct CQ channels, non-blocking */
      hca_ptr->ib_trans.ib_cq =
          ibv_create_comp_channel(hca_ptr->ib_hca_handle);
      if (hca_ptr->ib_trans.ib_cq == NULL) {
            dapl_log(DAPL_DBG_TYPE_ERR,
                   " open_hca: ibv_create_comp_channel ERR %s\n",
                   strerror(errno));
            goto bail;
      }

      if (dapls_config_comp_channel(hca_ptr->ib_trans.ib_cq)) {
            goto bail;
      }

      if (dapli_cq_thread_init(hca_ptr)) {
            dapl_log(DAPL_DBG_TYPE_ERR,
                   " open_hca: cq_thread_init failed for %s\n",
                   ibv_get_device_name(hca_ptr->ib_trans.ib_dev));
            goto bail;
      }
#endif                        /* CQ_WAIT_OBJECT */

      /* initialize cr_list lock */
      dat_status = dapl_os_lock_init(&hca_ptr->ib_trans.lock);
      if (dat_status != DAT_SUCCESS) {
            dapl_log(DAPL_DBG_TYPE_ERR,
                   " open_hca: failed to init cr_list lock\n");
            goto bail;
      }

      /* initialize CM list for listens on this HCA */
      dapl_llist_init_head(&hca_ptr->ib_trans.list);

      /* create thread to process inbound connect request */
      hca_ptr->ib_trans.cr_state = IB_THREAD_INIT;
      dat_status = dapl_os_thread_create(cr_thread,
                                 (void *)hca_ptr,
                                 &hca_ptr->ib_trans.thread);
      if (dat_status != DAT_SUCCESS) {
            dapl_log(DAPL_DBG_TYPE_ERR,
                   " open_hca: failed to create thread\n");
            goto bail;
      }

      /* wait for thread */
      while (hca_ptr->ib_trans.cr_state != IB_THREAD_RUN) {
            dapl_os_sleep_usec(2000);
      }

      dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
                 " open_hca: devname %s, port %d, hostname_IP %s\n",
                 ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
                 hca_ptr->port_num, inet_ntoa(((struct sockaddr_in *)
                                       &hca_ptr->hca_address)->
                                      sin_addr));
      dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
                 " open_hca: LID 0x%x GID Subnet 0x" F64x " ID 0x" F64x
                 "\n", ntohs(hca_ptr->ib_trans.lid), (unsigned long long)
                 htonll(hca_ptr->ib_trans.gid.global.subnet_prefix),
                 (unsigned long long)htonll(hca_ptr->ib_trans.gid.global.
                                    interface_id));

      ibv_free_device_list(dev_list);
      return dat_status;

      bail:
      ibv_close_device(hca_ptr->ib_hca_handle);
      hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
      err:
      ibv_free_device_list(dev_list);
      return DAT_INTERNAL_ERROR;
}

/*
 * dapls_ib_close_hca
 *
 * Open HCA
 *
 * Input:
 *      DAPL_HCA   provide CA handle
 *
 * Output:
 *      none
 *
 * Return:
 *      DAT_SUCCESS
 *    dapl_convert_errno 
 *
 */
DAT_RETURN dapls_ib_close_hca(IN DAPL_HCA * hca_ptr)
{
      dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " close_hca: %p\n", hca_ptr);

#ifndef CQ_WAIT_OBJECT
      dapli_cq_thread_destroy(hca_ptr);
      dapl_os_lock_destroy(&hca_ptr->ib_trans.cq_lock);
#endif                        /* CQ_WAIT_OBJECT */

      if (hca_ptr->ib_hca_handle != IB_INVALID_HANDLE) {
            if (ibv_close_device(hca_ptr->ib_hca_handle))
                  return (dapl_convert_errno(errno, "ib_close_device"));
            hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
      }

      /* destroy cr_thread and lock */
      hca_ptr->ib_trans.cr_state = IB_THREAD_CANCEL;
      if (send(g_scm[1], "w", sizeof "w", 0) == -1)
            dapl_log(DAPL_DBG_TYPE_UTIL,
                   " thread_destroy: thread wakeup err = %s\n",
                   strerror(errno));
      while (hca_ptr->ib_trans.cr_state != IB_THREAD_EXIT) {
            dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
                       " close_hca: waiting for cr_thread\n");
            if (send(g_scm[1], "w", sizeof "w", 0) == -1)
                  dapl_log(DAPL_DBG_TYPE_UTIL,
                         " thread_destroy: thread wakeup err = %s\n",
                         strerror(errno));
            dapl_os_sleep_usec(2000);
      }
      dapl_os_lock_destroy(&hca_ptr->ib_trans.lock);

      return (DAT_SUCCESS);
}

/*
 * dapls_ib_query_hca
 *
 * Query the hca attribute
 *
 * Input:
 *    hca_handl         hca handle  
 *    ia_attr                 attribute of the ia
 *    ep_attr                 attribute of the ep
 *    ip_addr                 ip address of DET NIC
 *
 * Output:
 *    none
 *
 * Returns:
 *    DAT_SUCCESS
 *    DAT_INVALID_HANDLE
 */

DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA * hca_ptr,
                        OUT DAT_IA_ATTR * ia_attr,
                        OUT DAT_EP_ATTR * ep_attr,
                        OUT DAT_SOCK_ADDR6 * ip_addr)
{
      struct ibv_device_attr dev_attr;
      struct ibv_port_attr port_attr;

      if (hca_ptr->ib_hca_handle == NULL) {
            dapl_dbg_log(DAPL_DBG_TYPE_ERR, " query_hca: BAD handle\n");
            return (DAT_INVALID_HANDLE);
      }

      /* local IP address of device, set during ia_open */
      if (ip_addr != NULL)
            memcpy(ip_addr, &hca_ptr->hca_address, sizeof(DAT_SOCK_ADDR6));

      if (ia_attr == NULL && ep_attr == NULL)
            return DAT_SUCCESS;

      /* query verbs for this device and port attributes */
      if (ibv_query_device(hca_ptr->ib_hca_handle, &dev_attr) ||
          ibv_query_port(hca_ptr->ib_hca_handle,
                     hca_ptr->port_num, &port_attr))
            return (dapl_convert_errno(errno, "ib_query_hca"));

      if (ia_attr != NULL) {
            (void)dapl_os_memzero(ia_attr, sizeof(*ia_attr));
            ia_attr->adapter_name[DAT_NAME_MAX_LENGTH - 1] = '\0';
            ia_attr->vendor_name[DAT_NAME_MAX_LENGTH - 1] = '\0';
            ia_attr->ia_address_ptr =
                (DAT_IA_ADDRESS_PTR) & hca_ptr->hca_address;

            dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
                       " query_hca: %s %s \n",
                       ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
                       inet_ntoa(((struct sockaddr_in *)
                              &hca_ptr->hca_address)->sin_addr));

            ia_attr->hardware_version_major = dev_attr.hw_ver;
            /* ia_attr->hardware_version_minor   = dev_attr.fw_ver; */
            ia_attr->max_eps = dev_attr.max_qp;
            ia_attr->max_dto_per_ep = dev_attr.max_qp_wr;
            ia_attr->max_rdma_read_in = dev_attr.max_qp_rd_atom;
            ia_attr->max_rdma_read_out = dev_attr.max_qp_init_rd_atom;
            ia_attr->max_rdma_read_per_ep_in = dev_attr.max_qp_rd_atom;
            ia_attr->max_rdma_read_per_ep_out =
                dev_attr.max_qp_init_rd_atom;
            ia_attr->max_rdma_read_per_ep_in_guaranteed = DAT_TRUE;
            ia_attr->max_rdma_read_per_ep_out_guaranteed = DAT_TRUE;
            ia_attr->max_evds = dev_attr.max_cq;
            ia_attr->max_evd_qlen = dev_attr.max_cqe;
            ia_attr->max_iov_segments_per_dto = dev_attr.max_sge;
            ia_attr->max_lmrs = dev_attr.max_mr;
            /* 32bit attribute from 64bit, 4G-1 limit, DAT v2 needs fix */
            ia_attr->max_lmr_block_size = 
                (dev_attr.max_mr_size >> 32) ? ~0 : dev_attr.max_mr_size;
            ia_attr->max_rmrs = dev_attr.max_mw;
            ia_attr->max_lmr_virtual_address = dev_attr.max_mr_size;
            ia_attr->max_rmr_target_address = dev_attr.max_mr_size;
            ia_attr->max_pzs = dev_attr.max_pd;
            ia_attr->max_message_size = port_attr.max_msg_sz;
            ia_attr->max_rdma_size = port_attr.max_msg_sz;
            ia_attr->max_iov_segments_per_rdma_read = dev_attr.max_sge;
            ia_attr->max_iov_segments_per_rdma_write = dev_attr.max_sge;
            ia_attr->num_transport_attr = 0;
            ia_attr->transport_attr = NULL;
            ia_attr->num_vendor_attr = 0;
            ia_attr->vendor_attr = NULL;
#ifdef DAT_EXTENSIONS
            ia_attr->extension_supported = DAT_EXTENSION_IB;
            ia_attr->extension_version = DAT_IB_EXTENSION_VERSION;
#endif
            hca_ptr->ib_trans.mtu = DAPL_MIN(port_attr.active_mtu,
                                     hca_ptr->ib_trans.mtu);
            hca_ptr->ib_trans.ack_timer =
                DAPL_MAX(dev_attr.local_ca_ack_delay,
                       hca_ptr->ib_trans.ack_timer);

            /* set MTU in transport specific named attribute */
            hca_ptr->ib_trans.named_attr.name = "DAT_IB_TRANSPORT_MTU";
            hca_ptr->ib_trans.named_attr.value =
                dapl_ib_mtu_str(hca_ptr->ib_trans.mtu);

            dapl_log(DAPL_DBG_TYPE_UTIL,
                       " query_hca: (%x.%x) ep %d ep_q %d evd %d"
                       " evd_q %d mtu %d\n",
                       ia_attr->hardware_version_major,
                       ia_attr->hardware_version_minor,
                       ia_attr->max_eps, ia_attr->max_dto_per_ep,
                       ia_attr->max_evds, ia_attr->max_evd_qlen,
                       128 << hca_ptr->ib_trans.mtu);

            dapl_log(DAPL_DBG_TYPE_UTIL,
                       " query_hca: msg %llu rdma %llu iov %d lmr %d rmr %d"
                       " ack_time %d mr %u\n",
                       ia_attr->max_message_size, ia_attr->max_rdma_size,
                       ia_attr->max_iov_segments_per_dto,
                       ia_attr->max_lmrs, ia_attr->max_rmrs,
                       hca_ptr->ib_trans.ack_timer,
                       ia_attr->max_lmr_block_size);
      }

      if (ep_attr != NULL) {
            (void)dapl_os_memzero(ep_attr, sizeof(*ep_attr));
            ep_attr->max_message_size = port_attr.max_msg_sz;
            ep_attr->max_rdma_size = port_attr.max_msg_sz;
            ep_attr->max_recv_dtos = dev_attr.max_qp_wr;
            ep_attr->max_request_dtos = dev_attr.max_qp_wr;
            ep_attr->max_recv_iov = dev_attr.max_sge;
            ep_attr->max_request_iov = dev_attr.max_sge;
            ep_attr->max_rdma_read_in = dev_attr.max_qp_rd_atom;
            ep_attr->max_rdma_read_out = dev_attr.max_qp_init_rd_atom;
            ep_attr->max_rdma_read_iov = dev_attr.max_sge;
            ep_attr->max_rdma_write_iov = dev_attr.max_sge;
            dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
                       " query_hca: MAX msg %llu mtu %d dto %d iov %d"
                       " rdma i%d,o%d\n",
                       ep_attr->max_message_size,
                       ep_attr->max_recv_dtos, ep_attr->max_recv_iov,
                       ep_attr->max_rdma_read_in,
                       ep_attr->max_rdma_read_out);
      }
      return DAT_SUCCESS;
}

/*
 * dapls_ib_setup_async_callback
 *
 * Set up an asynchronous callbacks of various kinds
 *
 * Input:
 *    ia_handle         IA handle
 *    handler_type            type of handler to set up
 *    callback_handle   handle param for completion callbacks
 *    callback          callback routine pointer
 *    context           argument for callback routine
 *
 * Output:
 *    none
 *
 * Returns:
 *    DAT_SUCCESS
 *    DAT_INSUFFICIENT_RESOURCES
 *    DAT_INVALID_PARAMETER
 *
 */
DAT_RETURN dapls_ib_setup_async_callback(IN DAPL_IA * ia_ptr,
                               IN DAPL_ASYNC_HANDLER_TYPE
                               handler_type, IN DAPL_EVD * evd_ptr,
                               IN ib_async_handler_t callback,
                               IN void *context)
{
      ib_hca_transport_t *hca_ptr;

      dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
                 " setup_async_cb: ia %p type %d handle %p cb %p ctx %p\n",
                 ia_ptr, handler_type, evd_ptr, callback, context);

      hca_ptr = &ia_ptr->hca_ptr->ib_trans;
      switch (handler_type) {
      case DAPL_ASYNC_UNAFILIATED:
            hca_ptr->async_unafiliated = (ib_async_handler_t) callback;
            hca_ptr->async_un_ctx = context;
            break;
      case DAPL_ASYNC_CQ_ERROR:
            hca_ptr->async_cq_error = (ib_async_cq_handler_t) callback;
            break;
      case DAPL_ASYNC_CQ_COMPLETION:
            hca_ptr->async_cq = (ib_async_dto_handler_t) callback;
            break;
      case DAPL_ASYNC_QP_ERROR:
            hca_ptr->async_qp_error = (ib_async_qp_handler_t) callback;
            break;
      default:
            break;
      }
      return DAT_SUCCESS;
}

/*
 * dapls_set_provider_specific_attr
 *
 * Input:
 *      attr_ptr        Pointer provider specific attributes
 *
 * Output:
 *      none
 *
 * Returns:
 *      void
 */
DAT_NAMED_ATTR ib_attrs[] = {
      {
       "DAT_IB_TRANSPORT_MTU", "1024"}
      ,
#ifdef DAT_EXTENSIONS
      {
       "DAT_EXTENSION_INTERFACE", "TRUE"}
      ,
      {
       DAT_IB_ATTR_FETCH_AND_ADD, "TRUE"}
      ,
      {
       DAT_IB_ATTR_CMP_AND_SWAP, "TRUE"}
      ,
      {
       DAT_IB_ATTR_IMMED_DATA, "TRUE"}
      ,
      {
       DAT_IB_ATTR_UD, "TRUE"}
      ,
#ifdef DAPL_COUNTERS
      {
       DAT_ATTR_COUNTERS, "TRUE"}
      ,
#endif                        /* DAPL_COUNTERS */
#endif
};

#define SPEC_ATTR_SIZE( x )     (sizeof( x ) / sizeof( DAT_NAMED_ATTR))

void dapls_query_provider_specific_attr(IN DAPL_IA * ia_ptr,
                              IN DAT_PROVIDER_ATTR * attr_ptr)
{
      attr_ptr->num_provider_specific_attr = SPEC_ATTR_SIZE(ib_attrs);
      attr_ptr->provider_specific_attr = ib_attrs;

      /* set MTU to actual settings */
      ib_attrs[0].value = ia_ptr->hca_ptr->ib_trans.named_attr.value;
}

Generated by  Doxygen 1.6.0   Back to index