@ -3,6 +3,7 @@
# include <stdio.h>
# include <stdlib.h>
# include <sys/mman.h>
# include "msgr_rdma.h"
# include "messenger.h"
@ -54,6 +55,7 @@ msgr_rdma_connection_t::~msgr_rdma_connection_t()
msgr_rdma_context_t * msgr_rdma_context_t : : create ( const char * ib_devname , uint8_t ib_port , uint8_t gid_index , uint32_t mtu , int log_level )
{
int res ;
bool odp = true ;
ibv_device * * dev_list = NULL ;
msgr_rdma_context_t * ctx = new msgr_rdma_context_t ( ) ;
ctx - > mtu = mtu ;
@ -117,9 +119,9 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
fprintf ( stderr , " RDMA device %s must have local LID because it's not Ethernet, but LID is zero \n " , ibv_get_device_name ( ctx - > dev ) ) ;
goto cleanup ;
}
if ( ibv_query_gid ( ctx - > context , ib_port , gid_index , & ctx - > my_gid ) )
if ( ( res = ibv_query_gid ( ctx - > context , ib_port , gid_index , & ctx - > my_gid ) ) ! = 0 )
{
fprintf ( stderr , " Couldn't read RDMA device %s GID index %d \n " , ibv_get_device_name ( ctx - > dev ) , gid_index ) ;
fprintf ( stderr , " Couldn't read RDMA device %s GID index %d: %s \n " , ibv_get_device_name ( ctx - > dev ) , gid_index , strerror ( res ) ) ;
goto cleanup ;
}
@ -131,9 +133,9 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
}
{
if ( ibv_query_device_ex ( ctx - > context , NULL , & ctx - > attrx ) )
if ( ( res = ibv_query_device_ex ( ctx - > context , NULL , & ctx - > attrx ) ) ! = 0 )
{
fprintf ( stderr , " Couldn't query RDMA device for its features \n " ) ;
fprintf ( stderr , " Couldn't query RDMA device %s for its features: % s \n " , ibv_get_device_name ( ctx - > dev ) , strerror ( res ) ) ;
goto cleanup ;
}
if ( ! ( ctx - > attrx . odp_caps . general_caps & IBV_ODP_SUPPORT ) | |
@ -141,15 +143,20 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
! ( ctx - > attrx . odp_caps . per_transport_caps . rc_odp_caps & IBV_ODP_SUPPORT_SEND ) | |
! ( ctx - > attrx . odp_caps . per_transport_caps . rc_odp_caps & IBV_ODP_SUPPORT_RECV ) )
{
fprintf ( stderr , " The RDMA device isn't implicit ODP (On-Demand Paging) capable or does not support RC send and receive with ODP \n " ) ;
goto cleanup ;
fprintf ( stderr , " Warning: RDMA device isn't implicit ODP (On-Demand Paging) capable, trying to lock all application memory \n " ) ;
if ( mlockall ( MCL_CURRENT | MCL_FUTURE | MCL_ONFAULT ) ! = 0 )
{
fprintf ( stderr , " mlockall() failed: %s \n " , strerror ( errno ) ) ;
goto cleanup ;
}
odp = false ;
}
}
ctx - > mr = ibv_reg_mr ( ctx - > pd , NULL , SIZE_MAX , IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_ON_DEMAND ) ;
ctx - > mr = ibv_reg_mr ( ctx - > pd , NULL , SIZE_MAX , IBV_ACCESS_LOCAL_WRITE | ( odp ? IBV_ACCESS_ON_DEMAND : 0 ) ) ;
if ( ! ctx - > mr )
{
fprintf ( stderr , " Couldn't register RDMA memory region \n " ) ;
fprintf ( stderr , " Couldn't register RDMA memory region: %s \n " , strerror ( errno ) ) ;
goto cleanup ;
}