diff --git a/usr/Makefile b/usr/Makefile index ea175cd..0f5b454 100644 --- a/usr/Makefile +++ b/usr/Makefile @@ -13,9 +13,11 @@ TGTD_OBJS += $(addprefix iscsi/, conn.o param.o session.o \ isns.o) TGTD_OBJS += bs_rdwr.o bs_aio.o -ifneq ($(ISCSI_RDMA),) +ifneq ($(findstring verbs.h,$(wildcard /usr/include/infiniband/*.h)), ) +ifneq ($(findstring rdma_cma.h,$(wildcard /usr/include/rdma/*.h)), ) TGTD_OBJS += iscsi/iser.o iscsi/iser_text.o -LIBS += -libverbs -lrdmacm +LIBS += -libverbs -lrdmacm -ldl +endif endif ifneq ($(FCOE),) diff --git a/usr/iscsi/iser.c b/usr/iscsi/iser.c index d0e46e3..3904aaf 100644 --- a/usr/iscsi/iser.c +++ b/usr/iscsi/iser.c @@ -32,6 +32,8 @@ #include <sys/epoll.h> #include <infiniband/verbs.h> #include <rdma/rdma_cma.h> +#include <dlfcn.h> +#include <syslog.h> #include "util.h" #include "iscsid.h" @@ -92,6 +94,86 @@ struct iscsi_sense_data { uint8_t data[0]; } __packed; +/* handle dynamic loading of ib libs */ +static void *pverbs; +static void *prdma; +typedef void (*ibv_ack_cq_events_t)(struct ibv_cq *cq, unsigned int nevents); +typedef struct ibv_pd *(*ibv_alloc_pd_t)(struct ibv_context *context); +typedef struct ibv_comp_channel *(*ibv_create_comp_channel_t)(struct ibv_context *context); +typedef struct ibv_cq *(*ibv_create_cq_t)(struct ibv_context *context, int cqe, + void *cq_context, + struct ibv_comp_channel *channel, + int comp_vector); +typedef int (*ibv_dereg_mr_t)(struct ibv_mr *mr); +typedef int (*ibv_destroy_qp_t)(struct ibv_qp *qp); +typedef int (*ibv_get_cq_event_t)(struct ibv_comp_channel *channel, + struct ibv_cq **cq, void **cq_context); +typedef int (*ibv_query_device_t)(struct ibv_context *context, + struct ibv_device_attr *device_attr); +typedef struct ibv_mr *(*ibv_reg_mr_t)(struct ibv_pd *pd, void *addr, + size_t length, enum ibv_access_flags access); +typedef int (*rdma_ack_cm_event_t)(struct rdma_cm_event *event); +typedef int (*rdma_bind_addr_t)(struct rdma_cm_id *id, struct sockaddr *addr); +typedef struct rdma_event_channel *(*rdma_create_event_channel_t)(void); +typedef int (*rdma_create_id_t)(struct rdma_event_channel *channel, + struct rdma_cm_id **id, void *context, + enum rdma_port_space ps); +typedef int (*rdma_create_qp_t)(struct rdma_cm_id *id, struct ibv_pd *pd, + struct ibv_qp_init_attr *qp_init_attr); +typedef int (*rdma_destroy_id_t)(struct rdma_cm_id *id); +typedef int (*rdma_disconnect_t)(struct rdma_cm_id *id); +typedef int (*rdma_get_cm_event_t)(struct rdma_event_channel *channel, + struct rdma_cm_event **event); +typedef int (*rdma_listen_t)(struct rdma_cm_id *id, int backlog); +typedef int (*rdma_accept_t)(struct rdma_cm_id *id, struct rdma_conn_param *conn_param); +typedef int (*rdma_reject_t)(struct rdma_cm_id *id, const void *private_data, + uint8_t private_data_len); +typedef struct { + ibv_ack_cq_events_t ibv_ack_cq_events; + ibv_alloc_pd_t ibv_alloc_pd; + ibv_create_comp_channel_t ibv_create_comp_channel; + ibv_create_cq_t ibv_create_cq; + ibv_dereg_mr_t ibv_dereg_mr; + ibv_destroy_qp_t ibv_destroy_qp; + ibv_get_cq_event_t ibv_get_cq_event; + ibv_query_device_t ibv_query_device; + ibv_reg_mr_t ibv_reg_mr; + rdma_ack_cm_event_t rdma_ack_cm_event; + rdma_bind_addr_t rdma_bind_addr; + rdma_create_event_channel_t rdma_create_event_channel; + rdma_create_id_t rdma_create_id; + rdma_create_qp_t rdma_create_qp; + rdma_destroy_id_t rdma_destroy_id; + rdma_disconnect_t rdma_disconnect; + rdma_get_cm_event_t rdma_get_cm_event; + rdma_listen_t rdma_listen; + rdma_accept_t rdma_accept; + rdma_reject_t rdma_reject; +} tgt_fptr_t; + +static tgt_fptr_t tgt_fptr; + +#define ibv_ack_cq_events(x...) tgt_fptr.ibv_ack_cq_events(x) +#define ibv_alloc_pd(x...) tgt_fptr.ibv_alloc_pd(x) +#define ibv_create_comp_channel(x...) tgt_fptr.ibv_create_comp_channel(x) +#define ibv_create_cq(x...) tgt_fptr.ibv_create_cq(x) +#define ibv_dereg_mr(x...) tgt_fptr.ibv_dereg_mr(x) +#define ibv_destroy_qp(x...) tgt_fptr.ibv_destroy_qp(x) +#define ibv_get_cq_event(x...) tgt_fptr.ibv_get_cq_event(x) +#define ibv_query_device(x...) tgt_fptr.ibv_query_device(x) +#define ibv_reg_mr(x...) tgt_fptr.ibv_reg_mr(x) +#define rdma_ack_cm_event(x...) tgt_fptr.rdma_ack_cm_event(x) +#define rdma_bind_addr(x...) tgt_fptr.rdma_bind_addr(x) +#define rdma_create_event_channel(x...) tgt_fptr.rdma_create_event_channel(x) +#define rdma_create_id(x...) tgt_fptr.rdma_create_id(x) +#define rdma_create_qp(x...) tgt_fptr.rdma_create_qp(x) +#define rdma_destroy_id(x...) tgt_fptr.rdma_destroy_id(x) +#define rdma_disconnect(x...) tgt_fptr.rdma_disconnect(x) +#define rdma_get_cm_event(x...) tgt_fptr.rdma_get_cm_event(x) +#define rdma_listen(x...) tgt_fptr.rdma_listen(x) +#define rdma_accept(x...) tgt_fptr.rdma_accept(x) +#define rdma_reject(x...) tgt_fptr.rdma_reject(x) + /* * Number of allocatable data buffers, each of this size. Do at least 128 * for linux iser. The membuf size is rounded up at initialization time @@ -3331,10 +3413,94 @@ static void iser_nop_work_handler(void *data) add_work(&nop_work, ISER_TIMER_INT_SEC); } +static int is_dlerror(const char *symbol) +{ + char *error; + + if ((error = dlerror()) != NULL) { + syslog(LOG_ERR, "symbol %s not found - %s", symbol, error); + return 1; + } + return 0; +} + static int iser_init(int index, char *args) { int err; + pverbs = dlopen("libibverbs.so",RTLD_LAZY); + if (!pverbs) { + goto Exit; /* do not register iser transport */ + } + + prdma = dlopen("librdmacm.so",RTLD_LAZY); + if (!prdma) { + goto Exit; /* do not register iser transport */ + } + + /* initialize function pointers */ + tgt_fptr.ibv_ack_cq_events = dlsym(pverbs, "ibv_ack_cq_events"); + if (is_dlerror("ibv_ack_cq_events")) + goto Exit; + tgt_fptr.ibv_alloc_pd = dlsym(pverbs, "ibv_alloc_pd"); + if (is_dlerror("ibv_alloc_pd")) + goto Exit; + tgt_fptr.ibv_create_comp_channel = dlsym(pverbs, "ibv_create_comp_channel"); + if (is_dlerror("ibv_create_comp_channel")) + goto Exit; + tgt_fptr.ibv_create_cq = dlsym(pverbs, "ibv_create_cq"); + if (is_dlerror("ibv_create_cq")) + goto Exit; + tgt_fptr.ibv_dereg_mr = dlsym(pverbs, "ibv_dereg_mr"); + if (is_dlerror("ibv_dereg_mr")) + goto Exit; + tgt_fptr.ibv_destroy_qp = dlsym(pverbs, "ibv_destroy_qp"); + if (is_dlerror("ibv_destroy_qp")) + goto Exit; + tgt_fptr.ibv_get_cq_event = dlsym(pverbs, "ibv_get_cq_event"); + if (is_dlerror("ibv_get_cq_event")) + goto Exit; + tgt_fptr.ibv_query_device = dlsym(pverbs, "ibv_query_device"); + if (is_dlerror("ibv_query_device")) + goto Exit; + tgt_fptr.ibv_reg_mr = dlsym(pverbs, "ibv_reg_mr"); + if (is_dlerror("ibv_reg_mr")) + goto Exit; + tgt_fptr.rdma_ack_cm_event = dlsym(prdma, "rdma_ack_cm_event"); + if (is_dlerror("rdma_ack_cm_event")) + goto Exit; + tgt_fptr.rdma_bind_addr = dlsym(prdma, "rdma_bind_addr"); + if (is_dlerror("rdma_bind_addr")) + goto Exit; + tgt_fptr.rdma_create_event_channel = dlsym(prdma, "rdma_create_event_channel"); + if (is_dlerror("rdma_create_event_channel")) + goto Exit; + tgt_fptr.rdma_create_id = dlsym(prdma, "rdma_create_id"); + if (is_dlerror("rdma_create_id")) + goto Exit; + tgt_fptr.rdma_create_qp = dlsym(prdma, "rdma_create_qp"); + if (is_dlerror("rdma_create_qp")) + goto Exit; + tgt_fptr.rdma_destroy_id = dlsym(prdma, "rdma_destroy_id"); + if (is_dlerror("rdma_destroy_id")) + goto Exit; + tgt_fptr.rdma_disconnect = dlsym(prdma, "rdma_disconnect"); + if (is_dlerror("rdma_disconnect")) + goto Exit; + tgt_fptr.rdma_get_cm_event = dlsym(prdma, "rdma_get_cm_event"); + if (is_dlerror("rdma_get_cm_event")) + goto Exit; + tgt_fptr.rdma_listen = dlsym(prdma, "rdma_listen"); + if (is_dlerror("rdma_listen")) + goto Exit; + tgt_fptr.rdma_accept = dlsym(prdma, "rdma_accept"); + if (is_dlerror("rdma_accept")) + goto Exit; + tgt_fptr.rdma_reject = dlsym(prdma, "rdma_reject"); + if (is_dlerror("rdma_reject")) + goto Exit; + + err = iser_ib_init(); if (err) return err; @@ -3347,12 +3513,25 @@ static int iser_init(int index, char *args) } return 0; + +Exit: + syslog(LOG_ERR, "%s - iser transport not used", dlerror()); + if (pverbs) + dlclose(pverbs); + if (prdma) + dlclose(prdma); + return -ENOTSUP; } static void iser_exit(void) { if (iser_send_nop) del_work(&nop_work); + + if (pverbs) + dlclose(pverbs); + if (prdma) + dlclose(prdma); } static int iser_target_create(struct target *t)