vitalif
/
qperf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added support for XRC, fixed bugs

master
Johann George 2008-11-16 18:10:02 -08:00
parent b41f827234
commit bee05d35b0
10 changed files with 650 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
AUTHORS
View File

@ -4,3 +4,4 @@ Thanks to the following for their contributions:
Dotan Barak
Ralph Campbell
Yevgeny Kliteynik
Dave Olson

4
configure.in
View File

@ -1,5 +1,5 @@
AC_INIT(qperf, 0.4.1, general@lists.openfabrics.org)
AM_INIT_AUTOMAKE(qperf, 0.4.1)
AC_INIT(qperf, 0.4.2, general@lists.openfabrics.org)
AM_INIT_AUTOMAKE(qperf, 0.4.2)
AC_PROG_CC
AC_CHECK_LIB(ibverbs, ibv_open_device, RDMA=1)
AC_CHECK_LIB(rdmacm, rdma_create_id)

2
qperf.spec
View File

@ -1,6 +1,6 @@
Name: qperf
Summary: Measure socket and RDMA performance
Version: 0.4.1
Version: 0.4.2
Release: 1
License: BSD 3-Clause, GPL v2
Group: Networking/Diagnostic

85
src/help.txt
View File

@ -94,6 +94,9 @@ Categories +RDMA
udp_lat
ver_rc_compare_swap
ver_rc_fetch_add
xrc_bi_bw
xrc_bw
xrc_lat
Examples
In these examples, we first run qperf on a node called myserver in server
mode by invoking it with no arguments. In all the subsequent examples, we
@ -111,12 +114,15 @@ Examples
qperf myserver rc_bi_bw
Opts
--access_recv OnOff (-ar) Turn on/off accessing received data
-aro Cause received data to be accessed
-ar1 Cause received data to be accessed
--alt_port Port (-ap) Set alternate path port
--loc_alt_port Port (-lap) Set local alternate path port
--rem_alt_port Port (-rap) Set remote alternate path port
--cpu_affinity PN (-ca) Set processor affinity
--loc_cpu_affinity PN (-lca) Set local processor affinity
--rem_cpu_affinity PN (-rca) Set remote processor affinity
--flip OnOff (-f) Flip on/off sender and receiver
-fo Flip (on) sender and receiver
-f1 Flip (on) sender and receiver
--help Topic (-h) Get more information on a topic
--host Node (-H) Identify server node
--id Device:Port (-i) Set RDMA device and port
@ -129,9 +135,9 @@ Opts
--cq_poll OnOff Set polling mode on/off
--loc_cq_poll OnOff (-lcp) Set local polling mode on/off
--rem_cq_poll OnOff (-rcp) Set remote polling mode on/off
-cpo Turn polling mode on
-lcpo Turn local polling mode on
-rcpo Turn remote polling mode on
-cp1 Turn polling mode on
-lcp1 Turn local polling mode on
-rcp1 Turn remote polling mode on
--ip_port Port (-ip) Set TCP port used for tests
--precision Digits (-e) Set precision reported
--rd_atomic Max (-nr) Set RDMA read/atomic count
@ -154,7 +160,7 @@ Opts
--unify_units (-uu) Unify units
--use_bits_per_sec (-ub) Use bits/sec rather than bytes/sec
--use_cm OnOff (-cm) Use RDMA Connection Manager or not
-cmo Use RDMA Connection Manager
-cm1 Use RDMA Connection Manager
--verbose (-v) Verbose; turn on all of -v[cstu]
--verbose_conf (-vc) Show configuration information
--verbose_stat (-vs) Show statistical information
@ -172,8 +178,14 @@ Options
If OnOff is non-zero, data is accessed once received. Otherwise,
data is ignored. By default, OnOff is 0. This can help to mimic
some applications.
-aro
-ar1
Cause received data to be accessed.
--alt_port Port (-ap)
Set alternate path port. This enables automatic path failover.
--loc_alt_port Port (-lap)
Set local alternate path port. This enables automatic path failover.
--rem_alt_port Port (-rap)
Set remote alternate path port. This enables automatic path failover.
--cpu_affinity PN (-ca)
Set cpu affinity to PN. CPUs are numbered sequentially from 0. If
PN is "any", any cpu is allowed otherwise the cpu is limited to the
@ -184,7 +196,7 @@ Options
Set remote processor affinity to PN.
--flip OnOff (-f)
If non-zero, cause sender and receiver to play opposite roles.
-fo
-f1
Cause sender and receiver to play opposite roles.
--help Topic (-h)
Print out information about Topic. To see the list of topics, type
@ -222,11 +234,11 @@ Options
Locally turn polling mode on or off.
--rem_cq_poll OnOff (-rcp)
Remotely turn polling mode on or off.
-cpo
-cp1
Turn polling mode on.
-lcpo
-lcp1
Turn local polling mode on.
-rcpo
-rcp1
Turn remote polling mode on.
--ip_port Port (-ip)
Use Port to run the socket tests. This is different from
@ -294,7 +306,7 @@ Options
necessary to use the CM for iWARP devices. The default is to
establish the connection without using the CM. This only works for
the tests that use the RC transport.
-cmo
-cm1
Use RDMA Connection Manager.
--verbose (-v)
Provide more detailed output. Turns on -vc, -vs, -vt and -vu.
@ -894,3 +906,52 @@ ver_rc_fetch_add +RDMA
Tests the RC Fetch and Add Atomic operation. The server's memory
location starts with zero and the client successively adds one. The
results are checked for correctness.
xrc_bw +RDMA
Purpose
XRC streaming one way bandwidth
Common Options
--access_recv OnOff (-ar) Access received data
--id Device:Port (-i) Set RDMA device and port
--msg_size Size (-m) Set message size
--cq_poll OnOff Set polling mode on/off
--time (-t) Set test duration
Other Options
--cpu_affinity, --listen_port, --mtu_size, --static_rate, --timeout
Display Options
--precision, --unify_nodes, --unify_units, --use_bits_per_sec,
--verbose
Description
The client sends messages to the server who notes how many it received.
The XRC Send/Receive mechanism is used.
xrc_bi_bw +RDMA
Purpose
XRC streaming two way bandwidth
Common Options
--access_recv OnOff (-ar) Access received data
--id Device:Port (-i) Set RDMA device and port
--msg_size Size (-m) Set message size
--cq_poll OnOff Set polling mode on/off
--time (-t) Set test duration
Other Options
--cpu_affinity, --listen_port, --mtu_size, --static_rate, --timeout
Display Options
--precision, --unify_nodes, --unify_units, --use_bits_per_sec,
--verbose
Description
Both the client and server exchange messages with each other using the
XRC Send/Receive mechanism and note how many were received.
xrc_lat +RDMA
Purpose
XRC one way latency
Common Options
--id Device:Port (-i) Set RDMA device and port
--msg_size Size (-m) Set message size
--cq_poll OnOff Set polling mode on/off
--time (-t) Set test duration
Other Options
--cpu_affinity, --listen_port, --mtu_size, --static_rate, --timeout
Display Options
--precision, --unify_nodes, --unify_units, --verbose
Description
A ping pong latency test where the server and client exchange messages
repeatedly using XRC Send/Receive.

50
src/qperf.c
View File

@ -62,7 +62,7 @@
*/
#define VER_MAJ 0 /* Major version */
#define VER_MIN 4 /* Minor version */
#define VER_INC 1 /* Incremental version */
#define VER_INC 2 /* Incremental version */
#define LISTENQ 5 /* Size of listen queue */
#define BUFSIZE 1024 /* Size of buffers */
@ -273,6 +273,7 @@ volatile int Finished;
PAR_NAME ParName[] ={
{ "access_recv", L_ACCESS_RECV, R_ACCESS_RECV },
{ "affinity", L_AFFINITY, R_AFFINITY },
{ "alt_port", L_ALT_PORT, R_ALT_PORT },
{ "flip", L_FLIP, R_FLIP },
{ "id", L_ID, R_ID },
{ "msg_size", L_MSG_SIZE, R_MSG_SIZE },
@ -299,6 +300,8 @@ PAR_INFO ParInfo[P_N] ={
{ R_ACCESS_RECV, 'l', &RReq.access_recv },
{ L_AFFINITY, 'l', &Req.affinity },
{ R_AFFINITY, 'l', &RReq.affinity },
{ L_ALT_PORT, 'l', &Req.alt_port },
{ R_ALT_PORT, 'l', &RReq.alt_port },
{ L_FLIP, 'l', &Req.flip },
{ R_FLIP, 'l', &RReq.flip },
{ L_ID, 'p', &Req.id },
@ -378,6 +381,13 @@ DICT Renamed[] = {
{ "-vS", "-vvs", },
{ "-vT", "-vvt", },
{ "-vU", "-vvu", },
/* options that are on */
{ "-aro", "-ar1" },
{ "-cmo", "-cm1" },
{ "-fo", "-f1" },
{ "-cpo", "-cp1" },
{ "-lcpo", "-lcp1" },
{ "-rcpo", "-rcp1" },
/* miscellaneous */
{ "-Ar", "-ar" },
{ "-M", "-mt" },
@ -393,7 +403,13 @@ DICT Renamed[] = {
OPTION Options[] ={
{ "--access_recv", "int", L_ACCESS_RECV, R_ACCESS_RECV },
{ "-ar", "int", L_ACCESS_RECV, R_ACCESS_RECV },
{ "-aro", "set1", L_ACCESS_RECV, R_ACCESS_RECV },
{ "-ar1", "set1", L_ACCESS_RECV, R_ACCESS_RECV },
{ "--alt_port", "int", L_ALT_PORT, R_ALT_PORT },
{ "-ap", "int", L_ALT_PORT, R_ALT_PORT },
{ "--loc_alt_port", "int", L_ALT_PORT, },
{ "-lap", "int", L_ALT_PORT, },
{ "--rem_alt_port", "int", R_ALT_PORT },
{ "-rap", "int", R_ALT_PORT },
{ "--cpu_affinity", "int", L_AFFINITY, R_AFFINITY },
{ "-ca", "int", L_AFFINITY, R_AFFINITY },
{ "--loc_cpu_affinity", "int", L_AFFINITY, },
@ -404,7 +420,7 @@ OPTION Options[] ={
{ "-D", "Sdebug", },
{ "--flip", "int", L_FLIP, R_FLIP },
{ "-f", "int", L_FLIP, R_FLIP },
{ "-fo", "set1", L_FLIP, R_FLIP },
{ "-f1", "set1", L_FLIP, R_FLIP },
{ "--help", "help" },
{ "-h", "help" },
{ "--host", "host", },
@ -425,13 +441,13 @@ OPTION Options[] ={
{ "-n", "int", L_NO_MSGS, R_NO_MSGS },
{ "--cq_poll", "int", L_POLL_MODE, R_POLL_MODE },
{ "-cp", "int", L_POLL_MODE, R_POLL_MODE },
{ "-cpo", "set1", L_POLL_MODE, R_POLL_MODE },
{ "-cp1", "set1", L_POLL_MODE, R_POLL_MODE },
{ "--loc_cq_poll", "int", L_POLL_MODE, },
{ "-lcp", "int", L_POLL_MODE, },
{ "-lcpo", "set1", L_POLL_MODE },
{ "-lcp1", "set1", L_POLL_MODE },
{ "--rem_cq_poll", "int", R_POLL_MODE },
{ "-rcp", "int", R_POLL_MODE },
{ "-rcpo", "set1", R_POLL_MODE },
{ "-rcp1", "set1", R_POLL_MODE },
{ "--ip_port", "int", L_PORT, R_PORT },
{ "-ip", "int", L_PORT, R_PORT },
{ "--precision", "precision", },
@ -476,7 +492,7 @@ OPTION Options[] ={
{ "-ub", "ub", },
{ "--use_cm", "int", L_USE_CM, R_USE_CM },
{ "-cm", "int", L_USE_CM, R_USE_CM },
{ "-cmo", "set1", L_USE_CM, R_USE_CM },
{ "-cm1", "set1", L_USE_CM, R_USE_CM },
{ "--verbose", "v", },
{ "-v", "v", },
{ "--verbose_conf", "vc", },
@ -543,6 +559,9 @@ TEST Tests[] ={
test(ud_lat),
test(ver_rc_compare_swap),
test(ver_rc_fetch_add),
test(xrc_bi_bw),
test(xrc_bw),
test(xrc_lat),
#endif
};
@ -786,6 +805,7 @@ do_option(OPTION *option, char ***argvp)
/* Help */
char **usage;
char *category = (*argvp)[1];
if (!category)
category = "main";
for (usage = Usage; *usage; usage += 2)
@ -1137,7 +1157,7 @@ server(void)
TEST *test;
int s = offset(REQ, req_index);
debug("waiting for request");
debug("ready for requests");
if (!server_recv_request())
continue;
pid = fork();
@ -1164,7 +1184,7 @@ server(void)
test = &Tests[Req.req_index];
TestName = test->name;
debug("request is %s", TestName);
debug("received request: %s", TestName);
init_lstat();
Finished = 0;
set_affinity();
@ -1283,7 +1303,7 @@ client(TEST *test)
RReq.ver_inc = VER_INC;
RReq.req_index = test - Tests;
TestName = test->name;
debug("sending request %s", TestName);
debug("sending request: %s", TestName);
init_lstat();
printf("%s:\n", TestName);
Finished = 0;
@ -1378,12 +1398,12 @@ exchange_results(void)
recv_mesg(&stat, sizeof(stat), "results");
dec_init(&stat);
dec_stat(&RStat);
send_sync("results");
send_sync("synchronization after test");
} else {
enc_init(&stat);
enc_stat(&LStat);
send_mesg(&stat, sizeof(stat), "results");
recv_sync("results");
recv_sync("synchronization after test");
}
}
@ -1583,7 +1603,7 @@ run_server_quit(void)
void
sync_test(void)
{
synchronize("test");
synchronize("synchronization before test");
start_test_timer(Req.time);
}
@ -1601,7 +1621,7 @@ start_test_timer(int seconds)
if (!seconds)
return;
debug("starting timer");
debug("starting timer for %d seconds", seconds);
itimerval.it_value.tv_sec = seconds;
itimerval.it_interval.tv_usec = 1;
setitimer(ITIMER_REAL, &itimerval, 0);
@ -2422,6 +2442,7 @@ enc_req(REQ *host)
enc_int(host->req_index, sizeof(host->req_index));
enc_int(host->access_recv, sizeof(host->access_recv));
enc_int(host->affinity, sizeof(host->affinity));
enc_int(host->alt_port, sizeof(host->alt_port));
enc_int(host->flip, sizeof(host->flip));
enc_int(host->msg_size, sizeof(host->msg_size));
enc_int(host->mtu_size, sizeof(host->mtu_size));
@ -2462,6 +2483,7 @@ dec_req_data(REQ *host)
host->req_index = dec_int(sizeof(host->req_index));
host->access_recv = dec_int(sizeof(host->access_recv));
host->affinity = dec_int(sizeof(host->affinity));
host->alt_port = dec_int(sizeof(host->alt_port));
host->flip = dec_int(sizeof(host->flip));
host->msg_size = dec_int(sizeof(host->msg_size));
host->mtu_size = dec_int(sizeof(host->mtu_size));

11
src/qperf.h
View File

@ -49,7 +49,7 @@
#define cardof(a) (sizeof(a)/sizeof(*a))
#define endof(a) (&a[cardof(a)])
#define streq(a, b) (strcmp(a, b) == 0)
#define offset(t, e) ((int)&((t *)0)->e)
#define offset(t, e) ((long)&((t *)0)->e)
#define is_client() (ServerName != 0)
#define is_sender() (Req.flip ? !is_client() : is_client())
@ -97,6 +97,8 @@ typedef enum {
R_ACCESS_RECV,
L_AFFINITY,
R_AFFINITY,
L_ALT_PORT,
R_ALT_PORT,
L_FLIP,
R_FLIP,
L_ID,
@ -153,6 +155,7 @@ typedef struct REQ {
uint16_t req_index; /* Request index (into Tests) */
uint32_t access_recv; /* Access data after receiving */
uint32_t affinity; /* Processor affinity */
uint32_t alt_port; /* Alternate path port number */
uint32_t flip; /* Flip sender/receiver */
uint32_t msg_size; /* Message Size */
uint32_t mtu_size; /* MTU Size */
@ -343,6 +346,12 @@ void run_client_ver_rc_compare_swap(void);
void run_server_ver_rc_compare_swap(void);
void run_client_ver_rc_fetch_add(void);
void run_server_ver_rc_fetch_add(void);
void run_client_xrc_bi_bw(void);
void run_server_xrc_bi_bw(void);
void run_client_xrc_bw(void);
void run_server_xrc_bw(void);
void run_client_xrc_lat(void);
void run_server_xrc_lat(void);
/*

703
src/rdma.c
View File

File diff suppressed because it is too large Load Diff

4
src/rds.c
View File

@ -94,6 +94,7 @@ run_client_rds_bw(void)
sync_test();
while (!Finished) {
int n = sendto(sockfd, buf, Req.msg_size, 0, (SA *)&RAddr, RLen);
if (Finished)
break;
if (n != Req.msg_size) {
@ -159,6 +160,7 @@ run_client_rds_lat(void)
sync_test();
while (!Finished) {
int n = sendto(sockfd, buf, Req.msg_size, 0, (SA *)&RAddr, RLen);
if (Finished)
break;
if (n != Req.msg_size) {
@ -202,6 +204,7 @@ run_server_rds_lat(void)
SS raddr;
socklen_t rlen = sizeof(raddr);
int n = recvfrom(sockfd, buf, Req.msg_size, 0, (SA *)&raddr, &rlen);
if (Finished)
break;
if (n != Req.msg_size) {
@ -454,6 +457,7 @@ qgetnameinfo(SA *sa, socklen_t salen, char *host, size_t hostlen,
char *serv, size_t servlen, int flags)
{
int stat = getnameinfo(sa, salen, host, hostlen, serv, servlen, flags);
if (stat < 0)
error(0, "getnameinfo failed: %s", gai_strerror(stat));
}

18
src/socket.c
View File

@ -273,6 +273,7 @@ stream_client_bw(KIND kind)
sync_test();
while (!Finished) {
int n = send_full(sockFD, buf, Req.msg_size);
if (Finished)
break;
if (n < 0) {
@ -304,6 +305,7 @@ stream_server_bw(KIND kind)
buf = qmalloc(Req.msg_size);
while (!Finished) {
int n = recv_full(sockFD, buf, Req.msg_size);
if (Finished)
break;
if (n < 0) {
@ -337,6 +339,7 @@ stream_client_lat(KIND kind)
sync_test();
while (!Finished) {
int n = send_full(sockFD, buf, Req.msg_size);
if (Finished)
break;
if (n < 0) {
@ -378,6 +381,7 @@ stream_server_lat(KIND kind)
buf = qmalloc(Req.msg_size);
while (!Finished) {
int n = recv_full(sockFD, buf, Req.msg_size);
if (Finished)
break;
if (n < 0) {
@ -418,6 +422,7 @@ datagram_client_bw(KIND kind)
sync_test();
while (!Finished) {
int n = write(sockFD, buf, Req.msg_size);
if (Finished)
break;
if (n < 0) {
@ -449,6 +454,7 @@ datagram_server_bw(KIND kind)
buf = qmalloc(Req.msg_size);
while (!Finished) {
int n = recv(sockFD, buf, Req.msg_size, 0);
if (Finished)
break;
if (n < 0) {
@ -481,6 +487,7 @@ datagram_client_lat(KIND kind)
sync_test();
while (!Finished) {
int n = write(sockFD, buf, Req.msg_size);
if (Finished)
break;
if (n < 0) {
@ -521,10 +528,11 @@ datagram_server_lat(KIND kind)
sync_test();
buf = qmalloc(Req.msg_size);
while (!Finished) {
struct sockaddr_storage clientAddr;
SS clientAddr;
socklen_t clientLen = sizeof(clientAddr);
int n = recvfrom(sockfd, buf, Req.msg_size, 0,
(SA *)&clientAddr, &clientLen);
if (Finished)
break;
if (n < 0) {
@ -584,6 +592,7 @@ client_init(int *fd, KIND kind)
if (!ai->ai_family)
continue;
*fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
setsockopt_one(*fd, SO_REUSEADDR);
if (connect(*fd, ai->ai_addr, ai->ai_addrlen) == SUCCESS0)
break;
close(*fd);
@ -637,6 +646,7 @@ stream_server_init(int *fd, KIND kind)
debug("accepted %s connection", kind_name(kind));
set_socket_buffer_size(*fd);
close(listenFD);
debug("receiving to %s port %d", kind_name(kind), port);
}
@ -737,7 +747,7 @@ static void
get_socket_port(int fd, uint32_t *port)
{
char p[NI_MAXSERV];
struct sockaddr_storage sa;
SS sa;
socklen_t salen = sizeof(sa);
if (getsockname(fd, (SA *)&sa, &salen) < 0)
@ -758,8 +768,10 @@ static int
send_full(int fd, void *ptr, int len)
{
int n = len;
while (!Finished && n) {
int i = write(fd, ptr, n);
if (i < 0)
return i;
ptr += i;
@ -779,8 +791,10 @@ static int
recv_full(int fd, void *ptr, int len)
{
int n = len;
while (!Finished && n) {
int i = read(fd, ptr, n);
if (i < 0)
return i;
ptr += i;

10
src/support.c
View File

@ -143,6 +143,7 @@ dec_int(int n)
{
uint64_t l = 0;
uint8_t *p = (DecodePtr += n);
while (n--)
l = (l << 8) | (*--p & 0xFF);
return l;
@ -178,6 +179,7 @@ void *
qmalloc(long n)
{
void *p = malloc(n);
if (!p)
error(0, "malloc failed");
return p;
@ -235,7 +237,7 @@ synchronize(char *msg)
{
send_sync(msg);
recv_sync(msg);
debug("synchronize %s completed", msg);
debug("synchronization complete");
}
@ -246,6 +248,7 @@ void
send_sync(char *msg)
{
int n = strlen(msg);
send_mesg(msg, n, msg);
}
@ -257,8 +260,8 @@ void
recv_sync(char *msg)
{
char data[64];
int n = strlen(msg);
if (n > sizeof(data))
error(BUG, "buffer in recv_sync() too small");
recv_mesg(data, n, msg);
@ -380,9 +383,9 @@ struct addrinfo *
getaddrinfo_port(char *node, int port, struct addrinfo *hints)
{
struct addrinfo *res;
char *service = qasprintf("%d", port);
int stat = getaddrinfo(node, service, hints, &res);
free(service);
if (stat != 0)
error(0, "getaddrinfo failed: %s", gai_strerror(stat));
@ -400,6 +403,7 @@ void
setsockopt_one(int fd, int optname)
{
int one = 1;
if (setsockopt(fd, SOL_SOCKET, optname, &one, sizeof(one)) >= 0)
return;
error(SYS, "setsockopt %d %d to 1 failed", SOL_SOCKET, optname);