vitalif
/
hsbench
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

hsbench: refactor stats and add json output support 

Signed-off-by: Mark Nelson <mnelson@redhat.com>
master
Mark Nelson 2019-08-19 11:25:25 -04:00
parent c4d9ef3872
commit 37eedcf948
2 changed files with 158 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
README.md
View File

@ -63,6 +63,8 @@ OPTIONS:
Prefix for buckets (default "hotsauce_bench") Prefix for buckets (default "hotsauce_bench")
-d int -d int
Maximum test duration in seconds <-1 for unlimited> (default 60) Maximum test duration in seconds <-1 for unlimited> (default 60)
-j string
Write JSON output to this file
-l int -l int
Number of times to repeat test (default 1) Number of times to repeat test (default 1)
-m string -m string
@ -108,64 +110,65 @@ Below is an example run of the benchmark using a 10s test duration, 10 threads,
running on an Intel P3700 NVMe device. running on an Intel P3700 NVMe device.
``` ```
$ ./hsbench -a 3JZ0SVK94Z55OZU5J1N0 -s OdzEPyDDZ0ls1haDUu1NVWkJDcnG74Lb7XylfXRM -u http://127.0.0.1:7480 -z 4K -d 10 -t 10 -b 10 -o test.csv $ ./hsbench -a 3JZ0SVK94Z55OZU5J1N0 -s OdzEPyDDZ0ls1haDUu1NVWkJDcnG74Lb7XylfXRM -u http://127.0.0.1:7480 -z 4K -d 10 -t 10 -b 10
2019/08/19 08:18:51 Hotsauce S3 Benchmark Version 0.1 2019/08/19 11:21:06 Hotsauce S3 Benchmark Version 0.1
2019/08/19 08:18:51 Parameters: 2019/08/19 11:21:06 Parameters:
2019/08/19 08:18:51 url=http://127.0.0.1:7480 2019/08/19 11:21:06 url=http://127.0.0.1:7480
2019/08/19 08:18:51 object_prefix= 2019/08/19 11:21:06 object_prefix=
2019/08/19 08:18:51 bucket_prefix=hotsauce_bench 2019/08/19 11:21:06 bucket_prefix=hotsauce_bench
2019/08/19 08:18:51 region=us-east-1 2019/08/19 11:21:06 region=us-east-1
2019/08/19 08:18:51 modes=cxipgdx 2019/08/19 11:21:06 modes=cxipgdx
2019/08/19 08:18:51 object_count=-1 2019/08/19 11:21:06 output=
2019/08/19 08:18:51 bucket_count=10 2019/08/19 11:21:06 json_output=
2019/08/19 08:18:51 duration=10 2019/08/19 11:21:06 object_count=-1
2019/08/19 08:18:51 threads=10 2019/08/19 11:21:06 bucket_count=10
2019/08/19 08:18:51 loops=1 2019/08/19 11:21:06 duration=10
2019/08/19 08:18:51 size=4K 2019/08/19 11:21:06 threads=10
2019/08/19 08:18:51 interval=1.000000 2019/08/19 11:21:06 loops=1
2019/08/19 08:18:51 Running Loop 0 BUCKET CLEAR TEST 2019/08/19 11:21:06 size=4K
2019/08/19 08:18:51 Loop: 0, Int: TOTAL, Dur(s): 0.0, Mode: BCLR, Ops: 0, MB/s: 0.00, IO/s: 0, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.0 ], Slowdowns: 0 2019/08/19 11:21:06 interval=1.000000
2019/08/19 08:18:51 Running Loop 0 BUCKET DELETE TEST 2019/08/19 11:21:06 Running Loop 0 BUCKET CLEAR TEST
2019/08/19 08:18:51 Loop: 0, Int: TOTAL, Dur(s): 0.0, Mode: BDEL, Ops: 0, MB/s: 0.00, IO/s: 0, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.0 ], Slowdowns: 0 2019/08/19 11:21:06 Loop: 0, Int: TOTAL, Dur(s): 0.0, Mode: BCLR, Ops: 0, MB/s: 0.00, IO/s: 0, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.0 ], Slowdowns: 0
2019/08/19 08:18:51 Running Loop 0 BUCKET INIT TEST 2019/08/19 11:21:06 Running Loop 0 BUCKET DELETE TEST
2019/08/19 08:18:51 Loop: 0, Int: TOTAL, Dur(s): 0.0, Mode: BINIT, Ops: 10, MB/s: 0.00, IO/s: 968, Lat(ms): [ min: 7.5, avg: 8.7, 99%: 10.2, max: 10.2 ], Slowdowns: 0 2019/08/19 11:21:06 Loop: 0, Int: TOTAL, Dur(s): 0.0, Mode: BDEL, Ops: 0, MB/s: 0.00, IO/s: 0, Lat(ms): [ min: 0.0, avg: 0.0, 99%: 0.0, max: 0.0 ], Slowdowns: 0
2019/08/19 08:18:51 Running Loop 0 OBJECT PUT TEST 2019/08/19 11:21:06 Running Loop 0 BUCKET INIT TEST
2019/08/19 08:18:52 Loop: 0, Int: 0, Dur(s): 1.0, Mode: PUT, Ops: 5255, MB/s: 20.53, IO/s: 5255, Lat(ms): [ min: 1.3, avg: 1.9, 99%: 3.4, max: 14.1 ], Slowdowns: 0 2019/08/19 11:21:06 Loop: 0, Int: TOTAL, Dur(s): 0.0, Mode: BINIT, Ops: 10, MB/s: 0.00, IO/s: 944, Lat(ms): [ min: 7.9, avg: 9.3, 99%: 10.4, max: 10.4 ], Slowdowns: 0
2019/08/19 08:18:53 Loop: 0, Int: 1, Dur(s): 1.0, Mode: PUT, Ops: 5237, MB/s: 20.46, IO/s: 5237, Lat(ms): [ min: 1.2, avg: 1.9, 99%: 3.4, max: 11.1 ], Slowdowns: 0 2019/08/19 11:21:06 Running Loop 0 OBJECT PUT TEST
2019/08/19 08:18:54 Loop: 0, Int: 2, Dur(s): 1.0, Mode: PUT, Ops: 5454, MB/s: 21.30, IO/s: 5454, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 2.9, max: 7.3 ], Slowdowns: 0 2019/08/19 11:21:07 Loop: 0, Int: 0, Dur(s): 1.0, Mode: PUT, Ops: 5209, MB/s: 20.35, IO/s: 5209, Lat(ms): [ min: 1.3, avg: 1.9, 99%: 3.5, max: 8.4 ], Slowdowns: 0
2019/08/19 08:18:55 Loop: 0, Int: 3, Dur(s): 1.0, Mode: PUT, Ops: 5318, MB/s: 20.77, IO/s: 5318, Lat(ms): [ min: 1.3, avg: 1.9, 99%: 3.7, max: 7.1 ], Slowdowns: 0 2019/08/19 11:21:08 Loop: 0, Int: 1, Dur(s): 1.0, Mode: PUT, Ops: 5076, MB/s: 19.83, IO/s: 5076, Lat(ms): [ min: 1.2, avg: 2.0, 99%: 3.1, max: 42.3 ], Slowdowns: 0
2019/08/19 08:18:56 Loop: 0, Int: 4, Dur(s): 1.0, Mode: PUT, Ops: 5364, MB/s: 20.95, IO/s: 5364, Lat(ms): [ min: 1.3, avg: 1.9, 99%: 2.9, max: 7.8 ], Slowdowns: 0 2019/08/19 11:21:09 Loop: 0, Int: 2, Dur(s): 1.0, Mode: PUT, Ops: 4319, MB/s: 16.87, IO/s: 4319, Lat(ms): [ min: 1.3, avg: 2.3, 99%: 4.4, max: 58.0 ], Slowdowns: 0
2019/08/19 08:18:57 Loop: 0, Int: 5, Dur(s): 1.0, Mode: PUT, Ops: 5219, MB/s: 20.39, IO/s: 5219, Lat(ms): [ min: 1.3, avg: 1.9, 99%: 3.2, max: 16.8 ], Slowdowns: 0 2019/08/19 11:21:10 Loop: 0, Int: 3, Dur(s): 1.0, Mode: PUT, Ops: 4288, MB/s: 16.75, IO/s: 4288, Lat(ms): [ min: 1.3, avg: 2.3, 99%: 3.5, max: 63.1 ], Slowdowns: 0
2019/08/19 08:18:58 Loop: 0, Int: 6, Dur(s): 1.0, Mode: PUT, Ops: 5200, MB/s: 20.31, IO/s: 5200, Lat(ms): [ min: 1.2, avg: 1.9, 99%: 3.1, max: 9.6 ], Slowdowns: 0 2019/08/19 11:21:11 Loop: 0, Int: 4, Dur(s): 1.0, Mode: PUT, Ops: 4549, MB/s: 17.77, IO/s: 4549, Lat(ms): [ min: 1.3, avg: 2.2, 99%: 6.8, max: 57.9 ], Slowdowns: 0
2019/08/19 08:18:59 Loop: 0, Int: 7, Dur(s): 1.0, Mode: PUT, Ops: 5250, MB/s: 20.51, IO/s: 5250, Lat(ms): [ min: 1.3, avg: 1.9, 99%: 3.0, max: 7.3 ], Slowdowns: 0 2019/08/19 11:21:12 Loop: 0, Int: 5, Dur(s): 1.0, Mode: PUT, Ops: 4447, MB/s: 17.37, IO/s: 4447, Lat(ms): [ min: 1.3, avg: 2.2, 99%: 3.5, max: 58.8 ], Slowdowns: 0
2019/08/19 08:19:00 Loop: 0, Int: 8, Dur(s): 1.0, Mode: PUT, Ops: 4701, MB/s: 18.36, IO/s: 4701, Lat(ms): [ min: 1.3, avg: 2.1, 99%: 3.5, max: 86.6 ], Slowdowns: 0 2019/08/19 11:21:13 Loop: 0, Int: 6, Dur(s): 1.0, Mode: PUT, Ops: 4260, MB/s: 16.64, IO/s: 4260, Lat(ms): [ min: 1.3, avg: 2.4, 99%: 5.8, max: 58.4 ], Slowdowns: 0
2019/08/19 08:19:01 Loop: 0, Int: 9, Dur(s): 1.0, Mode: PUT, Ops: 5269, MB/s: 20.58, IO/s: 5269, Lat(ms): [ min: 1.3, avg: 1.9, 99%: 3.2, max: 11.4 ], Slowdowns: 0 2019/08/19 11:21:14 Loop: 0, Int: 7, Dur(s): 1.0, Mode: PUT, Ops: 5202, MB/s: 20.32, IO/s: 5202, Lat(ms): [ min: 1.3, avg: 1.9, 99%: 3.1, max: 10.3 ], Slowdowns: 0
2019/08/19 08:19:01 Loop: 0, Int: TOTAL, Dur(s): 10.0, Mode: PUT, Ops: 52277, MB/s: 20.42, IO/s: 5227, Lat(ms): [ min: 1.2, avg: 1.9, 99%: 3.2, max: 86.6 ], Slowdowns: 0 2019/08/19 11:21:15 Loop: 0, Int: 8, Dur(s): 1.0, Mode: PUT, Ops: 5213, MB/s: 20.36, IO/s: 5213, Lat(ms): [ min: 1.3, avg: 1.9, 99%: 3.0, max: 13.3 ], Slowdowns: 0
2019/08/19 08:19:01 Running Loop 0 OBJECT GET TEST 2019/08/19 11:21:16 Loop: 0, Int: 9, Dur(s): 1.0, Mode: PUT, Ops: 5210, MB/s: 20.35, IO/s: 5210, Lat(ms): [ min: 1.2, avg: 1.9, 99%: 3.0, max: 9.6 ], Slowdowns: 0
2019/08/19 08:19:03 Loop: 0, Int: 0, Dur(s): 1.0, Mode: GET, Ops: 1275, MB/s: 4.98, IO/s: 1275, Lat(ms): [ min: 0.6, avg: 0.9, 99%: 1.4, max: 1.5 ], Slowdowns: 0 2019/08/19 11:21:16 Loop: 0, Int: TOTAL, Dur(s): 10.0, Mode: PUT, Ops: 47783, MB/s: 18.66, IO/s: 4777, Lat(ms): [ min: 1.2, avg: 2.1, 99%: 3.4, max: 63.1 ], Slowdowns: 0
2019/08/19 08:19:04 Loop: 0, Int: 1, Dur(s): 1.0, Mode: GET, Ops: 1286, MB/s: 5.02, IO/s: 1286, Lat(ms): [ min: 0.6, avg: 0.9, 99%: 1.3, max: 1.8 ], Slowdowns: 0 2019/08/19 11:21:16 Running Loop 0 OBJECT GET TEST
2019/08/19 08:19:05 Loop: 0, Int: 2, Dur(s): 1.0, Mode: GET, Ops: 1112, MB/s: 4.34, IO/s: 1112, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.1, max: 1.6 ], Slowdowns: 0 2019/08/19 11:21:17 Loop: 0, Int: 0, Dur(s): 1.0, Mode: GET, Ops: 1211, MB/s: 4.73, IO/s: 1211, Lat(ms): [ min: 0.6, avg: 0.9, 99%: 1.5, max: 2.0 ], Slowdowns: 0
2019/08/19 08:19:06 Loop: 0, Int: 3, Dur(s): 1.0, Mode: GET, Ops: 1128, MB/s: 4.41, IO/s: 1128, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.2, max: 5.7 ], Slowdowns: 0 2019/08/19 11:21:18 Loop: 0, Int: 1, Dur(s): 1.0, Mode: GET, Ops: 1182, MB/s: 4.62, IO/s: 1182, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.3, max: 2.1 ], Slowdowns: 0
2019/08/19 08:19:07 Loop: 0, Int: 4, Dur(s): 1.0, Mode: GET, Ops: 1112, MB/s: 4.34, IO/s: 1112, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.2, max: 1.7 ], Slowdowns: 0 2019/08/19 11:21:19 Loop: 0, Int: 2, Dur(s): 1.0, Mode: GET, Ops: 1110, MB/s: 4.34, IO/s: 1110, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.2, max: 1.5 ], Slowdowns: 0
2019/08/19 08:19:08 Loop: 0, Int: 5, Dur(s): 1.0, Mode: GET, Ops: 1174, MB/s: 4.59, IO/s: 1174, Lat(ms): [ min: 0.6, avg: 0.9, 99%: 1.3, max: 1.6 ], Slowdowns: 0 2019/08/19 11:21:20 Loop: 0, Int: 3, Dur(s): 1.0, Mode: GET, Ops: 1072, MB/s: 4.19, IO/s: 1072, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.1, max: 1.3 ], Slowdowns: 0
2019/08/19 08:19:09 Loop: 0, Int: 6, Dur(s): 1.0, Mode: GET, Ops: 1106, MB/s: 4.32, IO/s: 1106, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.1, max: 1.7 ], Slowdowns: 0 2019/08/19 11:21:21 Loop: 0, Int: 4, Dur(s): 1.0, Mode: GET, Ops: 1098, MB/s: 4.29, IO/s: 1098, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.2, max: 3.9 ], Slowdowns: 0
2019/08/19 08:19:10 Loop: 0, Int: 7, Dur(s): 1.0, Mode: GET, Ops: 1132, MB/s: 4.42, IO/s: 1132, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.2, max: 11.6 ], Slowdowns: 0 2019/08/19 11:21:22 Loop: 0, Int: 5, Dur(s): 1.0, Mode: GET, Ops: 1115, MB/s: 4.36, IO/s: 1115, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.1, max: 1.4 ], Slowdowns: 0
2019/08/19 08:19:11 Loop: 0, Int: 8, Dur(s): 1.0, Mode: GET, Ops: 1074, MB/s: 4.20, IO/s: 1074, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.2, max: 1.4 ], Slowdowns: 0 2019/08/19 11:21:23 Loop: 0, Int: 6, Dur(s): 1.0, Mode: GET, Ops: 1110, MB/s: 4.34, IO/s: 1110, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.1, max: 1.5 ], Slowdowns: 0
2019/08/19 08:19:12 Loop: 0, Int: 9, Dur(s): 1.0, Mode: GET, Ops: 1080, MB/s: 4.22, IO/s: 1080, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.2, max: 1.6 ], Slowdowns: 0 2019/08/19 11:21:24 Loop: 0, Int: 7, Dur(s): 1.0, Mode: GET, Ops: 1079, MB/s: 4.21, IO/s: 1079, Lat(ms): [ min: 0.6, avg: 0.7, 99%: 1.1, max: 1.7 ], Slowdowns: 0
2019/08/19 08:19:12 Loop: 0, Int: TOTAL, Dur(s): 10.0, Mode: GET, Ops: 11481, MB/s: 4.47, IO/s: 1144, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.2, max: 11.6 ], Slowdowns: 0 2019/08/19 11:21:25 Loop: 0, Int: 8, Dur(s): 1.0, Mode: GET, Ops: 1089, MB/s: 4.25, IO/s: 1089, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.2, max: 1.6 ], Slowdowns: 0
2019/08/19 08:19:12 Running Loop 0 OBJECT DELETE TEST 2019/08/19 11:21:26 Loop: 0, Int: 9, Dur(s): 1.0, Mode: GET, Ops: 1156, MB/s: 4.52, IO/s: 1156, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.2, max: 5.4 ], Slowdowns: 0
2019/08/19 08:19:13 Loop: 0, Int: 0, Dur(s): 1.0, Mode: DEL, Ops: 5591, MB/s: 21.84, IO/s: 5591, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 3.3, max: 11.0 ], Slowdowns: 0 2019/08/19 11:21:26 Loop: 0, Int: TOTAL, Dur(s): 10.0, Mode: GET, Ops: 11222, MB/s: 4.37, IO/s: 1118, Lat(ms): [ min: 0.6, avg: 0.8, 99%: 1.2, max: 5.4 ], Slowdowns: 0
2019/08/19 08:19:14 Loop: 0, Int: 1, Dur(s): 1.0, Mode: DEL, Ops: 5607, MB/s: 21.90, IO/s: 5607, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 3.1, max: 13.9 ], Slowdowns: 0 2019/08/19 11:21:26 Running Loop 0 OBJECT DELETE TEST
2019/08/19 08:19:15 Loop: 0, Int: 2, Dur(s): 1.0, Mode: DEL, Ops: 5680, MB/s: 22.19, IO/s: 5680, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 2.7, max: 7.4 ], Slowdowns: 0 2019/08/19 11:21:27 Loop: 0, Int: 0, Dur(s): 1.0, Mode: DEL, Ops: 5673, MB/s: 22.16, IO/s: 5673, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 2.7, max: 10.0 ], Slowdowns: 0
2019/08/19 08:19:16 Loop: 0, Int: 3, Dur(s): 1.0, Mode: DEL, Ops: 5691, MB/s: 22.23, IO/s: 5691, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 2.9, max: 6.5 ], Slowdowns: 0 2019/08/19 11:21:28 Loop: 0, Int: 1, Dur(s): 1.0, Mode: DEL, Ops: 5597, MB/s: 21.86, IO/s: 5597, Lat(ms): [ min: 1.1, avg: 1.8, 99%: 2.9, max: 10.5 ], Slowdowns: 0
2019/08/19 08:19:17 Loop: 0, Int: 4, Dur(s): 1.0, Mode: DEL, Ops: 5565, MB/s: 21.74, IO/s: 5565, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 3.6, max: 11.0 ], Slowdowns: 0 2019/08/19 11:21:29 Loop: 0, Int: 2, Dur(s): 1.0, Mode: DEL, Ops: 5123, MB/s: 20.01, IO/s: 5123, Lat(ms): [ min: 1.1, avg: 1.9, 99%: 3.2, max: 67.1 ], Slowdowns: 0
2019/08/19 08:19:18 Loop: 0, Int: 5, Dur(s): 1.0, Mode: DEL, Ops: 5739, MB/s: 22.42, IO/s: 5739, Lat(ms): [ min: 1.2, avg: 1.7, 99%: 2.7, max: 3.5 ], Slowdowns: 0 2019/08/19 11:21:30 Loop: 0, Int: 3, Dur(s): 1.0, Mode: DEL, Ops: 5547, MB/s: 21.67, IO/s: 5547, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 2.7, max: 15.2 ], Slowdowns: 0
2019/08/19 08:19:19 Loop: 0, Int: 6, Dur(s): 1.0, Mode: DEL, Ops: 5613, MB/s: 21.93, IO/s: 5613, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 2.9, max: 7.1 ], Slowdowns: 0 2019/08/19 11:21:31 Loop: 0, Int: 4, Dur(s): 1.0, Mode: DEL, Ops: 5604, MB/s: 21.89, IO/s: 5604, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 2.7, max: 11.6 ], Slowdowns: 0
2019/08/19 08:19:20 Loop: 0, Int: 7, Dur(s): 1.0, Mode: DEL, Ops: 5591, MB/s: 21.84, IO/s: 5591, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 2.9, max: 16.3 ], Slowdowns: 0 2019/08/19 11:21:32 Loop: 0, Int: 5, Dur(s): 1.0, Mode: DEL, Ops: 5610, MB/s: 21.91, IO/s: 5610, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 2.8, max: 8.4 ], Slowdowns: 0
2019/08/19 08:19:21 Loop: 0, Int: 8, Dur(s): 1.0, Mode: DEL, Ops: 5784, MB/s: 22.59, IO/s: 5784, Lat(ms): [ min: 1.2, avg: 1.7, 99%: 2.9, max: 14.3 ], Slowdowns: 0 2019/08/19 11:21:33 Loop: 0, Int: 6, Dur(s): 1.0, Mode: DEL, Ops: 5526, MB/s: 21.59, IO/s: 5526, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 2.8, max: 11.6 ], Slowdowns: 0
2019/08/19 08:19:21 Loop: 0, Int: TOTAL, Dur(s): 9.3, Mode: DEL, Ops: 52277, MB/s: 22.04, IO/s: 5643, Lat(ms): [ min: 1.1, avg: 1.8, 99%: 3.0, max: 16.3 ], Slowdowns: 0 2019/08/19 11:21:34 Loop: 0, Int: 7, Dur(s): 1.0, Mode: DEL, Ops: 5538, MB/s: 21.63, IO/s: 5538, Lat(ms): [ min: 1.2, avg: 1.8, 99%: 2.8, max: 8.7 ], Slowdowns: 0
2019/08/19 08:19:21 Running Loop 0 BUCKET DELETE TEST 2019/08/19 11:21:35 Loop: 0, Int: TOTAL, Dur(s): 8.6, Mode: DEL, Ops: 47783, MB/s: 21.61, IO/s: 5532, Lat(ms): [ min: 1.1, avg: 1.8, 99%: 2.8, max: 67.1 ], Slowdowns: 0
2019/08/19 08:19:21 Loop: 0, Int: TOTAL, Dur(s): 0.1, Mode: BDEL, Ops: 10, MB/s: 0.00, IO/s: 126, Lat(ms): [ min: 71.3, avg: 74.7, 99%: 79.3, max: 79.3 ], Slowdowns: 0 2019/08/19 11:21:35 Running Loop 0 BUCKET DELETE TEST
2019/08/19 11:21:35 Loop: 0, Int: TOTAL, Dur(s): 0.0, Mode: BDEL, Ops: 10, MB/s: 0.00, IO/s: 250, Lat(ms): [ min: 35.1, avg: 37.1, 99%: 39.4, max: 39.4 ], Slowdowns: 0
``` ```
One notable point is that like the s3-benchmark program it is based on, hsbench has relatively low CPU overhead compared to some other S3 benchmarks. During the 4K PUT phase of the above test: One notable point is that like the s3-benchmark program it is based on, hsbench has relatively low CPU overhead compared to some other S3 benchmarks. During the 4K PUT phase of the above test:

156
hsbench.go
View File

@ -12,6 +12,7 @@ import (
"crypto/tls" "crypto/tls"
"encoding/base64" "encoding/base64"
"encoding/csv" "encoding/csv"
"encoding/json"
"flag" "flag"
"fmt" "fmt"
"io" "io"
@ -36,7 +37,7 @@ import (
) )
// Global variables // Global variables
var access_key, secret_key, url_host, bucket_prefix, object_prefix, region, modes, output, sizeArg string var access_key, secret_key, url_host, bucket_prefix, object_prefix, region, modes, output, json_output, sizeArg string
var buckets []string var buckets []string
var duration_secs, threads, loops int var duration_secs, threads, loops int
var object_data []byte var object_data []byte
@ -45,7 +46,6 @@ var running_threads, bucket_count, object_count, object_size, op_counter int64
var object_count_flag bool var object_count_flag bool
var endtime time.Time var endtime time.Time
var interval float64 var interval float64
var csvWriter *csv.Writer
// Our HTTP transport used for the roundtripper below // Our HTTP transport used for the roundtripper below
var HTTPTransport http.RoundTripper = &http.Transport{ var HTTPTransport http.RoundTripper = &http.Transport{
@ -183,40 +183,40 @@ func (is *IntervalStats) makeOutputStats() OutputStats {
} }
type OutputStats struct { type OutputStats struct {
loop int Loop int
intervalName string IntervalName string
seconds float64 Seconds float64
mode string Mode string
ops int Ops int
mbps float64 Mbps float64
iops float64 Iops float64
minLat float64 MinLat float64
avgLat float64 AvgLat float64
NinetyNineLat float64 NinetyNineLat float64
maxLat float64 MaxLat float64
slowdowns int64 Slowdowns int64
} }
func (o *OutputStats) log() { func (o *OutputStats) log() {
log.Printf( log.Printf(
"Loop: %d, Int: %s, Dur(s): %.1f, Mode: %s, Ops: %d, MB/s: %.2f, IO/s: %.0f, Lat(ms): [ min: %.1f, avg: %.1f, 99%%: %.1f, max: %.1f ], Slowdowns: %d", "Loop: %d, Int: %s, Dur(s): %.1f, Mode: %s, Ops: %d, MB/s: %.2f, IO/s: %.0f, Lat(ms): [ min: %.1f, avg: %.1f, 99%%: %.1f, max: %.1f ], Slowdowns: %d",
o.loop, o.Loop,
o.intervalName, o.IntervalName,
o.seconds, o.Seconds,
o.mode, o.Mode,
o.ops, o.Ops,
o.mbps, o.Mbps,
o.iops, o.Iops,
o.minLat, o.MinLat,
o.avgLat, o.AvgLat,
o.NinetyNineLat, o.NinetyNineLat,
o.maxLat, o.MaxLat,
o.slowdowns) o.Slowdowns)
} }
func (o *OutputStats) csv_header(w *csv.Writer) { func (o *OutputStats) csv_header(w *csv.Writer) {
if w == nil { if w == nil {
log.Fatal("OutputStats Passed nil csv writer") log.Fatal("OutputStats passed nil CSV writer")
} }
s := []string{ s := []string{
@ -233,7 +233,7 @@ func (o *OutputStats) csv_header(w *csv.Writer) {
"Slowdowns"} "Slowdowns"}
if err := w.Write(s); err != nil { if err := w.Write(s); err != nil {
log.Fatal("Error writing to csv: ", err) log.Fatal("Error writing to CSV writer: ", err)
} }
} }
@ -243,24 +243,39 @@ func (o *OutputStats) csv(w *csv.Writer) {
} }
s := []string { s := []string {
strconv.Itoa(o.loop), strconv.Itoa(o.Loop),
o.intervalName, o.IntervalName,
strconv.FormatFloat(o.seconds, 'f', 2, 64), strconv.FormatFloat(o.Seconds, 'f', 2, 64),
o.mode, o.Mode,
strconv.Itoa(o.ops), strconv.Itoa(o.Ops),
strconv.FormatFloat(o.mbps, 'f', 2, 64), strconv.FormatFloat(o.Mbps, 'f', 2, 64),
strconv.FormatFloat(o.iops, 'f', 2, 64), strconv.FormatFloat(o.Iops, 'f', 2, 64),
strconv.FormatFloat(o.minLat, 'f', 2, 64), strconv.FormatFloat(o.MinLat, 'f', 2, 64),
strconv.FormatFloat(o.avgLat, 'f', 2, 64), strconv.FormatFloat(o.AvgLat, 'f', 2, 64),
strconv.FormatFloat(o.NinetyNineLat, 'f', 2, 64), strconv.FormatFloat(o.NinetyNineLat, 'f', 2, 64),
strconv.FormatFloat(o.maxLat, 'f', 2, 64), strconv.FormatFloat(o.MaxLat, 'f', 2, 64),
strconv.FormatInt(o.slowdowns, 10)} strconv.FormatInt(o.Slowdowns, 10)}
if err := w.Write(s); err != nil { if err := w.Write(s); err != nil {
log.Fatal("Error writing to csv: ", err) log.Fatal("Error writing to CSV writer: ",err)
} }
} }
func (o *OutputStats) json(jfile *os.File) {
if jfile == nil {
log.Fatal("OutputStats passed nil JSON file")
}
jdata, err := json.Marshal(o)
if err != nil {
log.Fatal("Error marshaling JSON: ", err)
}
log.Println(string(jdata))
_, err = jfile.WriteString(string(jdata) + "\n")
if err != nil {
log.Fatal("Error writing to JSON file: ", err)
}
}
type ThreadStats struct { type ThreadStats struct {
start int64 start int64
curInterval int64 curInterval int64
@ -687,7 +702,7 @@ func runBucketsClear(thread_num int, stats *Stats) {
atomic.AddInt64(&running_threads, -1) atomic.AddInt64(&running_threads, -1)
} }
func runWrapper(loop int, r rune) { func runWrapper(loop int, r rune) []OutputStats {
op_counter = -1 op_counter = -1
running_threads = int64(threads) running_threads = int64(threads)
intervalNano := int64(interval*1000000000) intervalNano := int64(interval*1000000000)
@ -752,25 +767,20 @@ func runWrapper(loop int, r rune) {
object_count_flag = true object_count_flag = true
} }
// Print Interval Output // Create the Output Stats
for i := int64(0); i >= 0;i++ { os := make([]OutputStats, 0)
for i := int64(0); i >= 0; i++ {
if o, ok := stats.makeOutputStats(i); ok { if o, ok := stats.makeOutputStats(i); ok {
if csvWriter != nil { os = append(os, o)
o.csv(csvWriter)
}
} else { } else {
break break
} }
} }
// Print Totals Output
if o, ok := stats.makeTotalStats(); ok { if o, ok := stats.makeTotalStats(); ok {
o.log() o.log()
if csvWriter != nil { os = append(os, o)
o.csv(csvWriter)
csvWriter.Flush()
}
} }
return os
} }
func init() { func init() {
@ -784,6 +794,7 @@ func init() {
myflag.StringVar(&region, "r", "us-east-1", "Region for testing") myflag.StringVar(&region, "r", "us-east-1", "Region for testing")
myflag.StringVar(&modes, "m", "cxipgdx", "Run modes in order. See NOTES for more info") myflag.StringVar(&modes, "m", "cxipgdx", "Run modes in order. See NOTES for more info")
myflag.StringVar(&output, "o", "", "Write CSV output to this file") myflag.StringVar(&output, "o", "", "Write CSV output to this file")
myflag.StringVar(&json_output, "j", "", "Write JSON output to this file")
myflag.Int64Var(&object_count, "n", -1, "Maximum number of objects <-1 for unlimited>") myflag.Int64Var(&object_count, "n", -1, "Maximum number of objects <-1 for unlimited>")
myflag.Int64Var(&bucket_count, "b", 1, "Number of buckets to distribute IOs across") myflag.Int64Var(&bucket_count, "b", 1, "Number of buckets to distribute IOs across")
myflag.IntVar(&duration_secs, "d", 60, "Maximum test duration in seconds <-1 for unlimited>") myflag.IntVar(&duration_secs, "d", 60, "Maximum test duration in seconds <-1 for unlimited>")
@ -886,6 +897,8 @@ func main() {
log.Printf("bucket_prefix=%s", bucket_prefix) log.Printf("bucket_prefix=%s", bucket_prefix)
log.Printf("region=%s", region) log.Printf("region=%s", region)
log.Printf("modes=%s", modes) log.Printf("modes=%s", modes)
log.Printf("output=%s", output)
log.Printf("json_output=%s", json_output)
log.Printf("object_count=%d", object_count) log.Printf("object_count=%d", object_count)
log.Printf("bucket_count=%d", bucket_count) log.Printf("bucket_count=%d", bucket_count)
log.Printf("duration=%d", duration_secs) log.Printf("duration=%d", duration_secs)
@ -902,22 +915,47 @@ func main() {
buckets = append(buckets, fmt.Sprintf("%s%012d", bucket_prefix, i)) buckets = append(buckets, fmt.Sprintf("%s%012d", bucket_prefix, i))
} }
// Init CSV file // Loop running the tests
oStats := make([]OutputStats, 0)
for loop := 0; loop < loops; loop++ {
for _, r := range modes {
oStats = append(oStats, runWrapper(loop, r)...)
}
}
// Write CSV Output
if output != "" { if output != "" {
file, err := os.OpenFile(output, os.O_CREATE|os.O_WRONLY, 0777) file, err := os.OpenFile(output, os.O_CREATE|os.O_WRONLY, 0777)
defer file.Close() defer file.Close()
if err != nil { if err != nil {
log.Fatal("Could not open CSV file for writing.") log.Fatal("Could not open CSV file for writing.")
} else { } else {
csvWriter = csv.NewWriter(file) csvWriter := csv.NewWriter(file)
o := OutputStats{} for i, o := range oStats {
o.csv_header(csvWriter) if i == 0 {
o.csv_header(csvWriter)
}
o.csv(csvWriter)
}
csvWriter.Flush()
} }
} }
// Loop running the tests
for loop := 0; loop < loops; loop++ { // Write JSON output
for _, r := range modes { if json_output != "" {
runWrapper(loop, r) file, err := os.OpenFile(json_output, os.O_CREATE|os.O_WRONLY, 0777)
} defer file.Close()
if err != nil {
log.Fatal("Could not open JSON file for writing.")
}
data, err := json.Marshal(oStats)
if err != nil {
log.Fatal("Error marshaling JSON: ", err)
}
_, err = file.Write(data)
if err != nil {
log.Fatal("Error writing to JSON file: ", err)
}
file.Sync()
} }
} }