commit
457598e1b6
63
hsbench.go
63
hsbench.go
|
@ -6,6 +6,7 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"code.cloudfoundry.org/bytefmt"
|
||||||
"crypto/hmac"
|
"crypto/hmac"
|
||||||
"crypto/md5"
|
"crypto/md5"
|
||||||
"crypto/sha1"
|
"crypto/sha1"
|
||||||
|
@ -15,6 +16,10 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/session"
|
||||||
|
"github.com/aws/aws-sdk-go/service/s3"
|
||||||
"io"
|
"io"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"log"
|
"log"
|
||||||
|
@ -29,11 +34,6 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
"code.cloudfoundry.org/bytefmt"
|
|
||||||
"github.com/aws/aws-sdk-go/aws"
|
|
||||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
|
||||||
"github.com/aws/aws-sdk-go/aws/session"
|
|
||||||
"github.com/aws/aws-sdk-go/service/s3"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Global variables
|
// Global variables
|
||||||
|
@ -148,19 +148,19 @@ type IntervalStats struct {
|
||||||
func (is *IntervalStats) makeOutputStats() OutputStats {
|
func (is *IntervalStats) makeOutputStats() OutputStats {
|
||||||
// Compute and log the stats
|
// Compute and log the stats
|
||||||
ops := len(is.latNano)
|
ops := len(is.latNano)
|
||||||
totalLat := int64(0);
|
totalLat := int64(0)
|
||||||
minLat := float64(0);
|
minLat := float64(0)
|
||||||
maxLat := float64(0);
|
maxLat := float64(0)
|
||||||
NinetyNineLat := float64(0);
|
NinetyNineLat := float64(0)
|
||||||
avgLat := float64(0);
|
avgLat := float64(0)
|
||||||
if ops > 0 {
|
if ops > 0 {
|
||||||
minLat = float64(is.latNano[0]) / 1000000
|
minLat = float64(is.latNano[0]) / 1000000
|
||||||
maxLat = float64(is.latNano[ops - 1]) / 1000000
|
maxLat = float64(is.latNano[ops-1]) / 1000000
|
||||||
for i := range is.latNano {
|
for i := range is.latNano {
|
||||||
totalLat += is.latNano[i]
|
totalLat += is.latNano[i]
|
||||||
}
|
}
|
||||||
avgLat = float64(totalLat) / float64(ops) / 1000000
|
avgLat = float64(totalLat) / float64(ops) / 1000000
|
||||||
NintyNineLatNano := is.latNano[int64(math.Round(0.99*float64(ops))) - 1]
|
NintyNineLatNano := is.latNano[int64(math.Round(0.99*float64(ops)))-1]
|
||||||
NinetyNineLat = float64(NintyNineLatNano) / 1000000
|
NinetyNineLat = float64(NintyNineLatNano) / 1000000
|
||||||
}
|
}
|
||||||
seconds := float64(is.intervalNano) / 1000000000
|
seconds := float64(is.intervalNano) / 1000000000
|
||||||
|
@ -242,7 +242,7 @@ func (o *OutputStats) csv(w *csv.Writer) {
|
||||||
log.Fatal("OutputStats Passed nil csv writer")
|
log.Fatal("OutputStats Passed nil csv writer")
|
||||||
}
|
}
|
||||||
|
|
||||||
s := []string {
|
s := []string{
|
||||||
strconv.Itoa(o.Loop),
|
strconv.Itoa(o.Loop),
|
||||||
o.IntervalName,
|
o.IntervalName,
|
||||||
strconv.FormatFloat(o.Seconds, 'f', 2, 64),
|
strconv.FormatFloat(o.Seconds, 'f', 2, 64),
|
||||||
|
@ -257,7 +257,7 @@ func (o *OutputStats) csv(w *csv.Writer) {
|
||||||
strconv.FormatInt(o.Slowdowns, 10)}
|
strconv.FormatInt(o.Slowdowns, 10)}
|
||||||
|
|
||||||
if err := w.Write(s); err != nil {
|
if err := w.Write(s); err != nil {
|
||||||
log.Fatal("Error writing to CSV writer: ",err)
|
log.Fatal("Error writing to CSV writer: ", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -293,7 +293,7 @@ func (ts *ThreadStats) updateIntervals(loop int, mode string, intervalNano int64
|
||||||
if intervalNano < 0 {
|
if intervalNano < 0 {
|
||||||
return ts.curInterval
|
return ts.curInterval
|
||||||
}
|
}
|
||||||
for ts.start + intervalNano*(ts.curInterval+1) < time.Now().UnixNano() {
|
for ts.start+intervalNano*(ts.curInterval+1) < time.Now().UnixNano() {
|
||||||
ts.curInterval++
|
ts.curInterval++
|
||||||
ts.intervals = append(
|
ts.intervals = append(
|
||||||
ts.intervals,
|
ts.intervals,
|
||||||
|
@ -386,14 +386,14 @@ func (stats *Stats) makeOutputStats(i int64) (OutputStats, bool) {
|
||||||
func (stats *Stats) makeTotalStats() (OutputStats, bool) {
|
func (stats *Stats) makeTotalStats() (OutputStats, bool) {
|
||||||
// Not safe to log if not all writers have completed.
|
// Not safe to log if not all writers have completed.
|
||||||
completions := atomic.LoadInt32(&stats.completions)
|
completions := atomic.LoadInt32(&stats.completions)
|
||||||
if (completions < int32(threads)) {
|
if completions < int32(threads) {
|
||||||
log.Printf("log, completions: %d", completions)
|
log.Printf("log, completions: %d", completions)
|
||||||
return OutputStats{}, false
|
return OutputStats{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
bytes := int64(0)
|
bytes := int64(0)
|
||||||
ops := int64(0)
|
ops := int64(0)
|
||||||
slowdowns := int64(0);
|
slowdowns := int64(0)
|
||||||
|
|
||||||
for t := 0; t < stats.threads; t++ {
|
for t := 0; t < stats.threads; t++ {
|
||||||
for i := 0; i < len(stats.threadStats[t].intervals); i++ {
|
for i := 0; i < len(stats.threadStats[t].intervals); i++ {
|
||||||
|
@ -501,7 +501,7 @@ func runUpload(thread_num int, fendtime time.Time, stats *Stats) {
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errcnt++
|
errcnt++
|
||||||
stats.addSlowDown(thread_num);
|
stats.addSlowDown(thread_num)
|
||||||
atomic.AddInt64(&op_counter, -1)
|
atomic.AddInt64(&op_counter, -1)
|
||||||
log.Printf("upload err", err)
|
log.Printf("upload err", err)
|
||||||
} else {
|
} else {
|
||||||
|
@ -545,7 +545,7 @@ func runDownload(thread_num int, fendtime time.Time, stats *Stats) {
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errcnt++
|
errcnt++
|
||||||
stats.addSlowDown(thread_num);
|
stats.addSlowDown(thread_num)
|
||||||
log.Printf("download err", err)
|
log.Printf("download err", err)
|
||||||
} else {
|
} else {
|
||||||
// Update the stats
|
// Update the stats
|
||||||
|
@ -595,7 +595,7 @@ func runDelete(thread_num int, stats *Stats) {
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errcnt++
|
errcnt++
|
||||||
stats.addSlowDown(thread_num);
|
stats.addSlowDown(thread_num)
|
||||||
log.Printf("delete err", err, "out", out.String())
|
log.Printf("delete err", err, "out", out.String())
|
||||||
} else {
|
} else {
|
||||||
// Update the stats
|
// Update the stats
|
||||||
|
@ -705,7 +705,7 @@ func runBucketsClear(thread_num int, stats *Stats) {
|
||||||
func runWrapper(loop int, r rune) []OutputStats {
|
func runWrapper(loop int, r rune) []OutputStats {
|
||||||
op_counter = -1
|
op_counter = -1
|
||||||
running_threads = int64(threads)
|
running_threads = int64(threads)
|
||||||
intervalNano := int64(interval*1000000000)
|
intervalNano := int64(interval * 1000000000)
|
||||||
endtime = time.Now().Add(time.Second * time.Duration(duration_secs))
|
endtime = time.Now().Add(time.Second * time.Duration(duration_secs))
|
||||||
var stats Stats
|
var stats Stats
|
||||||
|
|
||||||
|
@ -721,37 +721,37 @@ func runWrapper(loop int, r rune) []OutputStats {
|
||||||
log.Printf("Running Loop %d BUCKET CLEAR TEST", loop)
|
log.Printf("Running Loop %d BUCKET CLEAR TEST", loop)
|
||||||
stats = makeStats(loop, "BCLR", threads, intervalNano)
|
stats = makeStats(loop, "BCLR", threads, intervalNano)
|
||||||
for n := 0; n < threads; n++ {
|
for n := 0; n < threads; n++ {
|
||||||
go runBucketsClear(n, &stats);
|
go runBucketsClear(n, &stats)
|
||||||
}
|
}
|
||||||
case 'x':
|
case 'x':
|
||||||
log.Printf("Running Loop %d BUCKET DELETE TEST", loop)
|
log.Printf("Running Loop %d BUCKET DELETE TEST", loop)
|
||||||
stats = makeStats(loop, "BDEL", threads, intervalNano)
|
stats = makeStats(loop, "BDEL", threads, intervalNano)
|
||||||
for n := 0; n < threads; n++ {
|
for n := 0; n < threads; n++ {
|
||||||
go runBucketDelete(n, &stats);
|
go runBucketDelete(n, &stats)
|
||||||
}
|
}
|
||||||
case 'i':
|
case 'i':
|
||||||
log.Printf("Running Loop %d BUCKET INIT TEST", loop)
|
log.Printf("Running Loop %d BUCKET INIT TEST", loop)
|
||||||
stats = makeStats(loop, "BINIT", threads, intervalNano)
|
stats = makeStats(loop, "BINIT", threads, intervalNano)
|
||||||
for n := 0; n < threads; n++ {
|
for n := 0; n < threads; n++ {
|
||||||
go runBucketsInit(n, &stats);
|
go runBucketsInit(n, &stats)
|
||||||
}
|
}
|
||||||
case 'p':
|
case 'p':
|
||||||
log.Printf("Running Loop %d OBJECT PUT TEST", loop)
|
log.Printf("Running Loop %d OBJECT PUT TEST", loop)
|
||||||
stats = makeStats(loop, "PUT", threads, intervalNano)
|
stats = makeStats(loop, "PUT", threads, intervalNano)
|
||||||
for n := 0; n < threads; n++ {
|
for n := 0; n < threads; n++ {
|
||||||
go runUpload(n, endtime, &stats);
|
go runUpload(n, endtime, &stats)
|
||||||
}
|
}
|
||||||
case 'g':
|
case 'g':
|
||||||
log.Printf("Running Loop %d OBJECT GET TEST", loop)
|
log.Printf("Running Loop %d OBJECT GET TEST", loop)
|
||||||
stats = makeStats(loop, "GET", threads, intervalNano)
|
stats = makeStats(loop, "GET", threads, intervalNano)
|
||||||
for n := 0; n < threads; n++ {
|
for n := 0; n < threads; n++ {
|
||||||
go runDownload(n, endtime, &stats);
|
go runDownload(n, endtime, &stats)
|
||||||
}
|
}
|
||||||
case 'd':
|
case 'd':
|
||||||
log.Printf("Running Loop %d OBJECT DELETE TEST", loop)
|
log.Printf("Running Loop %d OBJECT DELETE TEST", loop)
|
||||||
stats = makeStats(loop, "DEL", threads, intervalNano)
|
stats = makeStats(loop, "DEL", threads, intervalNano)
|
||||||
for n := 0; n < threads; n++ {
|
for n := 0; n < threads; n++ {
|
||||||
go runDelete(n, &stats);
|
go runDelete(n, &stats)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -804,7 +804,7 @@ func init() {
|
||||||
myflag.Float64Var(&interval, "ri", 1.0, "Number of seconds between report intervals")
|
myflag.Float64Var(&interval, "ri", 1.0, "Number of seconds between report intervals")
|
||||||
// define custom usage output with notes
|
// define custom usage output with notes
|
||||||
notes :=
|
notes :=
|
||||||
`
|
`
|
||||||
NOTES:
|
NOTES:
|
||||||
- Valid mode types for the -m mode string are:
|
- Valid mode types for the -m mode string are:
|
||||||
c: clear all existing objects from buckets (requires lookups)
|
c: clear all existing objects from buckets (requires lookups)
|
||||||
|
@ -823,7 +823,7 @@ NOTES:
|
||||||
fmt.Fprintf(flag.CommandLine.Output(), "\nUSAGE: %s [OPTIONS]\n\n", os.Args[0])
|
fmt.Fprintf(flag.CommandLine.Output(), "\nUSAGE: %s [OPTIONS]\n\n", os.Args[0])
|
||||||
fmt.Fprintf(flag.CommandLine.Output(), "OPTIONS:\n")
|
fmt.Fprintf(flag.CommandLine.Output(), "OPTIONS:\n")
|
||||||
myflag.PrintDefaults()
|
myflag.PrintDefaults()
|
||||||
fmt.Fprintf(flag.CommandLine.Output(), notes);
|
fmt.Fprintf(flag.CommandLine.Output(), notes)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := myflag.Parse(os.Args[1:]); err != nil {
|
if err := myflag.Parse(os.Args[1:]); err != nil {
|
||||||
|
@ -845,13 +845,12 @@ NOTES:
|
||||||
}
|
}
|
||||||
invalid_mode := false
|
invalid_mode := false
|
||||||
for _, r := range modes {
|
for _, r := range modes {
|
||||||
if (
|
if r != 'i' &&
|
||||||
r != 'i' &&
|
|
||||||
r != 'c' &&
|
r != 'c' &&
|
||||||
r != 'p' &&
|
r != 'p' &&
|
||||||
r != 'g' &&
|
r != 'g' &&
|
||||||
r != 'd' &&
|
r != 'd' &&
|
||||||
r != 'x') {
|
r != 'x' {
|
||||||
s := fmt.Sprintf("Invalid mode '%s' passed to -m", string(r))
|
s := fmt.Sprintf("Invalid mode '%s' passed to -m", string(r))
|
||||||
log.Printf(s)
|
log.Printf(s)
|
||||||
invalid_mode = true
|
invalid_mode = true
|
||||||
|
|
Loading…
Reference in New Issue