Merge pull request #97 from Tantalor93/json

README.md

@@ -91,6 +91,7 @@ Flags:
       --precision=[1-5]        Significant figure for histogram precision.
       --distribution           Display distribution histogram of timings to stdout. Enabled by default. Specifying --no-distribution disables histogram display.
       --csv=/path/to/file.csv  Export distribution to CSV.
+      --json                   Report benchmark results as JSON.
       --silent                 Disable stdout.
       --color                  ANSI Color output. Enabled by default. By specifying --no-color disables coloring.
       --plot=/path/to/folder   Plot benchmark results and export them to the directory.

cmd/dnspyre/benchmark.go

@@ -57,7 +57,8 @@ type Benchmark struct {
 	HistMax     time.Duration
 	HistPre     int
 
-	Csv string
+	Csv  string
+	JSON bool
 
 	Silent bool
 	Color  bool
@@ -126,7 +127,7 @@ func (b *Benchmark) Run(ctx context.Context) ([]*ResultStats, error) {
 		defer cancel()
 	}
 
-	if !b.Silent {
+	if !b.Silent && !b.JSON {
 		fmt.Printf("Using %s hostnames\n", highlightStr(len(questions)))
 	}
 
@@ -182,7 +183,7 @@ func (b *Benchmark) Run(ctx context.Context) ([]*ResultStats, error) {
 		limits = fmt.Sprintf("(limited to %s QPS)", highlightStr(b.Rate))
 	}
 
-	if !b.Silent {
+	if !b.Silent && !b.JSON {
 		fmt.Printf("Benchmarking %s via %s with %s concurrent requests %s\n", highlightStr(b.Server), highlightStr(network), highlightStr(b.Concurrency), limits)
 	}
 

cmd/dnspyre/jsonreporter.go

@@ -0,0 +1,116 @@
+package dnspyre
+
+import (
+	"encoding/json"
+	"math"
+	"os"
+	"time"
+
+	"github.com/HdrHistogram/hdrhistogram-go"
+	"github.com/miekg/dns"
+)
+
+type jsonReporter struct{}
+
+type latencyStats struct {
+	MinMs  int64 `json:"minMs"`
+	MeanMs int64 `json:"meanMs"`
+	StdMs  int64 `json:"stdMs"`
+	MaxMs  int64 `json:"maxMs"`
+	P99Ms  int64 `json:"p99Ms"`
+	P95Ms  int64 `json:"p95Ms"`
+	P90Ms  int64 `json:"p90Ms"`
+	P75Ms  int64 `json:"p75Ms"`
+	P50Ms  int64 `json:"p50Ms"`
+}
+
+type histogramPoint struct {
+	LatencyMs int64 `json:"latencyMs"`
+	Count     int64 `json:"count"`
+}
+
+type jsonResult struct {
+	TotalRequests            int64            `json:"totalRequests"`
+	TotalSuccessCodes        int64            `json:"totalSuccessCodes"`
+	TotalErrors              int64            `json:"totalErrors"`
+	TotalIDmismatch          int64            `json:"TotalIDmismatch"`
+	TotalTruncatedResponses  int64            `json:"totalTruncatedResponses"`
+	ResponseRcodes           map[string]int64 `json:"responseRcodes,omitempty"`
+	QuestionTypes            map[string]int64 `json:"questionTypes"`
+	QueriesPerSecond         float64          `json:"queriesPerSecond"`
+	BenchmarkDurationSeconds float64          `json:"benchmarkDurationSeconds"`
+	LatencyStats             latencyStats     `json:"latencyStats"`
+	LatencyDistribution      []histogramPoint `json:"latencyDistribution,omitempty"`
+}
+
+func (s *jsonReporter) print(b *Benchmark, timings *hdrhistogram.Histogram, codeTotals map[int]int64, totalCounters Counters, qtypeTotals map[string]int64, topErrs orderedMap, t time.Duration) error {
+	sumerrs := int64(0)
+	for _, v := range topErrs.m {
+		sumerrs += int64(v)
+	}
+
+	codeTotalsMapped := make(map[string]int64)
+	if b.Rcodes {
+		for k, v := range codeTotals {
+			codeTotalsMapped[dns.RcodeToString[k]] = v
+		}
+	}
+
+	var res []histogramPoint
+
+	if b.HistDisplay {
+		dist := timings.Distribution()
+		for _, d := range dist {
+			res = append(res, histogramPoint{
+				LatencyMs: time.Duration(d.To/2 + d.From/2).Milliseconds(),
+				Count:     d.Count,
+			})
+		}
+
+		var dedupRes []histogramPoint
+		i := -1
+		for _, r := range res {
+			if i >= 0 && i < len(res) {
+				if dedupRes[i].LatencyMs == r.LatencyMs {
+					dedupRes[i].Count += r.Count
+				} else {
+					dedupRes = append(dedupRes, r)
+					i++
+				}
+			} else {
+				dedupRes = append(dedupRes, r)
+				i++
+			}
+		}
+	}
+
+	result := jsonResult{
+		TotalRequests:            totalCounters.Total,
+		TotalSuccessCodes:        totalCounters.Success,
+		TotalErrors:              sumerrs,
+		TotalIDmismatch:          totalCounters.IDmismatch,
+		TotalTruncatedResponses:  totalCounters.Truncated,
+		QueriesPerSecond:         math.Round(float64(totalCounters.Total)/t.Seconds()*100) / 100,
+		BenchmarkDurationSeconds: roundDuration(t).Seconds(),
+		ResponseRcodes:           codeTotalsMapped,
+		QuestionTypes:            qtypeTotals,
+		LatencyStats: latencyStats{
+			MinMs:  time.Duration(timings.Min()).Milliseconds(),
+			MeanMs: time.Duration(timings.Mean()).Milliseconds(),
+			StdMs:  time.Duration(timings.StdDev()).Milliseconds(),
+			MaxMs:  time.Duration(timings.Max()).Milliseconds(),
+			P99Ms:  time.Duration(timings.ValueAtQuantile(99)).Milliseconds(),
+			P95Ms:  time.Duration(timings.ValueAtQuantile(95)).Milliseconds(),
+			P90Ms:  time.Duration(timings.ValueAtQuantile(90)).Milliseconds(),
+			P75Ms:  time.Duration(timings.ValueAtQuantile(75)).Milliseconds(),
+			P50Ms:  time.Duration(timings.ValueAtQuantile(50)).Milliseconds(),
+		},
+		LatencyDistribution: res,
+	}
+
+	if err := json.NewEncoder(os.Stdout).Encode(result); err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/dnspyre/report.go

@@ -122,8 +122,16 @@ func (b *Benchmark) PrintReport(stats []*ResultStats, t time.Duration) error {
 		writeBars(csv, timings.Distribution())
 	}
 
+	if b.Silent {
+		return nil
+	}
+	topErrs := orderedMap{m: top3errs, order: top3errorsInOrder}
+	if b.JSON {
+		j := jsonReporter{}
+		return j.print(b, timings, codeTotals, totalCounters, qtypeTotals, topErrs, t)
+	}
 	s := standardReporter{}
-	return s.print(b, timings, codeTotals, totalCounters, qtypeTotals, orderedMap{m: top3errs, order: top3errorsInOrder}, t)
+	return s.print(b, timings, codeTotals, totalCounters, qtypeTotals, topErrs, t)
 }
 
 func (b *Benchmark) fileName(dir, name string) string {

cmd/dnspyre/report_test.go

@@ -8,39 +8,8 @@ import (
 	"github.com/miekg/dns"
 )
 
-func Example_printReport() {
-	b := Benchmark{
-		HistPre: 1,
-	}
-
-	h := hdrhistogram.New(0, 0, 1)
-	h.RecordValue(5)
-	h.RecordValue(10)
-	d1 := Datapoint{5, time.Unix(0, 0)}
-	d2 := Datapoint{10, time.Unix(0, 0)}
-	rs := ResultStats{
-		Codes: map[int]int64{
-			dns.RcodeSuccess: 2,
-		},
-		Qtypes: map[string]int64{
-			"A": 2,
-		},
-		Hist:    h,
-		Timings: []Datapoint{d1, d2},
-		Counters: &Counters{
-			Total:      1,
-			ConnError:  2,
-			IOError:    3,
-			Success:    4,
-			IDmismatch: 6,
-			Truncated:  7,
-		},
-		Errors: []error{
-			errors.New("test"),
-			errors.New("test2"),
-			errors.New("test"),
-		},
-	}
+func Example_standard_printReport() {
+	b, rs := testData()
 
 	b.PrintReport([]*ResultStats{&rs}, time.Second)
 
@@ -75,3 +44,50 @@ func Example_printReport() {
 	// test	2 (66.67)%
 	// test2	1 (33.33)%
 }
+
+func Example_json_printReport() {
+	b, rs := testData()
+	b.JSON = true
+	b.Rcodes = true
+	b.HistDisplay = true
+
+	b.PrintReport([]*ResultStats{&rs}, time.Second)
+
+	// Output: {"totalRequests":1,"totalSuccessCodes":4,"totalErrors":3,"TotalIDmismatch":6,"totalTruncatedResponses":7,"responseRcodes":{"NOERROR":2},"questionTypes":{"A":2},"queriesPerSecond":1,"benchmarkDurationSeconds":1,"latencyStats":{"minMs":0,"meanMs":0,"stdMs":0,"maxMs":0,"p99Ms":0,"p95Ms":0,"p90Ms":0,"p75Ms":0,"p50Ms":0},"latencyDistribution":[{"latencyMs":0,"count":0},{"latencyMs":0,"count":0},{"latencyMs":0,"count":0},{"latencyMs":0,"count":0},{"latencyMs":0,"count":0},{"latencyMs":0,"count":1},{"latencyMs":0,"count":0},{"latencyMs":0,"count":0},{"latencyMs":0,"count":0},{"latencyMs":0,"count":0},{"latencyMs":0,"count":1}]}
+}
+
+func testData() (Benchmark, ResultStats) {
+	b := Benchmark{
+		HistPre: 1,
+	}
+
+	h := hdrhistogram.New(0, 0, 1)
+	h.RecordValue(5)
+	h.RecordValue(10)
+	d1 := Datapoint{5, time.Unix(0, 0)}
+	d2 := Datapoint{10, time.Unix(0, 0)}
+	rs := ResultStats{
+		Codes: map[int]int64{
+			dns.RcodeSuccess: 2,
+		},
+		Qtypes: map[string]int64{
+			"A": 2,
+		},
+		Hist:    h,
+		Timings: []Datapoint{d1, d2},
+		Counters: &Counters{
+			Total:      1,
+			ConnError:  2,
+			IOError:    3,
+			Success:    4,
+			IDmismatch: 6,
+			Truncated:  7,
+		},
+		Errors: []error{
+			errors.New("test"),
+			errors.New("test2"),
+			errors.New("test"),
+		},
+	}
+	return b, rs
+}

cmd/dnspyre/root.go

@@ -54,7 +54,8 @@ var (
 	pHistPre     = pApp.Flag("precision", "Significant figure for histogram precision.").Default("1").PlaceHolder("[1-5]").Int()
 	pHistDisplay = pApp.Flag("distribution", "Display distribution histogram of timings to stdout. Enabled by default. Specifying --no-distribution disables histogram display.").Default("true").Bool()
 
-	pCsv = pApp.Flag("csv", "Export distribution to CSV.").Default("").PlaceHolder("/path/to/file.csv").String()
+	pCsv  = pApp.Flag("csv", "Export distribution to CSV.").Default("").PlaceHolder("/path/to/file.csv").String()
+	pJSON = pApp.Flag("json", "Report benchmark results as JSON.").Bool()
 
 	pSilent = pApp.Flag("silent", "Disable stdout.").Default("false").Bool()
 	pColor  = pApp.Flag("color", "ANSI Color output. Enabled by default. By specifying --no-color disables coloring.").Default("true").Bool()
@@ -103,6 +104,7 @@ func Execute() {
 		HistPre:      *pHistPre,
 		HistDisplay:  *pHistDisplay,
 		Csv:          *pCsv,
+		JSON:         *pJSON,
 		Silent:       *pSilent,
 		Color:        *pColor,
 		PlotDir:      *pPlotDir,

cmd/dnspyre/stdreporter.go

@@ -15,10 +15,6 @@ import (
 type standardReporter struct{}
 
 func (s *standardReporter) print(b *Benchmark, timings *hdrhistogram.Histogram, codeTotals map[int]int64, totalCounters Counters, qtypeTotals map[string]int64, topErrs orderedMap, t time.Duration) error {
-	if b.Silent {
-		return nil
-	}
-
 	b.printProgress(totalCounters)
 
 	if len(codeTotals) > 0 {

docs/examples.md

@@ -1296,3 +1296,23 @@ generates graphs like these:
 ![latency line](graphs/latency-lineplot.png)
 
 </details>
+
+### Output benchmark results as JSON
+By specifying `--json` flag, dnspyre can output benchmark results in a JSON format, which is better for further automatic processing
+```
+dnspyre --duration 5s --server 8.8.8.8 google.com  --recurse --json
+```
+
+<details>
+<summary>output</summary>
+
+```
+{"totalRequests":792,"totalSuccessCodes":792,"totalErrors":0,"TotalIDmismatch":0,"totalTruncatedResponses":0,"responseRcodes":{"NOERROR":792},"questionTypes":{"A":792},"queriesPerSecond":158.19,"benchmarkDurationSeconds":5.01,"latencyStats":{"minMs":4,"meanMs":6,"stdMs":1,"maxMs":15,"p99Ms":13,"p95Ms":8,"p90Ms":7,"p75Ms":6,"p50Ms":6},"latencyDistribution":[{"latencyMs":0,"count":0},{"latencyMs":0,"count":0},{"latencyMs":0,"count":0},{"latencyMs":0,"count":0},{"latencyMs":1,"count":0},{"latencyMs":1,"count":0},{"latencyMs":1,"count":0},{"latencyMs":1,"count":0},{"latencyMs":2,"count":0},{"latencyMs":2,"count":0},{"latencyMs":2,"count":0},{"latencyMs":3,"count":0},{"latencyMs":3,"count":0},{"latencyMs":3,"count":0},{"latencyMs":3,"count":0},{"latencyMs":4,"count":0},{"latencyMs":4,"count":0},{"latencyMs":4,"count":0},{"latencyMs":4,"count":10},{"latencyMs":5,"count":9},{"latencyMs":5,"count":16},{"latencyMs":5,"count":60},{"latencyMs":5,"count":311},{"latencyMs":6,"count":225},{"latencyMs":6,"count":40},{"latencyMs":6,"count":13},{"latencyMs":6,"count":22},{"latencyMs":7,"count":14},{"latencyMs":7,"count":17},{"latencyMs":7,"count":9},{"latencyMs":7,"count":12},{"latencyMs":8,"count":9},{"latencyMs":8,"count":3},{"latencyMs":9,"count":4},{"latencyMs":9,"count":2},{"latencyMs":10,"count":1},{"latencyMs":10,"count":1},{"latencyMs":11,"count":0},{"latencyMs":11,"count":0},{"latencyMs":12,"count":0},{"latencyMs":12,"count":0},{"latencyMs":13,"count":6},{"latencyMs":13,"count":4},{"latencyMs":14,"count":1},{"latencyMs":14,"count":2},{"latencyMs":15,"count":1}]}
+```
+
+</details>
+
+example of chaining of dnspyre with [jq](https://stedolan.github.io/jq/) for getting pretty JSON
+```
+dnspyre --duration 5s --server 8.8.8.8 google.com --no-distribution --recurse --json | jq '.'
+```