Record/Report: add support of millisecond resolution.

cmd/help.go

@@ -158,7 +158,6 @@ Options:
  -g, --grep COLNAME:PATTERN	filter values in specfied column (format: colname:filtertext)
  -l, --limit INT		print only limited number of rows per sample (default: unlimited)
  -t, --strlimit INT		maximum string size to print (default: 32, 0 disables)
- -r, --rate DURATION		statistics changes rate interval (default: 1s)
 
 Report options:
  -A, --activity			show pg_stat_activity statistics

cmd/report/report.go

@@ -26,15 +26,14 @@ type options struct {
 	showStatements  string // Show stats from pg_stat_statements
 	showProgress    string // Show stats from pg_stat_progress_* stats
 
-	inputFile      string        // Input file with statistics
-	tsStart, tsEnd string        // Show stats within an interval
-	orderColName   string        // Name of the column used for sorting
-	orderDesc      bool          // Specify to use descendant order
-	orderAsc       bool          // Specify to use ascendant order
-	filter         string        // Perform filtering
-	rowLimit       int           // Number of rows per timestamp
-	strLimit       int           // Trim all strings longer than this limit
-	rate           time.Duration // Stats rate
+	inputFile      string // Input file with statistics
+	tsStart, tsEnd string // Show stats within an interval
+	orderColName   string // Name of the column used for sorting
+	orderDesc      bool   // Specify to use descendant order
+	orderAsc       bool   // Specify to use ascendant order
+	filter         string // Perform filtering
+	rowLimit       int    // Number of rows per timestamp
+	strLimit       int    // Trim all strings longer than this limit
 }
 
 var (
@@ -78,7 +77,6 @@ func init() {
 	CommandDefinition.Flags().StringVarP(&opts.filter, "grep", "g", "", "grep values in specified column (format: colname:filter_pattern)")
 	CommandDefinition.Flags().IntVarP(&opts.rowLimit, "limit", "l", 0, "print only limited number of rows per sample")
 	CommandDefinition.Flags().IntVarP(&opts.strLimit, "strlimit", "t", 32, "maximum string size for long lines to print (default: 32)")
-	CommandDefinition.Flags().DurationVarP(&opts.rate, "rate", "r", time.Second, "statistics changes rate interval (default: 1s)")
 }
 
 // validate parses and validates options passed by user and returns options ready for 'pgcenter report'.
@@ -89,11 +87,6 @@ func (opts options) validate() (report.Config, error) {
 		return report.Config{}, fmt.Errorf("report type is not specified, quit")
 	}
 
-	if opts.rate < time.Second {
-		fmt.Println("INFO: round rate interval to minimum allowed 1 second.")
-		opts.rate = time.Second
-	}
-
 	// Define report start/end interval.
 	tsStart, tsEnd, err := setReportInterval(opts.tsStart, opts.tsEnd)
 	if err != nil {
@@ -124,7 +117,6 @@ func (opts options) validate() (report.Config, error) {
 		FilterRE:      re,
 		RowLimit:      opts.rowLimit,
 		TruncLimit:    opts.strLimit,
-		Rate:          opts.rate,
 	}, nil
 }
 

cmd/report/report_test.go

@@ -13,11 +13,11 @@ func Test_options_validate(t *testing.T) {
 		opts  options
 		want  report.Config
 	}{
-		{valid: true, opts: options{showActivity: true, tsStart: "2021-01-01 12:00:00", tsEnd: "2021-01-01 13:00:00", rate: time.Second}},
-		{valid: true, opts: options{showActivity: true, tsStart: "2021-01-01 12:00:00", tsEnd: "2021-01-01 13:00:00", rate: 0}},
-		{valid: false, opts: options{tsStart: "2021-01-01 12:00:00", tsEnd: "2021-01-01 13:00:00", rate: time.Second}}, // no report type specified
-		{valid: false, opts: options{showActivity: true, tsStart: "2021-01-32", rate: time.Second}},                    // invalid report start timestamp
-		{valid: false, opts: options{showActivity: true, filter: `colname:"["`, rate: time.Second}},                    // invalid regexp
+		{valid: true, opts: options{showActivity: true, tsStart: "2021-01-01 12:00:00", tsEnd: "2021-01-01 13:00:00"}},
+		{valid: true, opts: options{showActivity: true, tsStart: "2021-01-01 12:00:00", tsEnd: "2021-01-01 13:00:00"}},
+		{valid: false, opts: options{tsStart: "2021-01-01 12:00:00", tsEnd: "2021-01-01 13:00:00"}}, // no report type specified
+		{valid: false, opts: options{showActivity: true, tsStart: "2021-01-32"}},                    // invalid report start timestamp
+		{valid: false, opts: options{showActivity: true, filter: `colname:"["`}},                    // invalid regexp
 	}
 
 	for _, tc := range testcases {

doc/Changelog

@@ -1,4 +1,5 @@
 pgcenter (master) unstable; urgency=low
+  * record/report: add support for millisecond resolution
   * add support of pg_stat_wal WAL usage statistics (Postgres 14)
   * add support of pg_stat_database sessions statistics (Postgres 14)
   * add support of pg_stat_progress_copy (Postgres 14)

record/recorder.go

@@ -132,8 +132,7 @@ func (c *tarRecorder) write(stats map[string]stat.PGresult) error {
 		}
 
 		now := time.Now()
-		filename := fmt.Sprintf("%s.%s.json", name, now.Format("20060102T150405"))
-		hdr := &tar.Header{Name: filename, Mode: 0644, Size: int64(len(data)), ModTime: now}
+		hdr := &tar.Header{Name: newFilenameString(now, name), Mode: 0644, Size: int64(len(data)), ModTime: now}
 		err = c.writer.WriteHeader(hdr)
 		if err != nil {
 			return err
@@ -158,3 +157,8 @@ func (c *tarRecorder) close() error {
 
 	return c.file.Close()
 }
+
+// newFilenameString returns a filename string with formatted timestamp and report name.
+func newFilenameString(ts time.Time, name string) string {
+	return fmt.Sprintf("%s.%s.json", name, ts.Format("20060102T150405.000"))
+}

record/recorder_test.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 )
 
 func Test_tarRecorder_open_close(t *testing.T) {
@@ -93,3 +94,21 @@ func Test_tarRecorder_write(t *testing.T) {
 	// Cleanup.
 	assert.NoError(t, os.Remove(filename))
 }
+
+func Test_newFilenameString(t *testing.T) {
+	testcases := []struct {
+		ts   time.Time
+		want string
+	}{
+		{ts: time.Date(2021, 06, 15, 12, 30, 15, 123456789, time.UTC), want: "example.20210615T123015.123.json"},
+		{ts: time.Date(2021, 06, 15, 12, 30, 15, 23456789, time.UTC), want: "example.20210615T123015.023.json"},
+		{ts: time.Date(2021, 06, 15, 12, 30, 15, 3456789, time.UTC), want: "example.20210615T123015.003.json"},
+		{ts: time.Date(2021, 06, 15, 12, 30, 15, 456789, time.UTC), want: "example.20210615T123015.000.json"},
+		{ts: time.Date(2021, 06, 15, 12, 30, 15, 789, time.UTC), want: "example.20210615T123015.000.json"},
+		{ts: time.Date(2021, 06, 15, 12, 30, 15, 0, time.UTC), want: "example.20210615T123015.000.json"},
+	}
+
+	for _, tc := range testcases {
+		assert.Equal(t, tc.want, newFilenameString(tc.ts, "example"))
+	}
+}

report/report.go

@@ -29,7 +29,6 @@ type Config struct {
 	FilterRE      *regexp.Regexp
 	RowLimit      int
 	TruncLimit    int
-	Rate          time.Duration
 }
 
 const (
@@ -234,18 +233,18 @@ func processData(app *app, v view.View, config Config, dataCh chan data, doneCh
 				continue
 			}
 
-			// Calculate time interval.
+			// Calculate interval and rate.
+			// Interval defines number of seconds used for calculating rate values.
+			// Rate is not used in calculations and printed on info header.
 			interval := d.ts.Sub(prevTs)
-			if config.Rate > interval {
-				_, err := fmt.Fprintf(
-					app.writer,
-					"WARNING: specified rate longer than stats snapshots interval, adjusting it to %s\n",
-					interval.String(),
-				)
-				if err != nil {
-					return err
-				}
-				config.Rate = interval
+			var itv int
+			var rate time.Duration
+			if interval < time.Second {
+				itv = 1
+				rate = interval
+			} else {
+				itv = int(interval / time.Second)
+				rate = time.Second
 			}
 
 			// When first data read, list of columns is known and it is possible to set up order.
@@ -258,7 +257,7 @@ func processData(app *app, v view.View, config Config, dataCh chan data, doneCh
 			}
 
 			// Calculate delta between current and previous stats snapshots.
-			diffStat, err := countDiff(d.res, prevStat, int(interval/config.Rate), v)
+			diffStat, err := countDiff(d.res, prevStat, itv, v)
 			if err != nil {
 				return err
 			}
@@ -273,7 +272,7 @@ func processData(app *app, v view.View, config Config, dataCh chan data, doneCh
 			}
 
 			// print the stats - calculated delta between previous and current stats snapshots
-			n, err := printStatSample(app.writer, &diffStat, v, config, d.ts)
+			n, err := printStatSample(app.writer, &diffStat, v, config, d.ts, rate)
 			if err != nil {
 				return err
 			}
@@ -308,7 +307,7 @@ func isFilenameOK(name string, report string) error {
 	s := strings.Split(name, ".")
 
 	// File name should be in the format: 'report_type.timestamp.json'.
-	if len(s) != 3 {
+	if len(s) != 4 {
 		return fmt.Errorf("bad file name format %s, skip", name)
 	}
 
@@ -324,13 +323,13 @@ func isFilenameOK(name string, report string) error {
 func isFilenameTimestampOK(name string, start, end time.Time) (time.Time, error) {
 	s := strings.Split(name, ".")
 
-	// File name should be in the format: 'report_type.timestamp.json'
-	if len(s) != 3 {
+	// File name should be in the format: 'report_type.timestamp.ms.json'
+	if len(s) != 4 {
 		return time.Time{}, fmt.Errorf("bad file name format %s, skip", name)
 	}
 
 	// Calculate timestamp when stats were recorded, parse timestamp considering it is in local timezone.
-	ts, err := time.ParseInLocation("20060102T150405", s[1], time.Now().Location())
+	ts, err := time.ParseInLocation("20060102T150405.000", s[1]+"."+s[2], time.Now().Location())
 	if err != nil {
 		return time.Time{}, err
 	}
@@ -386,13 +385,12 @@ func formatStatSample(d *stat.PGresult, view *view.View, c Config) {
 func printReportHeader(w io.Writer, c Config) error {
 	tmpl := "INFO: reading from %s\n" +
 		"INFO: report %s\n" +
-		"INFO: start from: %s, to: %s, with rate: %s\n"
+		"INFO: start from: %s, to: %s\n"
 	msg := fmt.Sprintf(tmpl,
 		c.InputFile,
 		c.ReportType,
 		c.TsStart.Format("2006-01-02 15:04:05 MST"),
 		c.TsEnd.Format("2006-01-02 15:04:05 MST"),
-		c.Rate.String(),
 	)
 
 	_, err := fmt.Fprint(w, msg)
@@ -408,27 +406,22 @@ func printStatHeader(w io.Writer, printedNum int, v view.View) (int, error) {
 		return printedNum, nil
 	}
 
-	_, err := fmt.Fprintf(w, "         ")
-	if err != nil {
-		return 0, err
-	}
-
 	for i, name := range v.Cols {
 		_, err := fmt.Fprintf(w, "\033[%d;%dm%-*s\033[0m", 37, 1, v.ColsWidth[i]+2, name)
 		if err != nil {
 			return 0, err
 		}
 	}
 
-	_, err = fmt.Fprintf(w, "\n")
+	_, err := fmt.Fprintf(w, "\n")
 	if err != nil {
 		return 0, err
 	}
 	return 0, nil
 }
 
 // printStatSample prints given stats
-func printStatSample(w io.Writer, res *stat.PGresult, view view.View, c Config, ts time.Time) (int, error) {
+func printStatSample(w io.Writer, res *stat.PGresult, view view.View, c Config, ts time.Time, interval time.Duration) (int, error) {
 	// print stats values
 	var printFirst = true // every first line in the snapshot should begin with timestamp when stats were taken
 	var linesPrinted int  // count lines printed per snapshot (for limiting purposes)
@@ -455,17 +448,13 @@ func printStatSample(w io.Writer, res *stat.PGresult, view view.View, c Config,
 
 		// print the row
 		if doPrint {
+			header := fmt.Sprintf("%s, rate: %s\n", ts.Format("2006/01/02 15:04:05"), interval.String())
 			if printFirst {
-				_, err := fmt.Fprintf(w, "%s ", ts.Format("15:04:05"))
+				_, err := fmt.Fprint(w, header)
 				if err != nil {
 					return 0, err
 				}
 				printFirst = false
-			} else {
-				_, err := fmt.Fprintf(w, "         ")
-				if err != nil {
-					return 0, err
-				}
 			}
 
 			for i := range res.Cols {