Realm stats
- The data below shows realm statistics. + The data below shows realm statistics and visualizations.
-
-
Statistics
-
- {{if $stats}}
-
- Loading chart...
-
- {{end}}
-
- {{if $userStats}}
-
- Loading chart...
-
- {{end}}
+
-
-
Export
-
-
-
Realm per-user stats - (CSV/JSON) -
+ -
-
Data
-
- {{if $stats}}
-
-
-
- {{end}}
-
- {{if or $stats $userStats}}
-
Date | -Issued | -Claimed | -
---|---|---|
{{$stat.Date.Format "2006-01-02"}} | -{{$stat.CodesIssued}} | -{{$stat.CodesClaimed}} | -
- This data is refreshed every 5 minutes.
-
- {{else}}
- No codes have been issued in this realm.
- {{end}} + diff --git a/docs/api.md b/docs/api.md index f43c41b13..ca7594987 100644 --- a/docs/api.md +++ b/docs/api.md @@ -195,6 +195,17 @@ Request a verification code to be issued. Accepts [optional] symptom date and te the padding. * `uuid` is optional as request input. The server will generate a uuid on response if omitted. * This is a handle which allows the issuer to track status of the issued verification code. +* `externalIssuerID` is an optional field supplied by the API caller to uniquely + identify the entity making this request. This is useful where callers are + using a single API key behind an ERP, or when callers are using the + verification server as an API with a different authentication system. This + field is optional. + + * The information provided is stored exactly as-is. If the identifier is + uniquely identifying PII (such as an email address, employee ID, SSN, etc), + the caller should apply a cryptographic hash before sending that data. **The + system does not sanitize or encrypt these external IDs, it is the caller's + responsibility to do so.** **IssueCodeResponse** diff --git a/internal/icsv/icsv.go b/internal/icsv/icsv.go new file mode 100644 index 000000000..e25809c3e --- /dev/null +++ b/internal/icsv/icsv.go @@ -0,0 +1,22 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package icsv defines an interface for things that can export as CSV. +package icsv + +// Marshaler is an interface for items that can convert to CSV. +type Marshaler interface { + // MarshalCSV produces CSV. + MarshalCSV() ([]byte, error) +} diff --git a/internal/routes/server.go b/internal/routes/server.go index 357b0946e..26513130b 100644 --- a/internal/routes/server.go +++ b/internal/routes/server.go @@ -390,10 +390,9 @@ func realmadminRoutes(r *mux.Router, c *realmadmin.Controller) { r.Handle("/settings", c.HandleSettings()).Methods("GET", "POST") r.Handle("/settings/enable-express", c.HandleEnableExpress()).Methods("POST") r.Handle("/settings/disable-express", c.HandleDisableExpress()).Methods("POST") - r.Handle("/stats", c.HandleShow(realmadmin.HTML)).Methods("GET") - r.Handle("/stats.json", c.HandleShow(realmadmin.JSON)).Methods("GET") - r.Handle("/stats.csv", c.HandleShow(realmadmin.CSV)).Methods("GET") - r.Handle("/stats/{date}", c.HandleStats()).Methods("GET") + r.Handle("/stats", c.HandleShow()).Methods("GET") + r.Handle("/stats.csv", c.HandleShow()).Methods("GET") + r.Handle("/stats.json", c.HandleShow()).Methods("GET") r.Handle("/events", c.HandleEvents()).Methods("GET") } diff --git a/internal/routes/server_test.go b/internal/routes/server_test.go index 262d3224f..791dcf8ab 100644 --- a/internal/routes/server_test.go +++ b/internal/routes/server_test.go @@ -228,9 +228,6 @@ func TestRoutes_realmadminRoutes(t *testing.T) { { req: httptest.NewRequest("GET", "/stats.csv", nil), }, - { - req: httptest.NewRequest("GET", "/stats/20201112", nil), - }, { req: httptest.NewRequest("GET", "/events", nil), }, diff --git a/pkg/api/api.go b/pkg/api/api.go index 723c0c843..15087b871 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -183,6 +183,19 @@ type IssueCodeRequest struct { // Optional: UUID is a handle which allows the issuer to track status // of the issued verification code. If omitted the server will generate the UUID. UUID string `json:"uuid"` + + // ExternalIssuerID is a optional information supplied by the API caller to + // uniquely identify the entity making this request. This is useful where + // callers are using a single API key behind an ERP, or when callers are using + // the verification server as an API with a different authentication system. + // This field is optional. + + // The information provided is stored exactly as-is. If the identifier is + // uniquely identifying PII (such as an email address, employee ID, SSN, etc), + // the caller should apply a cryptographic hash before sending that data. The + // system does not sanitize or encrypt these external IDs, it is the caller's + // responsibility to do so. + ExternalIssuerID string `json:"externalIssuerID"` } // IssueCodeResponse defines the response type for IssueCodeRequest. diff --git a/pkg/clients/apis.go b/pkg/clients/apis.go index 53a55b38d..e7b5434ec 100644 --- a/pkg/clients/apis.go +++ b/pkg/clients/apis.go @@ -26,13 +26,8 @@ import ( // IssueCode uses the ADMIN API to issue a verification code. // Currently does not accept the SMS param. -func IssueCode(ctx context.Context, hostname string, apiKey, testType, symptomDate string, tzMinOffset int, timeout time.Duration) (*api.IssueCodeRequest, *api.IssueCodeResponse, error) { +func IssueCode(ctx context.Context, hostname, apiKey string, request *api.IssueCodeRequest) (*api.IssueCodeResponse, error) { url := hostname + "/api/issue" - request := api.IssueCodeRequest{ - TestType: testType, - SymptomDate: symptomDate, - TZOffset: float32(tzMinOffset), - } client := &http.Client{ Timeout: timeout, } @@ -43,9 +38,9 @@ func IssueCode(ctx context.Context, hostname string, apiKey, testType, symptomDa headers.Add("X-API-Key", apiKey) if err := jsonclient.MakeRequest(ctx, client, url, headers, request, &response); err != nil { - return &request, nil, err + return nil, err } - return &request, &response, nil + return &response, nil } // CheckCodeStatus uses the ADMIN API to retrieve the status of an OTP code. diff --git a/pkg/clients/e2e.go b/pkg/clients/e2e.go index eca435804..9c5e57a66 100644 --- a/pkg/clients/e2e.go +++ b/pkg/clients/e2e.go @@ -57,6 +57,7 @@ func RunEndToEnd(ctx context.Context, config *config.E2ETestConfig) error { iterations++ } symptomDate := time.Now().UTC().Add(-48 * time.Hour).Format("2006-01-02") + adminID := "" revisionToken := "" now := time.Now().UTC() @@ -89,8 +90,16 @@ func RunEndToEnd(ctx context.Context, config *config.E2ETestConfig) error { } code, err := func() (*api.IssueCodeResponse, error) { defer recordLatency(ctx, time.Now(), "/api/issue") + // Issue the verification code. - codeRequest, code, err := IssueCode(ctx, config.VerificationAdminAPIServer, config.VerificationAdminAPIKey, testType, symptomDate, 0, timeout) + codeRequest := &api.IssueCodeRequest{ + TestType: testType, + SymptomDate: symptomDate, + TZOffset: 0, + ExternalIssuerID: adminID, + } + + code, err := IssueCode(ctx, config.VerificationAdminAPIServer, config.VerificationAdminAPIKey, codeRequest) if err != nil { result = observability.ResultNotOK() return nil, fmt.Errorf("error issuing verification code: %w", err) diff --git a/pkg/controller/issueapi/issue.go b/pkg/controller/issueapi/issue.go index 5ef8743ed..7a8245c49 100644 --- a/pkg/controller/issueapi/issue.go +++ b/pkg/controller/issueapi/issue.go @@ -304,10 +304,12 @@ func (c *Controller) HandleIssue() http.Handler { SymptomDate: parsedDates[0], TestDate: parsedDates[1], MaxSymptomAge: c.config.GetAllowedSymptomAge(), - IssuingUser: user, - IssuingApp: authApp, RealmID: realm.ID, UUID: rUUID, + + IssuingUser: user, + IssuingApp: authApp, + IssuingExternalID: request.ExternalIssuerID, } code, longCode, uuid, err := codeRequest.Issue(ctx, c.config.GetCollisionRetryCount()) diff --git a/pkg/controller/middleware/i18n.go b/pkg/controller/middleware/i18n.go index b311f4dc4..0ca28a860 100644 --- a/pkg/controller/middleware/i18n.go +++ b/pkg/controller/middleware/i18n.go @@ -37,15 +37,13 @@ func ProcessLocale(locales *i18n.LocaleMap) mux.MiddlewareFunc { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - // TODO(sethvargo): extract from session/cookie as well param := r.URL.Query().Get(QueryKeyLanguage) - cookie := "" header := r.Header.Get(HeaderAcceptLanguage) // Find the "best" language from the given parameters. They are in // priority order. m := controller.TemplateMapFromContext(ctx) - m["locale"] = locales.Lookup(param, cookie, header) + m["locale"] = locales.Lookup(param, header) next.ServeHTTP(w, r) }) diff --git a/pkg/controller/modeler/modeler_test.go b/pkg/controller/modeler/modeler_test.go index bd50f93e6..8bbf6baae 100644 --- a/pkg/controller/modeler/modeler_test.go +++ b/pkg/controller/modeler/modeler_test.go @@ -89,7 +89,7 @@ func TestRebuildModel(t *testing.T) { line := []uint{50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50} for _, y := range line { if err := db.RawDB(). - Create(&database.RealmStats{ + Create(&database.RealmStat{ Date: nextDate(), RealmID: realm.ID, CodesIssued: y, @@ -124,7 +124,7 @@ func TestRebuildModel(t *testing.T) { line := []uint{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20} for _, y := range line { if err := db.RawDB(). - Create(&database.RealmStats{ + Create(&database.RealmStat{ Date: nextDate(), RealmID: realm.ID, CodesIssued: y, @@ -159,7 +159,7 @@ func TestRebuildModel(t *testing.T) { line := []uint{1, 26, 61, 13, 19, 50, 9, 20, 91, 187, 39, 4, 2, 5, 1} for _, y := range line { if err := db.RawDB(). - Create(&database.RealmStats{ + Create(&database.RealmStat{ Date: nextDate(), RealmID: realm.ID, CodesIssued: y, @@ -194,7 +194,7 @@ func TestRebuildModel(t *testing.T) { line := []uint{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} for _, y := range line { if err := db.RawDB(). - Create(&database.RealmStats{ + Create(&database.RealmStat{ Date: nextDate(), RealmID: realm.ID, CodesIssued: y, @@ -233,7 +233,7 @@ func TestRebuildModel(t *testing.T) { for _, y := range line { if err := db.RawDB(). - Create(&database.RealmStats{ + Create(&database.RealmStat{ Date: nextDate(), RealmID: realm.ID, CodesIssued: y, diff --git a/pkg/controller/realmadmin/show.go b/pkg/controller/realmadmin/show.go index fe6cbf989..5ddf428cb 100644 --- a/pkg/controller/realmadmin/show.go +++ b/pkg/controller/realmadmin/show.go @@ -16,36 +16,97 @@ package realmadmin import ( "context" - "encoding/csv" + "encoding/json" + "fmt" "net/http" "strconv" + "strings" "time" + "github.com/google/exposure-notifications-verification-server/internal/icsv" "github.com/google/exposure-notifications-verification-server/pkg/cache" "github.com/google/exposure-notifications-verification-server/pkg/controller" "github.com/google/exposure-notifications-verification-server/pkg/database" ) -var cacheTimeout = 5 * time.Minute +const cacheTimeout = 30 * time.Minute -// ResultType specifies which type of renderer you want. -type ResultType int +func (c *Controller) HandleShow() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() -const ( - HTML ResultType = iota - JSON - CSV -) + realm := controller.RealmFromContext(ctx) + if realm == nil { + controller.MissingRealm(w, r, c.h) + return + } + + now := time.Now().UTC() + past := now.Add(-30 * 24 * time.Hour) + + pth := r.URL.Path + switch { + case strings.HasSuffix(pth, ".csv"): + var filename string + var stats icsv.Marshaler + var err error + + nowFormatted := now.Format("20060102150405") + + switch r.URL.Query().Get("scope") { + case "external": + filename = fmt.Sprintf("%s-external-issuer-stats.csv", nowFormatted) + stats, err = c.getExternalIssuerStats(ctx, realm, now, past) + case "user": + filename = fmt.Sprintf("%s-user-stats.csv", nowFormatted) + stats, err = c.getUserStats(ctx, realm, now, past) + default: + filename = fmt.Sprintf("%s-realm-stats.csv", nowFormatted) + stats, err = c.getRealmStats(ctx, realm, now, past) + } + + if err != nil { + controller.InternalError(w, r, c.h, err) + return + } + + c.h.RenderCSV(w, http.StatusOK, filename, stats) + case strings.HasSuffix(pth, ".json"): + var stats json.Marshaler + var err error + + switch r.URL.Query().Get("scope") { + case "external": + stats, err = c.getExternalIssuerStats(ctx, realm, now, past) + case "user": + stats, err = c.getUserStats(ctx, realm, now, past) + default: + stats, err = c.getRealmStats(ctx, realm, now, past) + } + + if err != nil { + controller.InternalError(w, r, c.h, err) + return + } + + c.h.RenderJSON(w, http.StatusOK, stats) + default: + // Fallback to HTML + c.renderHTML(ctx, w, realm) + return + } + }) +} -// wantUser returns true if we want per-user requests. -func wantUser(r *http.Request) bool { - _, has := r.URL.Query()["user"] - return has +func (c *Controller) renderHTML(ctx context.Context, w http.ResponseWriter, realm *database.Realm) { + m := controller.TemplateMapFromContext(ctx) + m.Title("Realm stats") + c.h.RenderHTML(w, "realmadmin/show", m) } // getRealmStats returns the realm stats for a given date range. -func (c *Controller) getRealmStats(ctx context.Context, realm *database.Realm, now, past time.Time) ([]*database.RealmStats, error) { - var stats []*database.RealmStats +func (c *Controller) getRealmStats(ctx context.Context, realm *database.Realm, now, past time.Time) (database.RealmStats, error) { + var stats database.RealmStats cacheKey := &cache.Key{ Namespace: "stats:realm", Key: strconv.FormatUint(uint64(realm.ID), 10), @@ -55,158 +116,35 @@ func (c *Controller) getRealmStats(ctx context.Context, realm *database.Realm, n }); err != nil { return nil, err } - return stats, nil } // getUserStats gets the per-user realm stats for a given date range. -func (c *Controller) getUserStats(ctx context.Context, realm *database.Realm, now, past time.Time) ([]*database.RealmUserStats, error) { - var userStats []*database.RealmUserStats +func (c *Controller) getUserStats(ctx context.Context, realm *database.Realm, now, past time.Time) (database.RealmUserStats, error) { + var userStats database.RealmUserStats cacheKey := &cache.Key{ Namespace: "stats:realm:per_user", Key: strconv.FormatUint(uint64(realm.ID), 10), } if err := c.cacher.Fetch(ctx, cacheKey, &userStats, cacheTimeout, func() (interface{}, error) { - return realm.CodesPerUser(c.db, past, now) + return realm.UserStats(c.db, past, now) }); err != nil { return nil, err } return userStats, nil } -func (c *Controller) HandleShow(result ResultType) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - - now := time.Now().UTC() - past := now.Add(-30 * 24 * time.Hour) - - realm := controller.RealmFromContext(ctx) - if realm == nil { - controller.MissingRealm(w, r, c.h) - return - } - - // Get the realm stats. - stats, err := c.getRealmStats(ctx, realm, now, past) - if err != nil { - controller.InternalError(w, r, c.h, err) - return - } - - // Also get the per-user stats. - userStats, err := c.getUserStats(ctx, realm, now, past) - if err != nil { - controller.InternalError(w, r, c.h, err) - return - } - - switch result { - case CSV: - err = c.renderCSV(r, w, stats, userStats) - case JSON: - err = c.renderJSON(r, w, stats, userStats) - case HTML: - err = c.renderHTML(ctx, w, realm, stats, userStats) - } - if err != nil { - controller.InternalError(w, r, c.h, err) - return - } - }) -} - -// formatData formats a slice of RealmUserStats into a format more conducive -// to charting in Javascript. -func formatData(userStats []*database.RealmUserStats) ([]string, [][]interface{}) { - // We need to format the per-user-per-day data properly for the charts. - // Create some LUTs to make this easier. - nameLUT := make(map[string]int) - datesLUT := make(map[time.Time]int) - for _, stat := range userStats { - if _, ok := nameLUT[stat.Name]; !ok { - nameLUT[stat.Name] = len(nameLUT) - } - if _, ok := datesLUT[stat.Date]; !ok { - datesLUT[stat.Date] = len(datesLUT) - } - } - - // Figure out the names. - names := make([]string, len(nameLUT)) - for name, i := range nameLUT { - names[i] = name - } - - // And combine up the data we want to send as well. - data := make([][]interface{}, len(datesLUT)) - for date, i := range datesLUT { - data[i] = make([]interface{}, len(names)+1) - data[i][0] = date.Format("Jan 2 2006") - } - for _, stat := range userStats { - i := datesLUT[stat.Date] - data[i][nameLUT[stat.Name]+1] = stat.CodesIssued - } - - // Now, we need to format the data properly. - return names, data -} - -func (c *Controller) renderHTML(ctx context.Context, w http.ResponseWriter, realm *database.Realm, stats []*database.RealmStats, userStats []*database.RealmUserStats) error { - names, format := formatData(userStats) - m := controller.TemplateMapFromContext(ctx) - m.Title("Realm stats") - m["user"] = realm - m["stats"] = stats - m["names"] = names - m["userStats"] = format - c.h.RenderHTML(w, "realmadmin/show", m) - - return nil -} - -// renderCSV renders a CSV response. -func (c *Controller) renderCSV(r *http.Request, w http.ResponseWriter, stats []*database.RealmStats, userStats []*database.RealmUserStats) error { - wr := csv.NewWriter(w) - defer wr.Flush() - - // Check if we want the realm stats or the per-user stats. We - // default to realm stats. - if wantUser(r) { - if err := wr.Write(database.RealmUserStatsCSVHeader); err != nil { - return err - } - - for _, u := range userStats { - if err := wr.Write(u.CSV()); err != nil { - return err - } - } - } else { - if err := wr.Write(database.RealmStatsCSVHeader); err != nil { - return err - } - - for _, s := range stats { - if err := wr.Write(s.CSV()); err != nil { - return err - } - } +// getExternalIssuerStats gets the external issuer stats for a given date range. +func (c *Controller) getExternalIssuerStats(ctx context.Context, realm *database.Realm, now, past time.Time) (database.ExternalIssuerStats, error) { + var stats database.ExternalIssuerStats + cacheKey := &cache.Key{ + Namespace: "stats:realm:per_external_issuer", + Key: strconv.FormatUint(uint64(realm.ID), 10), } - - w.Header().Set("Content-Type", "text/csv") - w.Header().Set("Content-Disposition", "attachment;filename=stats.csv") - return nil -} - -// renderJSON renders a JSON response. -func (c *Controller) renderJSON(r *http.Request, w http.ResponseWriter, stats []*database.RealmStats, userStats []*database.RealmUserStats) error { - if wantUser(r) { - c.h.RenderJSON(w, http.StatusOK, userStats) - } else { - c.h.RenderJSON(w, http.StatusOK, stats) + if err := c.cacher.Fetch(ctx, cacheKey, &stats, cacheTimeout, func() (interface{}, error) { + return realm.ExternalIssuerStats(c.db, past, now) + }); err != nil { + return nil, err } - w.Header().Set("Content-Disposition", "attachment;filename=stats.json") - return nil + return stats, nil } diff --git a/pkg/controller/realmadmin/show_test.go b/pkg/controller/realmadmin/show_test.go deleted file mode 100644 index 59773e450..000000000 --- a/pkg/controller/realmadmin/show_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package realmadmin - -import ( - "reflect" - "sort" - "testing" - "time" - - "github.com/google/exposure-notifications-server/pkg/timeutils" - "github.com/google/exposure-notifications-verification-server/pkg/database" -) - -func TestFormatStats(t *testing.T) { - now := timeutils.Midnight(time.Now()) - yesterday := timeutils.Midnight(now.Add(-24 * time.Hour)) - tests := []struct { - data []*database.RealmUserStats - names []string - numDays int - }{ - {[]*database.RealmUserStats{}, []string{}, 0}, - { - []*database.RealmUserStats{ - {UserID: 1, Name: "Rocky", CodesIssued: 10, Date: now}, - {UserID: 1, Name: "Bullwinkle", CodesIssued: 1, Date: now}, - }, - []string{"Rocky", "Bullwinkle"}, - 1, - }, - { - []*database.RealmUserStats{ - {UserID: 1, Name: "Rocky", CodesIssued: 10, Date: yesterday}, - {UserID: 1, Name: "Rocky", CodesIssued: 10, Date: now}, - }, - []string{"Rocky"}, - 2, - }, - } - - for i, test := range tests { - names, format := formatData(test.data) - sort.Strings(test.names) - sort.Strings(names) - if !reflect.DeepEqual(test.names, names) { - t.Errorf("[%d] %v != %v", i, names, test.names) - } - if len(format) != test.numDays { - t.Errorf("[%d] len(format) = %d, expected %d", i, len(format), test.numDays) - } - for _, f := range format { - if len(f) != len(test.names)+1 { - t.Errorf("[%d] len(codesIssued) = %d, expected %d", i, len(f), len(test.names)+1) - } - } - } -} diff --git a/pkg/controller/realmadmin/stats.go b/pkg/controller/realmadmin/stats.go deleted file mode 100644 index ccf6bc672..000000000 --- a/pkg/controller/realmadmin/stats.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package realmadmin - -import ( - "net/http" - "time" - - "github.com/google/exposure-notifications-verification-server/pkg/cache" - "github.com/google/exposure-notifications-verification-server/pkg/controller" - "github.com/google/exposure-notifications-verification-server/pkg/database" - "github.com/gorilla/mux" -) - -// HandleStats returns an http handler for sending JSON encoded per-user stats. -func (c *Controller) HandleStats() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - vars := mux.Vars(r) - - realm := controller.RealmFromContext(ctx) - if realm == nil { - controller.MissingRealm(w, r, c.h) - return - } - - date, err := time.Parse("2006-01-02", vars["date"]) - if err != nil { - c.h.RenderJSON(w, http.StatusBadRequest, err) - return - } - - // Also get the per-user stats. - var stats []*database.RealmUserStats - cacheKey := &cache.Key{ - Namespace: "stats:realm:per_user", - Key: vars["date"], - } - if err := c.cacher.Fetch(ctx, cacheKey, &stats, cacheTimeout, func() (interface{}, error) { - return realm.CodesPerUser(c.db, date, date) - }); err != nil { - controller.InternalError(w, r, c.h, err) - return - } - - c.h.RenderJSON(w, http.StatusOK, stats) - }) -} diff --git a/pkg/database/external_issuer_stats.go b/pkg/database/external_issuer_stats.go new file mode 100644 index 000000000..beed114cb --- /dev/null +++ b/pkg/database/external_issuer_stats.go @@ -0,0 +1,155 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "sort" + "strconv" + "time" + + "github.com/google/exposure-notifications-verification-server/internal/icsv" +) + +var _ icsv.Marshaler = (ExternalIssuerStats)(nil) + +// ExternalIssuerStats is a collection of external issuer stats. +type ExternalIssuerStats []*ExternalIssuerStat + +// ExternalIssuerStat represents statistics related to a user in the database. +type ExternalIssuerStat struct { + Date time.Time `gorm:"column:date; type:date;"` + RealmID uint `gorm:"column:realm_id; type:int"` + IssuerID string `gorm:"column:issuer_id; type:varchar(255)"` + CodesIssued uint `gorm:"column:codes_issued; type:int;"` +} + +// MarshalCSV returns bytes in CSV format. +func (s ExternalIssuerStats) MarshalCSV() ([]byte, error) { + // Do nothing if there's no records + if len(s) == 0 { + return nil, nil + } + + var b bytes.Buffer + w := csv.NewWriter(&b) + + if err := w.Write([]string{"date", "realm_id", "issuer_id", "codes_issued"}); err != nil { + return nil, fmt.Errorf("failed to write CSV header: %w", err) + } + + for i, stat := range s { + if err := w.Write([]string{ + stat.Date.Format("2006-01-02"), + strconv.FormatUint(uint64(stat.RealmID), 10), + stat.IssuerID, + strconv.FormatUint(uint64(stat.CodesIssued), 10), + }); err != nil { + return nil, fmt.Errorf("failed to write CSV entry %d: %w", i, err) + } + } + + w.Flush() + if err := w.Error(); err != nil { + return nil, fmt.Errorf("failed to create CSV: %w", err) + } + + return b.Bytes(), nil +} + +type jsonExternalIssuerStat struct { + RealmID uint `json:"realm_id"` + Stats []*jsonExternalIssuerStatStats `json:"statistics"` +} + +type jsonExternalIssuerStatStats struct { + Date time.Time `json:"date"` + IssuerData []*jsonExternalIssuerStatIssuerData `json:"issuer_data"` +} + +type jsonExternalIssuerStatIssuerData struct { + IssuerID string `json:"issuer_id"` + CodesIssued uint `json:"codes_issued"` +} + +// MarshalJSON is a custom JSON marshaller. +func (s ExternalIssuerStats) MarshalJSON() ([]byte, error) { + // Do nothing if there's no records + if len(s) == 0 { + return json.Marshal(struct{}{}) + } + + m := make(map[time.Time][]*jsonExternalIssuerStatIssuerData) + for _, stat := range s { + if m[stat.Date] == nil { + m[stat.Date] = make([]*jsonExternalIssuerStatIssuerData, 0, 8) + } + + m[stat.Date] = append(m[stat.Date], &jsonExternalIssuerStatIssuerData{ + IssuerID: stat.IssuerID, + CodesIssued: stat.CodesIssued, + }) + } + + stats := make([]*jsonExternalIssuerStatStats, 0, len(m)) + for k, v := range m { + stats = append(stats, &jsonExternalIssuerStatStats{ + Date: k, + IssuerData: v, + }) + } + + // Sort in descending order. + sort.Slice(stats, func(i, j int) bool { + return stats[i].Date.After(stats[j].Date) + }) + + var result jsonExternalIssuerStat + result.RealmID = s[0].RealmID + result.Stats = stats + + b, err := json.Marshal(result) + if err != nil { + return nil, fmt.Errorf("failed to marshal json: %w", err) + } + return b, nil +} + +func (s *ExternalIssuerStats) UnmarshalJSON(b []byte) error { + if len(b) == 0 { + return nil + } + + var result jsonExternalIssuerStat + if err := json.Unmarshal(b, &result); err != nil { + return err + } + + for _, stat := range result.Stats { + for _, r := range stat.IssuerData { + *s = append(*s, &ExternalIssuerStat{ + Date: stat.Date, + RealmID: result.RealmID, + IssuerID: r.IssuerID, + CodesIssued: r.CodesIssued, + }) + } + } + + return nil +} diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go index d02014524..c84eba630 100644 --- a/pkg/database/migrations.go +++ b/pkg/database/migrations.go @@ -878,7 +878,7 @@ func (db *Database) getMigrations(ctx context.Context) *gormigrate.Gormigrate { ID: "00036-AddRealmStats", Migrate: func(tx *gorm.DB) error { logger.Debugw("db migrations: adding realm stats") - if err := tx.AutoMigrate(&RealmStats{}).Error; err != nil { + if err := tx.AutoMigrate(&RealmStat{}).Error; err != nil { return err } statements := []string{ @@ -893,7 +893,7 @@ func (db *Database) getMigrations(ctx context.Context) *gormigrate.Gormigrate { return nil }, Rollback: func(tx *gorm.DB) error { - if err := tx.DropTable(&RealmStats{}).Error; err != nil { + if err := tx.DropTable(&RealmStat{}).Error; err != nil { return err } return nil @@ -1698,12 +1698,23 @@ func (db *Database) getMigrations(ctx context.Context) *gormigrate.Gormigrate { { ID: "00068-EnablePGAudit", Migrate: func(tx *gorm.DB) error { - if err := tx.Exec(`CREATE EXTENSION pgaudit`).Error; err != nil { - logger.Warnw("failed to enable pgaudit", "error", err) - } + _ = tx.Exec(`CREATE EXTENSION pgaudit`).Error return nil }, }, + { + ID: "00069-AddExternalIssuerStats", + Migrate: func(tx *gorm.DB) error { + if err := tx.AutoMigrate(&ExternalIssuerStat{}).Error; err != nil { + return err + } + sql := `CREATE UNIQUE INDEX IF NOT EXISTS idx_external_issuer_stats_date_issuer_id_realm_id ON external_issuer_stats (date, issuer_id, realm_id)` + return tx.Exec(sql).Error + }, + Rollback: func(tx *gorm.DB) error { + return tx.DropTable(&ExternalIssuerStat{}).Error + }, + }, }) } diff --git a/pkg/database/realm.go b/pkg/database/realm.go index aad1d7407..832512f96 100644 --- a/pkg/database/realm.go +++ b/pkg/database/realm.go @@ -1255,25 +1255,120 @@ func (r *Realm) DestroySigningKeyVersion(ctx context.Context, db *Database, id i // Stats returns the usage statistics for this realm. If no stats exist, // returns an empty array. -func (r *Realm) Stats(db *Database, start, stop time.Time) ([]*RealmStats, error) { - var stats []*RealmStats - +func (r *Realm) Stats(db *Database, start, stop time.Time) (RealmStats, error) { start = timeutils.Midnight(start) stop = timeutils.Midnight(stop) + if start.After(stop) { + return nil, ErrBadDateRange + } - if err := db.db. - Model(&RealmStats{}). - Where("realm_id = ?", r.ID). - Where("(date >= ? AND date <= ?)", start, stop). - Order("date DESC"). - Find(&stats). - Error; err != nil { + sql := ` + SELECT + d.date AS date, + $1 AS realm_id, + COALESCE(s.codes_issued, 0) AS codes_issued, + COALESCE(s.codes_claimed, 0) AS codes_claimed + FROM ( + SELECT date::date FROM generate_series($2, $3, '1 day'::interval) date + ) d + LEFT JOIN realm_stats s ON s.realm_id = $1 AND s.date = d.date + ORDER BY date DESC` + + var stats []*RealmStat + if err := db.db.Raw(sql, r.ID, start, stop).Scan(&stats).Error; err != nil { if IsNotFound(err) { return stats, nil } return nil, err } + return stats, nil +} + +// ExternalIssuerStats returns the external issuer stats for this realm. If no +// stats exist, returns an empty slice. +func (r *Realm) ExternalIssuerStats(db *Database, start, stop time.Time) (ExternalIssuerStats, error) { + start = timeutils.UTCMidnight(start) + stop = timeutils.UTCMidnight(stop) + if start.After(stop) { + return nil, ErrBadDateRange + } + + // Pull the stats by generating the full date range and full list of external + // issuers that generated data in that range, then join on stats. This will + // ensure we have a full list (with values of 0 where appropriate) to ensure + // continuity in graphs. + sql := ` + SELECT + d.date AS date, + $1 AS realm_id, + d.issuer_id AS issuer_id, + COALESCE(s.codes_issued, 0) AS codes_issued + FROM ( + SELECT + d.date AS date, + i.issuer_id AS issuer_id + FROM generate_series($2, $3, '1 day'::interval) d + CROSS JOIN ( + SELECT DISTINCT(issuer_id) + FROM external_issuer_stats + WHERE realm_id = $1 AND date >= $2 AND date <= $3 + ) AS i + ) d + LEFT JOIN external_issuer_stats s ON s.realm_id = $1 AND s.issuer_id = d.issuer_id AND s.date = d.date + ORDER BY date DESC, issuer_id` + + var stats []*ExternalIssuerStat + if err := db.db.Raw(sql, r.ID, start, stop).Scan(&stats).Error; err != nil { + if IsNotFound(err) { + return stats, nil + } + return nil, err + } + return stats, nil +} + +// UserStats returns a set of UserStats for a given date range. +func (r *Realm) UserStats(db *Database, start, stop time.Time) (RealmUserStats, error) { + start = timeutils.UTCMidnight(start) + stop = timeutils.UTCMidnight(stop) + + if start.After(stop) { + return nil, ErrBadDateRange + } + // Pull the stats by generating the full date range and full list of users + // that generated data in that range, then join on stats. This will ensure we + // have a full list (with values of 0 where appropriate) to ensure continuity + // in graphs. + sql := ` + SELECT + d.date AS date, + $1 AS realm_id, + d.user_id AS user_id, + u.name AS name, + COALESCE(s.codes_issued, 0) AS codes_issued + FROM ( + SELECT + d.date AS date, + i.user_id AS user_id + FROM generate_series($2, $3, '1 day'::interval) d + CROSS JOIN ( + SELECT DISTINCT(user_id) + FROM user_stats + WHERE realm_id = $1 AND date >= $2 AND date <= $3 + ) AS i + ) d + LEFT JOIN user_stats s ON s.realm_id = $1 AND s.user_id = d.user_id AND s.date = d.date + LEFT JOIN users u ON u.id = d.user_id + ORDER BY date DESC, user_id` + + var stats []*RealmUserStat + if err := db.db.Raw(sql, r.ID, start, stop).Scan(&stats).Error; err != nil { + if IsNotFound(err) { + return stats, nil + } + return nil, err + } return stats, nil } @@ -1333,51 +1428,3 @@ func ToCIDRList(s string) ([]string, error) { sort.Strings(cidrs) return cidrs, nil } - -// RealmUserStats carries the per-user-per-day-per-realm Codes issued. -// This is a structure joined from multiple tables in the DB. -type RealmUserStats struct { - UserID uint `json:"user_id"` - Name string `json:"name"` - CodesIssued uint `json:"codes_issued"` - Date time.Time `json:"date"` -} - -// RealmUserStatsCSVHeader is a header for CSV stats -var RealmUserStatsCSVHeader = []string{"User ID", "Name", "Codes Issued", "Date"} - -// CSV returns a slice of the data from a RealmUserStats for CSV writing. -func (s *RealmUserStats) CSV() []string { - return []string{ - fmt.Sprintf("%d", s.UserID), - s.Name, - fmt.Sprintf("%d", s.CodesIssued), - s.Date.Format("2006-01-02"), - } -} - -// CodesPerUser returns a set of UserStats for a given date range. -func (r *Realm) CodesPerUser(db *Database, start, stop time.Time) ([]*RealmUserStats, error) { - start = timeutils.UTCMidnight(start) - stop = timeutils.UTCMidnight(stop) - if start.After(stop) { - return nil, ErrBadDateRange - } - - var stats []*RealmUserStats - if err := db.db. - Model(&UserStats{}). - Select("users.id, users.name, codes_issued, date"). - Where("realm_id = ?", r.ID). - Where("date >= ? AND date <= ?", start, stop). - Joins("INNNER JOIN users ON users.id = user_id"). - Order("date DESC"). - Scan(&stats). - Error; err != nil { - if IsNotFound(err) { - return stats, nil - } - return nil, err - } - return stats, nil -} diff --git a/pkg/database/realm_stats.go b/pkg/database/realm_stats.go index 07e8870d6..8f5f85eda 100644 --- a/pkg/database/realm_stats.go +++ b/pkg/database/realm_stats.go @@ -15,34 +15,131 @@ package database import ( + "bytes" + "encoding/csv" + "encoding/json" "fmt" + "sort" + "strconv" "time" + + "github.com/google/exposure-notifications-verification-server/internal/icsv" ) -// RealmStats represents statistics related to a user in the database. -type RealmStats struct { - Date time.Time `gorm:"date; not null"` - RealmID uint `gorm:"realm_id; not null"` - CodesIssued uint `gorm:"codes_issued; default: 0"` - CodesClaimed uint `gorm:"codes_claimed; default: 0"` +var _ icsv.Marshaler = (RealmStats)(nil) + +// RealmStats represents a logical collection of stats of a realm. +type RealmStats []*RealmStat + +// RealmStat represents statistics related to a user in the database. +type RealmStat struct { + Date time.Time `gorm:"date; not null;"` + RealmID uint `gorm:"realm_id; not null;"` + CodesIssued uint `gorm:"codes_issued; default:0;"` + CodesClaimed uint `gorm:"codes_claimed; default:0;"` } -// RealmStatsCSVHeader is a header for CSV files for RealmStats. -var RealmStatsCSVHeader = []string{"Date", "Realm ID", "Codes Issued", "Codes Claimed"} +// MarshalCSV returns bytes in CSV format. +func (s RealmStats) MarshalCSV() ([]byte, error) { + // Do nothing if there's no records + if len(s) == 0 { + return nil, nil + } + + var b bytes.Buffer + w := csv.NewWriter(&b) + + if err := w.Write([]string{"date", "codes_issued", "codes_claimed"}); err != nil { + return nil, fmt.Errorf("failed to write CSV header: %w", err) + } + + for i, stat := range s { + if err := w.Write([]string{ + stat.Date.Format("2006-01-02"), + strconv.FormatUint(uint64(stat.CodesIssued), 10), + strconv.FormatUint(uint64(stat.CodesClaimed), 10), + }); err != nil { + return nil, fmt.Errorf("failed to write CSV entry %d: %w", i, err) + } + } -// CSV returns the CSV encoded values for a RealmStats. -func (r *RealmStats) CSV() []string { - return []string{ - r.Date.Format("2006-01-02"), - fmt.Sprintf("%d", r.RealmID), - fmt.Sprintf("%d", r.CodesIssued), - fmt.Sprintf("%d", r.CodesClaimed), + w.Flush() + if err := w.Error(); err != nil { + return nil, fmt.Errorf("failed to create CSV: %w", err) } + + return b.Bytes(), nil +} + +type jsonRealmStat struct { + RealmID uint `json:"realm_id"` + Stats []*jsonRealmStatStats `json:"statistics"` +} + +type jsonRealmStatStats struct { + Date time.Time `json:"date"` + Data *jsonRealmStatStatsData `json:"data"` +} + +type jsonRealmStatStatsData struct { + CodesIssued uint `json:"codes_issued"` + CodesClaimed uint `json:"codes_claimed"` } -// TableName sets the RealmStats table name -func (RealmStats) TableName() string { - return "realm_stats" +// MarshalJSON is a custom JSON marshaller. +func (s RealmStats) MarshalJSON() ([]byte, error) { + // Do nothing if there's no records + if len(s) == 0 { + return json.Marshal(struct{}{}) + } + + var stats []*jsonRealmStatStats + for _, stat := range s { + stats = append(stats, &jsonRealmStatStats{ + Date: stat.Date, + Data: &jsonRealmStatStatsData{ + CodesIssued: stat.CodesIssued, + CodesClaimed: stat.CodesClaimed, + }, + }) + } + + // Sort in descending order. + sort.Slice(stats, func(i, j int) bool { + return stats[i].Date.After(stats[j].Date) + }) + + var result jsonRealmStat + result.RealmID = s[0].RealmID + result.Stats = stats + + b, err := json.Marshal(result) + if err != nil { + return nil, fmt.Errorf("failed to marshal json: %w", err) + } + return b, nil +} + +func (s *RealmStats) UnmarshalJSON(b []byte) error { + if len(b) == 0 { + return nil + } + + var result jsonRealmStat + if err := json.Unmarshal(b, &result); err != nil { + return err + } + + for _, stat := range result.Stats { + *s = append(*s, &RealmStat{ + Date: stat.Date, + RealmID: result.RealmID, + CodesIssued: stat.Data.CodesIssued, + CodesClaimed: stat.Data.CodesClaimed, + }) + } + + return nil } // HistoricalCodesIssued returns a slice of the historical codes issued for diff --git a/pkg/database/realm_test.go b/pkg/database/realm_test.go index e4df33f5c..981752301 100644 --- a/pkg/database/realm_test.go +++ b/pkg/database/realm_test.go @@ -26,6 +26,8 @@ import ( ) func TestSMS(t *testing.T) { + t.Parallel() + realm := NewRealmWithDefaults("test") realm.SMSTextTemplate = "This is your Exposure Notifications Verification code: [enslink] Expires in [longexpires] hours" realm.RegionCode = "US-WA" @@ -92,15 +94,11 @@ func TestPerUserRealmStats(t *testing.T) { t.Error("len(users) = 0, expected ≠ 0") } - stats, err := realm.CodesPerUser(db, startDate, endDate) + stats, err := realm.UserStats(db, startDate, endDate) if err != nil { t.Fatalf("error getting stats: %v", err) } - if len(stats) != numDays*len(users) { - t.Errorf("len(stats) = %d, expected %d", len(stats), numDays*len(users)) - } - for i := 0; i < len(stats)-1; i++ { if stats[i].Date != stats[i+1].Date { if !stats[i].Date.After(stats[i+1].Date) { diff --git a/pkg/database/realm_user_stats.go b/pkg/database/realm_user_stats.go new file mode 100644 index 000000000..0bc7fa102 --- /dev/null +++ b/pkg/database/realm_user_stats.go @@ -0,0 +1,162 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "sort" + "strconv" + "time" + + "github.com/google/exposure-notifications-verification-server/internal/icsv" +) + +var _ icsv.Marshaler = (RealmUserStats)(nil) + +// RealmUserStats is a grouping collection of RealmUserStat. +type RealmUserStats []*RealmUserStat + +// RealmUserStat is an interim data structure representing a single date/user +// statistic. It does not correspond to a single database table, but is rather a +// join across multiple tables. +type RealmUserStat struct { + Date time.Time + RealmID uint + UserID uint + Name string + CodesIssued uint +} + +// MarshalCSV returns bytes in CSV format. +func (s RealmUserStats) MarshalCSV() ([]byte, error) { + // Do nothing if there's no records + if len(s) == 0 { + return nil, nil + } + + var b bytes.Buffer + w := csv.NewWriter(&b) + + if err := w.Write([]string{"date", "realm_id", "user_id", "name", "codes_issued"}); err != nil { + return nil, fmt.Errorf("failed to write CSV header: %w", err) + } + + for i, stat := range s { + if err := w.Write([]string{ + stat.Date.Format("2006-01-02"), + strconv.FormatUint(uint64(stat.RealmID), 10), + strconv.FormatUint(uint64(stat.UserID), 10), + stat.Name, + strconv.FormatUint(uint64(stat.CodesIssued), 10), + }); err != nil { + return nil, fmt.Errorf("failed to write CSV entry %d: %w", i, err) + } + } + + w.Flush() + if err := w.Error(); err != nil { + return nil, fmt.Errorf("failed to create CSV: %w", err) + } + + return b.Bytes(), nil +} + +type jsonRealmUserStat struct { + RealmID uint `json:"realm_id"` + Stats []*jsonRealmUserStatStats `json:"statistics"` +} + +type jsonRealmUserStatStats struct { + Date time.Time `json:"date"` + IssuerData []*jsonRealmUserStatIssuerData `json:"issuer_data"` +} + +type jsonRealmUserStatIssuerData struct { + UserID uint `json:"user_id"` + Name string `json:"name"` + CodesIssued uint `json:"codes_issued"` +} + +// MarshalJSON is a custom JSON marshaller. +func (s RealmUserStats) MarshalJSON() ([]byte, error) { + // Do nothing if there's no records + if len(s) == 0 { + return json.Marshal(struct{}{}) + } + + m := make(map[time.Time][]*jsonRealmUserStatIssuerData) + for _, stat := range s { + if m[stat.Date] == nil { + m[stat.Date] = make([]*jsonRealmUserStatIssuerData, 0, 8) + } + + m[stat.Date] = append(m[stat.Date], &jsonRealmUserStatIssuerData{ + UserID: stat.UserID, + Name: stat.Name, + CodesIssued: stat.CodesIssued, + }) + } + + stats := make([]*jsonRealmUserStatStats, 0, len(m)) + for k, v := range m { + stats = append(stats, &jsonRealmUserStatStats{ + Date: k, + IssuerData: v, + }) + } + + // Sort in descending order. + sort.Slice(stats, func(i, j int) bool { + return stats[i].Date.After(stats[j].Date) + }) + + var result jsonRealmUserStat + result.RealmID = s[0].RealmID + result.Stats = stats + + b, err := json.Marshal(result) + if err != nil { + return nil, fmt.Errorf("failed to marshal json: %w", err) + } + return b, nil +} + +func (s *RealmUserStats) UnmarshalJSON(b []byte) error { + if len(b) == 0 { + return nil + } + + var result jsonRealmUserStat + if err := json.Unmarshal(b, &result); err != nil { + return err + } + + for _, stat := range result.Stats { + for _, r := range stat.IssuerData { + *s = append(*s, &RealmUserStat{ + Date: stat.Date, + RealmID: result.RealmID, + UserID: r.UserID, + Name: r.Name, + CodesIssued: r.CodesIssued, + }) + } + } + + return nil +} diff --git a/pkg/database/token_test.go b/pkg/database/token_test.go index c9847e05d..77e7ec8e7 100644 --- a/pkg/database/token_test.go +++ b/pkg/database/token_test.go @@ -27,6 +27,8 @@ import ( ) func TestSubject(t *testing.T) { + t.Parallel() + testDay, err := time.Parse("2006-01-02", "2020-07-07") if err != nil { t.Fatalf("test setup error: %v", err) @@ -94,7 +96,11 @@ func TestSubject(t *testing.T) { } for _, tc := range cases { + tc := tc + t.Run(tc.Name, func(t *testing.T) { + t.Parallel() + got, err := ParseSubject(tc.Sub) if err != nil { if tc.Error == "" { @@ -276,6 +282,7 @@ func TestIssueToken(t *testing.T) { t.Run(tc.Name, func(t *testing.T) { t.Parallel() + db := NewTestDatabase(t) realm := NewRealmWithDefaults(fmt.Sprintf("TestIssueToken/%s", tc.Name)) @@ -369,6 +376,7 @@ func TestIssueToken(t *testing.T) { func TestPurgeTokens(t *testing.T) { t.Parallel() + db := NewTestDatabase(t) now := time.Now() diff --git a/pkg/database/user_stats.go b/pkg/database/user_stats.go index 1ee0eb84b..47dfafce6 100644 --- a/pkg/database/user_stats.go +++ b/pkg/database/user_stats.go @@ -22,15 +22,10 @@ import ( // UserStats represents statistics related to a user in the database. type UserStats struct { - Date time.Time `gorm:"date"` - UserID uint `gorm:"user_id"` - RealmID uint `gorm:"realm_id"` - CodesIssued uint `gorm:"codes_issued"` -} - -// TableName sets the UserStats table name -func (UserStats) TableName() string { - return "user_stats" + Date time.Time `gorm:"date;"` + UserID uint `gorm:"user_id;"` + RealmID uint `gorm:"realm_id;"` + CodesIssued uint `gorm:"codes_issued;"` } // SaveUserStats saves some UserStats to the database. diff --git a/pkg/database/user_test.go b/pkg/database/user_test.go index a5dd71666..cc60c0b60 100644 --- a/pkg/database/user_test.go +++ b/pkg/database/user_test.go @@ -75,7 +75,7 @@ func TestUserLifecycle(t *testing.T) { // Update password changed now := time.Now().UTC() - now = now.Truncate(time.Second) // db loses nanos + now = now.Truncate(time.Second) // db loses nanoseconds if err := db.PasswordChanged(email, now); err != nil { t.Fatalf("error updating password changed time: %v", err) } @@ -143,7 +143,9 @@ func TestPurgeUsers(t *testing.T) { } expectExists(t, db, user.ID) - db.PurgeUsers(time.Duration(0)) + if _, err := db.PurgeUsers(time.Duration(0)); err != nil { + t.Fatal(err) + } // Find user by ID - Expect deleted { diff --git a/pkg/database/vercode.go b/pkg/database/vercode.go index fc33cb033..1c8efbed3 100644 --- a/pkg/database/vercode.go +++ b/pkg/database/vercode.go @@ -20,6 +20,7 @@ import ( "encoding/base64" "errors" "fmt" + "strings" "time" "github.com/google/exposure-notifications-server/pkg/timeutils" @@ -51,6 +52,7 @@ var ( ErrInvalidTestType = errors.New("invalid test type, must be confirmed, likely, or negative") ErrCodeAlreadyExpired = errors.New("code already expired") + ErrCodeAlreadyClaimed = errors.New("code already claimed") ErrCodeTooShort = errors.New("verification code must be at least 6 digits") ErrTestTooOld = errors.New("test date is more than 14 day ago") ) @@ -70,8 +72,22 @@ type VerificationCode struct { TestDate *time.Time ExpiresAt time.Time LongExpiresAt time.Time + + // IssuingUserID is the ID of the user in the database that created this + // verification code. This is only populated if the code was created via the + // UI. IssuingUserID uint - IssuingAppID uint + + // IssuingAppID is the ID of the app in the database that created this + // verification code. This is only populated if the code was created via the + // API. + IssuingAppID uint + + // IssuingExternalID is an optional ID to an external system that created this + // verification code. This is only populated if the code was created via the + // API AND the API caller supplied it in the request. This ID has no meaning + // in this system. It can be up to 255 characters in length. + IssuingExternalID string } // TableName sets the VerificationCode table name @@ -79,6 +95,18 @@ func (VerificationCode) TableName() string { return "verification_codes" } +// BeforeSave is used by callbacks. +func (v *VerificationCode) BeforeSave(scope *gorm.Scope) error { + if len(v.IssuingExternalID) > 255 { + v.AddError("issuingExternalID", "cannot exceed 255 characters") + } + + if len(v.Errors()) > 0 { + return fmt.Errorf("email config validation failed: %s", strings.Join(v.ErrorMessages(), ", ")) + } + return nil +} + // AfterCreate runs after the verification code has been saved, primarily used // to update statistics about usage. If the executions fail, an error is logged // but the transaction continues. This is called automatically by gorm. @@ -99,6 +127,20 @@ func (v *VerificationCode) AfterCreate(scope *gorm.Scope) { } } + // If the request was an API request, we might have an external issuer ID. + if len(v.IssuingExternalID) != 0 { + sql := ` + INSERT INTO external_issuer_stats (date, realm_id, issuer_id, codes_issued) + VALUES ($1, $2, $3, 1) + ON CONFLICT (date, realm_id, issuer_id) DO UPDATE + SET codes_issued = external_issuer_stats.codes_issued + 1 + ` + + if err := scope.DB().Exec(sql, date, v.RealmID, v.IssuingExternalID).Error; err != nil { + scope.Log(fmt.Sprintf("failed to update audit stats: %v", err)) + } + } + // If the issuer was a app, update the app stats for the day. if v.IssuingAppID != 0 { sql := ` @@ -271,10 +313,10 @@ func (db *Database) ExpireCode(uuid string) (*VerificationCode, error) { return err } if vc.IsExpired() { - return errors.New("code already expired") + return ErrCodeAlreadyExpired } if vc.Claimed { - return errors.New("code already caimed") + return ErrCodeAlreadyClaimed } vc.ExpiresAt = time.Now() diff --git a/pkg/database/vercode_test.go b/pkg/database/vercode_test.go index 6b1da197b..407404ff1 100644 --- a/pkg/database/vercode_test.go +++ b/pkg/database/vercode_test.go @@ -27,6 +27,7 @@ import ( func TestVerificationCode_FindVerificationCode(t *testing.T) { t.Parallel() + db := NewTestDatabase(t) uuid := "5148c75c-2bc5-4874-9d1c-f9185d0e1b8a" @@ -135,6 +136,7 @@ func TestVerificationCode_ListRecentCodes(t *testing.T) { t.Parallel() db := NewTestDatabase(t) + var realmID uint = 123 var userID uint = 456 @@ -211,6 +213,8 @@ func TestVerificationCode_ExpireVerificationCode(t *testing.T) { } func TestVerCodeValidate(t *testing.T) { + t.Parallel() + maxAge := time.Hour * 24 * 14 oldTest := time.Now().Add(-1 * 20 * oneDay) cases := []struct { @@ -265,7 +269,11 @@ func TestVerCodeValidate(t *testing.T) { } for _, tc := range cases { + tc := tc + t.Run(tc.Name, func(t *testing.T) { + t.Parallel() + if err := tc.Code.Validate(maxAge); err != tc.Err { t.Fatalf("wrong error, want %v, got: %v", tc.Err, err) } @@ -274,6 +282,8 @@ func TestVerCodeValidate(t *testing.T) { } func TestVerCodeIsExpired(t *testing.T) { + t.Parallel() + code := VerificationCode{ Code: "12345678", TestType: "confirmed", @@ -287,6 +297,7 @@ func TestVerCodeIsExpired(t *testing.T) { func TestDeleteVerificationCode(t *testing.T) { t.Parallel() + db := NewTestDatabase(t) maxAge := time.Hour @@ -314,6 +325,7 @@ func TestDeleteVerificationCode(t *testing.T) { func TestVerificationCodesCleanup(t *testing.T) { t.Parallel() + db := NewTestDatabase(t) now := time.Now() @@ -402,8 +414,9 @@ func TestStatDatesOnCreate(t *testing.T) { // all dates, and a bunch of corner cases. This is intended as a // smokescreen. t.Parallel() + db := NewTestDatabase(t) - db.db.LogMode(true) + fmtString := "2006-01-02" now := time.Now() nowStr := now.Format(fmtString) @@ -415,25 +428,27 @@ func TestStatDatesOnCreate(t *testing.T) { }{ { &VerificationCode{ - Code: "111111", - LongCode: "111111", - TestType: "negative", - ExpiresAt: now.Add(time.Second), - LongExpiresAt: now.Add(time.Second), - IssuingUserID: 100, // need for RealmUserStats - IssuingAppID: 200, // need for AuthorizedAppStats - RealmID: 300, // need for RealmStats + Code: "111111", + LongCode: "111111", + TestType: "negative", + ExpiresAt: now.Add(time.Second), + LongExpiresAt: now.Add(time.Second), + IssuingUserID: 100, // need for RealmUserStats + IssuingAppID: 200, // need for AuthorizedAppStats + IssuingExternalID: "aa-bb-cc", // need for ExternalIssuerStats + RealmID: 300, // need for RealmStats }, - nowStr}, + nowStr, + }, } for i, test := range tests { if err := db.SaveVerificationCode(test.code, maxAge); err != nil { - t.Errorf("[%d] error saving code: %v", i, err) + t.Fatalf("[%d] error saving code: %v", i, err) } { - var stats []*RealmUserStats + var stats []*RealmUserStat if err := db.db. Model(&UserStats{}). Select("*"). @@ -454,6 +469,28 @@ func TestStatDatesOnCreate(t *testing.T) { } } + if len(test.code.IssuingExternalID) != 0 { + var stats []*ExternalIssuerStat + if err := db.db. + Model(&ExternalIssuerStats{}). + Select("*"). + Scan(&stats). + Error; err != nil { + if IsNotFound(err) { + t.Fatalf("[%d] Error grabbing external issuer stats %v", i, err) + } + } + if len(stats) != 1 { + t.Fatalf("[%d] expected one user stat", i) + } + if stats[0].CodesIssued != uint(i+1) { + t.Errorf("[%d] expected stat.CodesIssued = %d, expected %d", i, stats[0].CodesIssued, i+1) + } + if f := stats[0].Date.Format(fmtString); f != test.statDate { + t.Errorf("[%d] expected stat.Date = %s, expected %s", i, f, test.statDate) + } + } + { var stats []*AuthorizedAppStats if err := db.db. @@ -477,7 +514,7 @@ func TestStatDatesOnCreate(t *testing.T) { } { - var stats []*RealmStats + var stats []*RealmStat if err := db.db. Model(&RealmStats{}). Select("*"). diff --git a/pkg/otp/code.go b/pkg/otp/code.go index 2fa7660ef..3a955c8b7 100644 --- a/pkg/otp/code.go +++ b/pkg/otp/code.go @@ -86,9 +86,12 @@ type Request struct { SymptomDate *time.Time TestDate *time.Time MaxSymptomAge time.Duration - IssuingUser *database.User - IssuingApp *database.AuthorizedApp UUID string + + // Issuing includes information about the issuer. + IssuingUser *database.User + IssuingApp *database.AuthorizedApp + IssuingExternalID string } // Issue will generate a verification code and save it to the database, based on @@ -124,17 +127,18 @@ func (o *Request) Issue(ctx context.Context, retryCount uint) (string, string, s } verificationCode = database.VerificationCode{ - RealmID: o.RealmID, - Code: code, - LongCode: longCode, - TestType: strings.ToLower(o.TestType), - SymptomDate: o.SymptomDate, - TestDate: o.TestDate, - ExpiresAt: o.ShortExpiresAt, - LongExpiresAt: o.LongExpiresAt, - IssuingUserID: issuingUserID, - IssuingAppID: issuingAppID, - UUID: o.UUID, + RealmID: o.RealmID, + Code: code, + LongCode: longCode, + TestType: strings.ToLower(o.TestType), + SymptomDate: o.SymptomDate, + TestDate: o.TestDate, + ExpiresAt: o.ShortExpiresAt, + LongExpiresAt: o.LongExpiresAt, + IssuingUserID: issuingUserID, + IssuingAppID: issuingAppID, + IssuingExternalID: o.IssuingExternalID, + UUID: o.UUID, } // If a verification code already exists, it will fail to save, and we retry. if err = o.DB.SaveVerificationCode(&verificationCode, o.MaxSymptomAge); err != nil { diff --git a/pkg/render/csv.go b/pkg/render/csv.go new file mode 100644 index 000000000..74d306551 --- /dev/null +++ b/pkg/render/csv.go @@ -0,0 +1,63 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render + +import ( + "fmt" + "net/http" + + "github.com/google/exposure-notifications-verification-server/internal/icsv" +) + +// RenderCSV renders the input as a CSV. It attempts to gracefully handle +// any rendering errors to avoid partial responses sent to the response by +// writing to a buffer first, then flushing the buffer to the response. +func (r *Renderer) RenderCSV(w http.ResponseWriter, code int, filename string, data icsv.Marshaler) { + // Avoid marshaling nil data. + if data == nil { + w.Header().Set("Content-Type", "text/csv") + w.WriteHeader(code) + } + + // Create CSV. + b, err := data.MarshalCSV() + if err != nil { + msg := "An internal error occurred." + if r.debug { + msg = err.Error() + } + + w.Header().Set("Content-Type", "text/csv") + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "%s", msg) + return + } + + // Ensure there's a filename. + if filename == "" { + filename = "data.csv" + } + + // Rendering worked, flush to the response. Force as a download. + w.Header().Set("Content-Type", "text/csv") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment;filename=%s", filename)) + w.WriteHeader(code) + if _, err := w.Write(b); err != nil { + // We couldn't write the buffer. We can't change the response header or + // content type if we got this far, so the best option we have is to log the + // error. + r.logger.Errorw("failed to write csv to response", "error", err) + } +} diff --git a/tools/get-code/main.go b/tools/get-code/main.go index ec7620d17..3b6a8275a 100644 --- a/tools/get-code/main.go +++ b/tools/get-code/main.go @@ -21,8 +21,8 @@ import ( "fmt" "os" "strconv" - "time" + "github.com/google/exposure-notifications-verification-server/pkg/api" "github.com/google/exposure-notifications-verification-server/pkg/clients" "github.com/google/exposure-notifications-server/pkg/logging" @@ -35,8 +35,8 @@ var ( onsetFlag = flag.String("onset", "", "Symptom onset date, YYYY-MM-DD format") tzOffsetFlag = flag.Int("tzOffset", 0, "timezone adjustment (minutes) from UTC for request") apikeyFlag = flag.String("apikey", "", "API Key to use") + adminIDFlag = flag.String("adminID", "", "AdminID for statistics tracking") addrFlag = flag.String("addr", "http://localhost:8080", "protocol, address and port on which to make the API call") - timeoutFlag = flag.Duration("timeout", 5*time.Second, "request time out duration in the format: 0h0m0s") ) func main() { @@ -59,7 +59,14 @@ func main() { func realMain(ctx context.Context) error { logger := logging.FromContext(ctx) - request, response, err := clients.IssueCode(ctx, *addrFlag, *apikeyFlag, *testFlag, *onsetFlag, *tzOffsetFlag, *timeoutFlag) + request := &api.IssueCodeRequest{ + TestType: *testFlag, + SymptomDate: *onsetFlag, + TZOffset: float32(*tzOffsetFlag), + ExternalIssuerID: *adminIDFlag, + } + + response, err := clients.IssueCode(ctx, *addrFlag, *apikeyFlag, request) logger.Infow("sent request", "request", request) if err != nil { return fmt.Errorf("failed to get token: %w", err) diff --git a/tools/seed/main.go b/tools/seed/main.go index d99043778..5d5d40452 100644 --- a/tools/seed/main.go +++ b/tools/seed/main.go @@ -18,14 +18,19 @@ package main import ( "context" + "encoding/hex" "fmt" + "math/rand" "os" "strconv" + "time" firebase "firebase.google.com/go" firebaseauth "firebase.google.com/go/auth" + "github.com/google/exposure-notifications-verification-server/pkg/api" "github.com/google/exposure-notifications-verification-server/pkg/config" "github.com/google/exposure-notifications-verification-server/pkg/database" + "github.com/jinzhu/gorm" "github.com/google/exposure-notifications-server/pkg/logging" @@ -202,6 +207,83 @@ func realMain(ctx context.Context) error { } logger.Infow("created device api key", "key", adminAPIKey) + // Generate some codes + now := time.Now().UTC() + users := []*database.User{user, unverified, super, admin} + externalIDs := make([]string, 4) + for i := range externalIDs { + b := make([]byte, 8) + if _, err := rand.Read(b); err != nil { + return fmt.Errorf("failed to read rand: %w", err) + } + externalIDs[i] = hex.EncodeToString(b) + } + + for day := 1; day <= 30; day++ { + max := rand.Intn(50) + for i := 0; i < max; i++ { + date := now.Add(time.Duration(day) * -24 * time.Hour) + + issuingUserID := uint(0) + issuingAppID := uint(0) + issuingExternalID := "" + + // Random determine if this was issued by an app (60% chance). + if rand.Intn(10) <= 6 { + issuingAppID = apps[rand.Intn(len(apps))].ID + + // Random determine if the code had an external audit. + if rand.Intn(2) == 0 { + b := make([]byte, 8) + if _, err := rand.Read(b); err != nil { + return fmt.Errorf("failed to read rand: %w", err) + } + issuingExternalID = externalIDs[rand.Intn(len(externalIDs))] + } + } else { + issuingUserID = users[rand.Intn(len(users))].ID + } + + code := fmt.Sprintf("%08d", rand.Intn(99999999)) + longCode := fmt.Sprintf("%015d", rand.Intn(999999999999999)) + testDate := now.Add(-48 * time.Hour) + + verificationCode := &database.VerificationCode{ + Model: gorm.Model{ + CreatedAt: date, + }, + RealmID: realm1.ID, + Code: code, + ExpiresAt: now.Add(15 * time.Minute), + LongCode: longCode, + LongExpiresAt: now.Add(24 * time.Hour), + TestType: "confirmed", + SymptomDate: &testDate, + TestDate: &testDate, + + IssuingUserID: issuingUserID, + IssuingAppID: issuingAppID, + IssuingExternalID: issuingExternalID, + } + // If a verification code already exists, it will fail to save, and we retry. + if err := db.SaveVerificationCode(verificationCode, 672*time.Hour); err != nil { + return fmt.Errorf("failed to create verification code: %w", err) + } + + // 40% chance that the code is claimed + if rand.Intn(10) <= 4 { + accept := map[string]struct{}{ + api.TestTypeConfirmed: {}, + api.TestTypeLikely: {}, + api.TestTypeNegative: {}, + } + if _, err := db.VerifyCodeAndIssueToken(realm1.ID, code, accept, 24*time.Hour); err != nil { + return fmt.Errorf("failed to claim token: %w", err) + } + } + } + } + return nil }