diff --git a/cmd/fastly-exporter/main.go b/cmd/fastly-exporter/main.go index 4629108..b6b2a2d 100644 --- a/cmd/fastly-exporter/main.go +++ b/cmd/fastly-exporter/main.go @@ -28,23 +28,23 @@ var programVersion = "dev" func main() { var ( - token string - listen string - namespace string - subsystem string - serviceShard string - serviceIDs stringslice - serviceAllowlist stringslice - serviceBlocklist stringslice - metricAllowlist stringslice - metricBlocklist stringslice - datacenterRefresh time.Duration - serviceRefresh time.Duration - apiTimeout time.Duration - rtTimeout time.Duration - debug bool - versionFlag bool - configFileExample bool + token string + listen string + namespace string + deprecatedSubsystem string + serviceShard string + serviceIDs stringslice + serviceAllowlist stringslice + serviceBlocklist stringslice + metricAllowlist stringslice + metricBlocklist stringslice + datacenterRefresh time.Duration + serviceRefresh time.Duration + apiTimeout time.Duration + rtTimeout time.Duration + debug bool + versionFlag bool + configFileExample bool ) fs := flag.NewFlagSet("fastly-exporter", flag.ContinueOnError) @@ -52,7 +52,7 @@ func main() { fs.StringVar(&token, "token", "", "Fastly API token (required)") fs.StringVar(&listen, "listen", "127.0.0.1:8080", "listen address for Prometheus metrics") fs.StringVar(&namespace, "namespace", "fastly", "Prometheus namespace") - fs.StringVar(&subsystem, "subsystem", "rt", "Prometheus subsystem") + fs.StringVar(&deprecatedSubsystem, "subsystem", "rt", "DEPRECATED -- will be fixed to 'rt' in a future version") fs.StringVar(&serviceShard, "service-shard", "", "if set, only include services whose hashed IDs modulo m equal n-1 (format 'n/m')") fs.Var(&serviceIDs, "service", "if set, only include this service ID (repeatable)") fs.Var(&serviceAllowlist, "service-allowlist", "if set, only include services whose names match this regex (repeatable)") @@ -92,11 +92,7 @@ func main() { var logger log.Logger { logger = log.NewLogfmtLogger(os.Stderr) - loglevel := level.AllowInfo() - if debug { - loglevel = level.AllowDebug() - } - logger = level.NewFilter(logger, loglevel) + logger = level.NewFilter(logger, getLogLevel(debug)) } if token == "" { @@ -106,6 +102,16 @@ func main() { } } + switch deprecatedSubsystem { + case "rt": + // good + case "origin": + level.Error(logger).Log("err", "-subsystem cannot be 'origin'") + os.Exit(1) + default: + level.Warn(logger).Log("msg", "-subsystem is DEPRECATED and will be fixed to 'rt' in a future version") + } + fs.Visit(func(f *flag.Flag) { if f.Name == "api-refresh" { level.Warn(logger).Log("msg", "-api-refresh is deprecated and will be removed in a future version, please use -service-refresh instead") @@ -278,7 +284,7 @@ func main() { var defaultGatherers prometheus.Gatherers { - dcs, err := datacenterCache.Gatherer(namespace, subsystem) + dcs, err := datacenterCache.Gatherer(namespace, deprecatedSubsystem) if err != nil { level.Error(apiLogger).Log("during", "create datacenter gatherer", "err", err) os.Exit(1) @@ -288,7 +294,7 @@ func main() { var registry *prom.Registry { - registry = prom.NewRegistry(programVersion, namespace, subsystem, metricNameFilter, defaultGatherers) + registry = prom.NewRegistry(programVersion, namespace, deprecatedSubsystem, metricNameFilter, defaultGatherers) } var manager *rt.Manager @@ -458,3 +464,12 @@ service-blocklist Dev metric-blocklist imgopto `) + +func getLogLevel(debug bool) level.Option { + switch { + case debug: + return level.AllowDebug() + default: + return level.AllowInfo() + } +} diff --git a/cmd/fieldgen/api_fields.json b/cmd/fieldgen/api_fields.json deleted file mode 100644 index cf30f5c..0000000 --- a/cmd/fieldgen/api_fields.json +++ /dev/null @@ -1,181 +0,0 @@ -[ - {"field_name": "AttackBlockedReqBodyBytes", "type": "uint64", "key": "attack_blocked_req_body_bytes"}, - {"field_name": "AttackBlockedReqHeaderBytes", "type": "uint64", "key": "attack_blocked_req_header_bytes"}, - {"field_name": "AttackLoggedReqBodyBytes", "type": "uint64", "key": "attack_logged_req_body_bytes"}, - {"field_name": "AttackLoggedReqHeaderBytes", "type": "uint64", "key": "attack_logged_req_header_bytes"}, - {"field_name": "AttackPassedReqBodyBytes", "type": "uint64", "key": "attack_passed_req_body_bytes"}, - {"field_name": "AttackPassedReqHeaderBytes", "type": "uint64", "key": "attack_passed_req_header_bytes"}, - {"field_name": "AttackReqBodyBytes", "type": "uint64", "key": "attack_req_body_bytes"}, - {"field_name": "AttackReqHeaderBytes", "type": "uint64", "key": "attack_req_header_bytes"}, - {"field_name": "AttackRespSynthBytes", "type": "uint64", "key": "attack_resp_synth_bytes"}, - {"field_name": "BackendReqBodyBytes", "type": "uint64", "key": "bereq_body_bytes"}, - {"field_name": "BackendReqHeaderBytes", "type": "uint64", "key": "bereq_header_bytes"}, - {"field_name": "Billed", "type": "uint64", "key": "billed"}, - {"field_name": "BilledBodyBytes", "type": "uint64", "key": "billed_body_bytes"}, - {"field_name": "BilledHeaderBytes", "type": "uint64", "key": "billed_header_bytes"}, - {"field_name": "Blacklisted", "type": "uint64", "key": "blacklist"}, - {"field_name": "BodySize", "type": "uint64", "key": "body_size"}, - {"field_name": "ComputeBackendReqBodyBytesTotal", "type": "uint64", "key": "compute_bereq_body_bytes"}, - {"field_name": "ComputeBackendReqErrorsTotal", "type": "uint64", "key": "compute_bereq_errors"}, - {"field_name": "ComputeBackendReqHeaderBytesTotal", "type": "uint64", "key": "compute_bereq_header_bytes"}, - {"field_name": "ComputeBackendReqTotal", "type": "uint64", "key": "compute_bereqs"}, - {"field_name": "ComputeBackendRespBodyBytesTotal", "type": "uint64", "key": "compute_beresp_body_bytes"}, - {"field_name": "ComputeBackendRespHeaderBytesTotal", "type": "uint64", "key": "compute_beresp_header_bytes"}, - {"field_name": "ComputeExecutionTimeMilliseconds", "type": "uint64", "key": "compute_execution_time_ms"}, - {"field_name": "ComputeGlobalsLimitExceededTotal", "type": "uint64", "key": "compute_globals_limit_exceeded"}, - {"field_name": "ComputeGuestErrorsTotal", "type": "uint64", "key": "compute_guest_errors"}, - {"field_name": "ComputeHeapLimitExceededTotal", "type": "uint64", "key": "compute_heap_limit_exceeded"}, - {"field_name": "ComputeRAMUsed", "type": "uint64", "key": "compute_ram_used"}, - {"field_name": "ComputeReqBodyBytesTotal", "type": "uint64", "key": "compute_req_body_bytes"}, - {"field_name": "ComputeReqHeaderBytesTotal", "type": "uint64", "key": "compute_req_header_bytes"}, - {"field_name": "ComputeRequests", "type": "uint64", "key": "compute_requests"}, - {"field_name": "ComputeRequestTimeMilliseconds", "type": "uint64", "key": "compute_request_time_ms"}, - {"field_name": "ComputeResourceLimitExceedTotal", "type": "uint64", "key": "compute_resource_limit_exceeded"}, - {"field_name": "ComputeRespBodyBytesTotal", "type": "uint64", "key": "compute_resp_body_bytes"}, - {"field_name": "ComputeRespHeaderBytesTotal", "type": "uint64", "key": "compute_resp_header_bytes"}, - {"field_name": "ComputeRespStatus1xx", "type": "uint64", "key": "compute_resp_status_1xx"}, - {"field_name": "ComputeRespStatus2xx", "type": "uint64", "key": "compute_resp_status_2xx"}, - {"field_name": "ComputeRespStatus3xx", "type": "uint64", "key": "compute_resp_status_3xx"}, - {"field_name": "ComputeRespStatus4xx", "type": "uint64", "key": "compute_resp_status_4xx"}, - {"field_name": "ComputeRespStatus5xx", "type": "uint64", "key": "compute_resp_status_5xx"}, - {"field_name": "ComputeRuntimeErrorsTotal", "type": "uint64", "key": "compute_runtime_errors"}, - {"field_name": "ComputeStackLimitExceededTotal", "type": "uint64", "key": "compute_stack_limit_exceeded"}, - {"field_name": "DeliverSubCount", "type": "uint64", "key": "deliver_sub_count"}, - {"field_name": "DeliverSubTime", "type": "uint64", "key": "deliver_sub_time"}, - {"field_name": "Edge", "type": "uint64", "key": "edge_requests"}, - {"field_name": "EdgeRespBodyBytes", "type": "uint64", "key": "edge_resp_body_bytes"}, - {"field_name": "EdgeRespHeaderBytes", "type": "uint64", "key": "edge_resp_header_bytes"}, - {"field_name": "Errors", "type": "uint64", "key": "errors"}, - {"field_name": "ErrorSubCount", "type": "uint64", "key": "error_sub_count"}, - {"field_name": "ErrorSubTime", "type": "uint64", "key": "error_sub_time"}, - {"field_name": "FetchSubCount", "type": "uint64", "key": "fetch_sub_count"}, - {"field_name": "FetchSubTime", "type": "uint64", "key": "fetch_sub_time"}, - {"field_name": "HashSubCount", "type": "uint64", "key": "hash_sub_count"}, - {"field_name": "HashSubTime", "type": "uint64", "key": "hash_sub_time"}, - {"field_name": "HeaderSize", "type": "uint64", "key": "header_size"}, - {"field_name": "HitRespBodyBytes", "type": "uint64", "key": "hit_resp_body_bytes"}, - {"field_name": "Hits", "type": "uint64", "key": "hits"}, - {"field_name": "HitsTime", "type": "float64", "key": "hits_time"}, - {"field_name": "HitSubCount", "type": "uint64", "key": "hit_sub_count"}, - {"field_name": "HitSubTime", "type": "uint64", "key": "hit_sub_time"}, - {"field_name": "HTTP2", "type": "uint64", "key": "http2"}, - {"field_name": "ImgOpto", "type": "uint64", "key": "imgopto"}, - {"field_name": "ImgOptoRespBodyBytes", "type": "uint64", "key": "imgopto_resp_body_bytes"}, - {"field_name": "ImgOptoRespHeaderBytes", "type": "uint64", "key": "imgopto_resp_header_bytes"}, - {"field_name": "ImgOptoShield", "type": "uint64", "key": "imgopto_shield"}, - {"field_name": "ImgOptoShieldRespBodyBytes", "type": "uint64", "key": "imgopto_shield_resp_body_bytes"}, - {"field_name": "ImgOptoShieldRespHeaderBytes", "type": "uint64", "key": "imgopto_shield_resp_header_bytes"}, - {"field_name": "ImgOptoTransform", "type": "uint64", "key": "imgopto_transforms"}, - {"field_name": "ImgOptoTransformRespBodyBytes", "type": "uint64", "key": "imgopto_transform_resp_body_bytes"}, - {"field_name": "ImgOptoTransformRespHeaderBytes", "type": "uint64", "key": "imgopto_transform_resp_header_bytes"}, - {"field_name": "ImgVideo", "type": "uint64", "key": "imgvideo"}, - {"field_name": "ImgVideoFrames", "type": "uint64", "key": "imgvideo_frames"}, - {"field_name": "ImgVideoRespBodyBytes", "type": "uint64", "key": "imgvideo_resp_body_bytes"}, - {"field_name": "ImgVideoRespHeaderBytes", "type": "uint64", "key": "imgvideo_resp_header_bytes"}, - {"field_name": "ImgVideoShield", "type": "uint64", "key": "imgvideo_shield"}, - {"field_name": "ImgVideoShieldFrames", "type": "uint64", "key": "imgvideo_shield_frames"}, - {"field_name": "ImgVideoShieldRespBodyBytes", "type": "uint64", "key": "imgvideo_shield_resp_body_bytes"}, - {"field_name": "ImgVideoShieldRespHeaderBytes", "type": "uint64", "key": "imgvideo_shield_resp_header_bytes"}, - {"field_name": "IPv6", "type": "uint64", "key": "ipv6"}, - {"field_name": "LogBytes", "type": "uint64", "key": "log_bytes"}, - {"field_name": "Logging", "type": "uint64", "key": "logging"}, - {"field_name": "Misses", "type": "uint64", "key": "miss"}, - {"field_name": "MissHistogram", "type": "map[string]uint64", "key": "miss_histogram"}, - {"field_name": "MissRespBodyBytes", "type": "uint64", "key": "miss_resp_body_bytes"}, - {"field_name": "MissSubCount", "type": "uint64", "key": "miss_sub_count"}, - {"field_name": "MissSubTime", "type": "uint64", "key": "miss_sub_time"}, - {"field_name": "MissTime", "type": "float64", "key": "miss_time"}, - {"field_name": "ObjectSize100k", "type": "uint64", "key": "object_size_100k"}, - {"field_name": "ObjectSize100m", "type": "uint64", "key": "object_size_100m"}, - {"field_name": "ObjectSize10k", "type": "uint64", "key": "object_size_10k"}, - {"field_name": "ObjectSize10m", "type": "uint64", "key": "object_size_10m"}, - {"field_name": "ObjectSize1g", "type": "uint64", "key": "object_size_1g"}, - {"field_name": "ObjectSize1k", "type": "uint64", "key": "object_size_1k"}, - {"field_name": "ObjectSize1m", "type": "uint64", "key": "object_size_1m"}, - {"field_name": "ObjectSizeOther", "type": "uint64", "key": "object_size_other"}, - {"field_name": "OriginFetchBodyBytes", "type": "uint64", "key": "origin_fetch_body_bytes"}, - {"field_name": "OriginFetches", "type": "uint64", "key": "origin_fetches"}, - {"field_name": "OriginFetchHeaderBytes", "type": "uint64", "key": "origin_fetch_header_bytes"}, - {"field_name": "OriginFetchRespBodyBytes", "type": "uint64", "key": "origin_fetch_resp_body_bytes"}, - {"field_name": "OriginFetchRespHeaderBytes", "type": "uint64", "key": "origin_fetch_resp_header_bytes"}, - {"field_name": "OriginRevalidations", "type": "uint64", "key": "origin_revalidations"}, - {"field_name": "OTFP", "type": "uint64", "key": "otfp"}, - {"field_name": "OTFPDeliverTime", "type": "uint64", "key": "otfp_deliver_time"}, - {"field_name": "OTFPManifest", "type": "uint64", "key": "otfp_manifests"}, - {"field_name": "OTFPRespBodyBytes", "type": "uint64", "key": "otfp_resp_body_bytes"}, - {"field_name": "OTFPRespHeaderBytes", "type": "uint64", "key": "otfp_resp_header_bytes"}, - {"field_name": "OTFPShield", "type": "uint64", "key": "otfp_shield"}, - {"field_name": "OTFPShieldRespBodyBytes", "type": "uint64", "key": "otfp_shield_resp_body_bytes"}, - {"field_name": "OTFPShieldRespHeaderBytes", "type": "uint64", "key": "otfp_shield_resp_header_bytes"}, - {"field_name": "OTFPShieldTime", "type": "uint64", "key": "otfp_shield_time"}, - {"field_name": "OTFPTransform", "type": "uint64", "key": "otfp_transforms"}, - {"field_name": "OTFPTransformRespBodyBytes", "type": "uint64", "key": "otfp_transform_resp_body_bytes"}, - {"field_name": "OTFPTransformRespHeaderBytes", "type": "uint64", "key": "otfp_transform_resp_header_bytes"}, - {"field_name": "OTFPTransformTime", "type": "uint64", "key": "otfp_transform_time"}, - {"field_name": "Passes", "type": "uint64", "key": "pass"}, - {"field_name": "PassRespBodyBytes", "type": "uint64", "key": "pass_resp_body_bytes"}, - {"field_name": "PassSubCount", "type": "uint64", "key": "pass_sub_count"}, - {"field_name": "PassSubTime", "type": "uint64", "key": "pass_sub_time"}, - {"field_name": "PassTime", "type": "float64", "key": "pass_time"}, - {"field_name": "PCI", "type": "uint64", "key": "pci"}, - {"field_name": "Pipe", "type": "uint64", "key": "pipe"}, - {"field_name": "PipeSubCount", "type": "uint64", "key": "pipe_sub_count"}, - {"field_name": "PipeSubTime", "type": "uint64", "key": "pipe_sub_time"}, - {"field_name": "PredeliverSubCount", "type": "uint64", "key": "predeliver_sub_count"}, - {"field_name": "PredeliverSubTime", "type": "uint64", "key": "predeliver_sub_time"}, - {"field_name": "PrehashSubCount", "type": "uint64", "key": "prehash_sub_count"}, - {"field_name": "PrehashSubTime", "type": "uint64", "key": "prehash_sub_time"}, - {"field_name": "RecvSubCount", "type": "uint64", "key": "recv_sub_count"}, - {"field_name": "RecvSubTime", "type": "uint64", "key": "recv_sub_time"}, - {"field_name": "ReqBodyBytes", "type": "uint64", "key": "req_body_bytes"}, - {"field_name": "ReqHeaderBytes", "type": "uint64", "key": "req_header_bytes"}, - {"field_name": "Requests", "type": "uint64", "key": "requests"}, - {"field_name": "RespBodyBytes", "type": "uint64", "key": "resp_body_bytes"}, - {"field_name": "RespHeaderBytes", "type": "uint64", "key": "resp_header_bytes"}, - {"field_name": "Restart", "type": "uint64", "key": "restarts"}, - {"field_name": "SegBlockOriginFetches", "type": "uint64", "key": "segblock_origin_fetches"}, - {"field_name": "SegBlockShieldFetches", "type": "uint64", "key": "segblock_shield_fetches"}, - {"field_name": "Shield", "type": "uint64", "key": "shield"}, - {"field_name": "ShieldFetchBodyBytes", "type": "uint64", "key": "shield_fetch_body_bytes"}, - {"field_name": "ShieldFetches", "type": "uint64", "key": "shield_fetches"}, - {"field_name": "ShieldFetchHeaderBytes", "type": "uint64", "key": "shield_fetch_header_bytes"}, - {"field_name": "ShieldFetchRespBodyBytes", "type": "uint64", "key": "shield_fetch_resp_body_bytes"}, - {"field_name": "ShieldFetchRespHeaderBytes", "type": "uint64", "key": "shield_fetch_resp_header_bytes"}, - {"field_name": "ShieldRespBodyBytes", "type": "uint64", "key": "shield_resp_body_bytes"}, - {"field_name": "ShieldRespHeaderBytes", "type": "uint64", "key": "shield_resp_header_bytes"}, - {"field_name": "ShieldRevalidations", "type": "uint64", "key": "shield_revalidations"}, - {"field_name": "Status1xx", "type": "uint64", "key": "status_1xx"}, - {"field_name": "Status200", "type": "uint64", "key": "status_200"}, - {"field_name": "Status204", "type": "uint64", "key": "status_204"}, - {"field_name": "Status206", "type": "uint64", "key": "status_206"}, - {"field_name": "Status2xx", "type": "uint64", "key": "status_2xx"}, - {"field_name": "Status301", "type": "uint64", "key": "status_301"}, - {"field_name": "Status302", "type": "uint64", "key": "status_302"}, - {"field_name": "Status304", "type": "uint64", "key": "status_304"}, - {"field_name": "Status3xx", "type": "uint64", "key": "status_3xx"}, - {"field_name": "Status400", "type": "uint64", "key": "status_400"}, - {"field_name": "Status401", "type": "uint64", "key": "status_401"}, - {"field_name": "Status403", "type": "uint64", "key": "status_403"}, - {"field_name": "Status404", "type": "uint64", "key": "status_404"}, - {"field_name": "Status416", "type": "uint64", "key": "status_416"}, - {"field_name": "Status429", "type": "uint64", "key": "status_429"}, - {"field_name": "Status4xx", "type": "uint64", "key": "status_4xx"}, - {"field_name": "Status500", "type": "uint64", "key": "status_500"}, - {"field_name": "Status501", "type": "uint64", "key": "status_501"}, - {"field_name": "Status502", "type": "uint64", "key": "status_502"}, - {"field_name": "Status503", "type": "uint64", "key": "status_503"}, - {"field_name": "Status504", "type": "uint64", "key": "status_504"}, - {"field_name": "Status505", "type": "uint64", "key": "status_505"}, - {"field_name": "Status5xx", "type": "uint64", "key": "status_5xx"}, - {"field_name": "Synths", "type": "uint64", "key": "synth"}, - {"field_name": "TLS", "type": "uint64", "key": "tls"}, - {"field_name": "TLSv10", "type": "uint64", "key": "tls_v10"}, - {"field_name": "TLSv11", "type": "uint64", "key": "tls_v11"}, - {"field_name": "TLSv12", "type": "uint64", "key": "tls_v12"}, - {"field_name": "TLSv13", "type": "uint64", "key": "tls_v13"}, - {"field_name": "Uncacheable", "type": "uint64", "key": "uncacheable"}, - {"field_name": "Video", "type": "uint64", "key": "video"}, - {"field_name": "WAFBlocked", "type": "uint64", "key": "waf_blocked"}, - {"field_name": "WAFLogged", "type": "uint64", "key": "waf_logged"}, - {"field_name": "WAFPassed", "type": "uint64", "key": "waf_passed"} -] diff --git a/cmd/fieldgen/exporter_metrics.json b/cmd/fieldgen/exporter_metrics.json deleted file mode 100644 index 180e7d9..0000000 --- a/cmd/fieldgen/exporter_metrics.json +++ /dev/null @@ -1,145 +0,0 @@ -[ - {"field_name": "AttackBlockedReqBodyBytesTotal", "type": "Counter", "metric_name": "attack_blocked_req_body_bytes_total", "extra_labels": [], "help": "Total body bytes received from requests that triggered a WAF rule that was blocked."}, - {"field_name": "AttackBlockedReqHeaderBytesTotal", "type": "Counter", "metric_name": "attack_blocked_req_header_bytes_total", "extra_labels": [], "help": "Total header bytes received from requests that triggered a WAF rule that was blocked."}, - {"field_name": "AttackLoggedReqBodyBytesTotal", "type": "Counter", "metric_name": "attack_logged_req_body_bytes_total", "extra_labels": [], "help": "Total body bytes received from requests that triggered a WAF rule that was logged."}, - {"field_name": "AttackLoggedReqHeaderBytesTotal", "type": "Counter", "metric_name": "attack_logged_req_header_bytes_total", "extra_labels": [], "help": "Total header bytes received from requests that triggered a WAF rule that was logged."}, - {"field_name": "AttackPassedReqBodyBytesTotal", "type": "Counter", "metric_name": "attack_passed_req_body_bytes_total", "extra_labels": [], "help": "Total body bytes received from requests that triggered a WAF rule that was passed."}, - {"field_name": "AttackPassedReqHeaderBytesTotal", "type": "Counter", "metric_name": "attack_passed_req_header_bytes_total", "extra_labels": [], "help": "Total header bytes received from requests that triggered a WAF rule that was passed."}, - {"field_name": "AttackReqBodyBytesTotal", "type": "Counter", "metric_name": "attack_req_body_bytes_total", "extra_labels": [], "help": "Total body bytes received from requests that triggered a WAF rule."}, - {"field_name": "AttackReqHeaderBytesTotal", "type": "Counter", "metric_name": "attack_req_header_bytes_total", "extra_labels": [], "help": "Total header bytes received from requests that triggered a WAF rule."}, - {"field_name": "AttackRespSynthBytesTotal", "type": "Counter", "metric_name": "attack_resp_synth_bytes_total", "extra_labels": [], "help": "Total bytes delivered for requests that triggered a WAF rule and returned a synthetic response."}, - {"field_name": "BackendReqBodyBytesTotal", "type": "Counter", "metric_name": "bereq_body_bytes_total", "extra_labels": [], "help": "Total body bytes sent to origin."}, - {"field_name": "BackendReqHeaderBytesTotal", "type": "Counter", "metric_name": "bereq_header_bytes_total", "extra_labels": [], "help": "Total header bytes sent to origin."}, - {"field_name": "BilledBodyBytesTotal", "type": "Counter", "metric_name": "billed_body_bytes_total", "extra_labels": [], "help": "TODO"}, - {"field_name": "BilledHeaderBytesTotal", "type": "Counter", "metric_name": "billed_header_bytes_total", "extra_labels": [], "help": "TODO"}, - {"field_name": "BilledTotal", "type": "Counter", "metric_name": "billed_total", "extra_labels": [], "help": "TODO"}, - {"field_name": "BlacklistedTotal", "type": "Counter", "metric_name": "blacklist_total", "extra_labels": [], "help": "TODO"}, - {"field_name": "BodySizeTotal", "type": "Counter", "metric_name": "body_size_total", "extra_labels": [], "help": "Total body bytes delivered (alias for resp_body_bytes)."}, - {"field_name": "ComputeBackendReqBodyBytesTotal", "type": "Counter", "metric_name": "compute_bereq_body_bytes_total", "extra_labels": [], "help": "Total body bytes sent to backends (origins) by Compute@Edge."}, - {"field_name": "ComputeBackendReqErrorsTotal", "type": "Counter", "metric_name": "compute_bereq_errors_total", "extra_labels": [], "help": "Number of backend request errors, including timeouts."}, - {"field_name": "ComputeBackendReqHeaderBytesTotal", "type": "Counter", "metric_name": "compute_bereq_header_bytes_total", "extra_labels": [], "help": "Total header bytes sent to backends (origins) by Compute@Edge."}, - {"field_name": "ComputeBackendReqTotal", "type": "Counter", "metric_name": "compute_bereq_total", "extra_labels": [], "help": "Number of backend requests started."}, - {"field_name": "ComputeBackendRespBodyBytesTotal", "type": "Counter", "metric_name": "compute_beresp_body_bytes_total", "extra_labels": [], "help": "Total body bytes received from backends (origins) by Compute@Edge."}, - {"field_name": "ComputeBackendRespHeaderBytesTotal", "type": "Counter", "metric_name": "compute_beresp_header_bytes_total", "extra_labels": [], "help": "Total header bytes received from backends (origins) by Compute@Edge."}, - {"field_name": "ComputeExecutionTimeTotal", "type": "Counter", "metric_name": "compute_execution_time_total", "extra_labels": [], "help": "The amount of active CPU time used to process your requests (in seconds)."}, - {"field_name": "ComputeGlobalsLimitExceededTotal", "type": "Counter", "metric_name": "compute_globals_limit_exceeded_total", "extra_labels": [], "help": "Number of times a guest exceeded its globals limit."}, - {"field_name": "ComputeGuestErrorsTotal", "type": "Counter", "metric_name": "compute_guest_errors_total", "extra_labels": [], "help": "Number of times a service experienced a guest code error."}, - {"field_name": "ComputeHeapLimitExceededTotal", "type": "Counter", "metric_name": "compute_heap_limit_exceeded_total", "extra_labels": [], "help": "Number of times a guest exceeded its heap limit."}, - {"field_name": "ComputeRAMUsedBytesTotal", "type": "Counter", "metric_name": "compute_ram_used_bytes_total", "extra_labels": [], "help": "The amount of RAM used for your site by Fastly."}, - {"field_name": "ComputeReqBodyBytesTotal", "type": "Counter", "metric_name": "compute_req_body_bytes_total", "extra_labels": [], "help": "Total body bytes received by Compute@Edge."}, - {"field_name": "ComputeReqHeaderBytesTotal", "type": "Counter", "metric_name": "compute_req_header_bytes_total", "extra_labels": [], "help": "Total header bytes received by Compute@Edge."}, - {"field_name": "ComputeRequestsTotal", "type": "Counter", "metric_name": "compute_requests_total", "extra_labels": [], "help": "The total number of requests that were received for your site by Fastly."}, - {"field_name": "ComputeRequestTimeTotal", "type": "Counter", "metric_name": "compute_request_time_total", "extra_labels": [], "help": "The total amount of time used to process your requests, including active CPU time (in seconds)."}, - {"field_name": "ComputeResourceLimitExceedTotal", "type": "Counter", "metric_name": "compute_resource_limit_exceeded_total", "extra_labels": [], "help": "Number of times a guest exceeded its resource limit, includes heap, stack, globals, and code execution timeout."}, - {"field_name": "ComputeRespBodyBytesTotal", "type": "Counter", "metric_name": "compute_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes sent from Compute@Edge to end user."}, - {"field_name": "ComputeRespHeaderBytesTotal", "type": "Counter", "metric_name": "compute_resp_header_bytes_total", "extra_labels": [], "help": "Total header bytes sent from Compute@Edge to end user."}, - {"field_name": "ComputeRespStatusTotal", "type": "Counter", "metric_name": "compute_resp_status_total", "extra_labels": ["status_group"], "help": "Number of responses delivered delivered by Compute@Edge, by status code group."}, - {"field_name": "ComputeRuntimeErrorsTotal", "type": "Counter", "metric_name": "compute_runtime_errors_total", "extra_labels": [], "help": "Number of times a service experienced a guest runtime error."}, - {"field_name": "ComputeStackLimitExceededTotal", "type": "Counter", "metric_name": "compute_stack_limit_exceeded_total", "extra_labels": [], "help": "Number of times a guest exceeded its stack limit."}, - {"field_name": "DeliverSubCountTotal", "type": "Counter", "metric_name": "deliver_sub_count_total", "extra_labels": [], "help": "Number of executions of the 'deliver' Varnish subroutine."}, - {"field_name": "DeliverSubTimeTotal", "type": "Counter", "metric_name": "deliver_sub_time_total", "extra_labels": [], "help": "Time spent inside the 'deliver' Varnish subroutine (in seconds)."}, - {"field_name": "EdgeRespBodyBytesTotal", "type": "Counter", "metric_name": "edge_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes delivered from Fastly to the end user."}, - {"field_name": "EdgeRespHeaderBytesTotal", "type": "Counter", "metric_name": "edge_resp_header_bytes_total", "extra_labels": [], "help": "Total header bytes delivered from Fastly to the end user."}, - {"field_name": "EdgeTotal", "type": "Counter", "metric_name": "edge_total", "extra_labels": [], "help": "Number of requests sent by end users to Fastly."}, - {"field_name": "ErrorsTotal", "type": "Counter", "metric_name": "errors_total", "extra_labels": [], "help": "Number of cache errors."}, - {"field_name": "ErrorSubCountTotal", "type": "Counter", "metric_name": "error_sub_count_total", "extra_labels": [], "help": "Number of executions of the 'error' Varnish subroutine."}, - {"field_name": "ErrorSubTimeTotal", "type": "Counter", "metric_name": "error_sub_time_total", "extra_labels": [], "help": "Time spent inside the 'error' Varnish subroutine (in seconds)."}, - {"field_name": "FetchSubCountTotal", "type": "Counter", "metric_name": "fetch_sub_count_total", "extra_labels": [], "help": "Number of executions of the 'fetch' Varnish subroutine."}, - {"field_name": "FetchSubTimeTotal", "type": "Counter", "metric_name": "fetch_sub_time_total", "extra_labels": [], "help": "Time spent inside the 'fetch' Varnish subroutine (in seconds)."}, - {"field_name": "HashSubCountTotal", "type": "Counter", "metric_name": "hash_sub_count_total", "extra_labels": [], "help": "Number of executions of the 'hash' Varnish subroutine."}, - {"field_name": "HashSubTimeTotal", "type": "Counter", "metric_name": "hash_sub_time_total", "extra_labels": [], "help": "Time spent inside the 'hash' Varnish subroutine (in seconds)."}, - {"field_name": "HeaderSizeTotal", "type": "Counter", "metric_name": "header_size_total", "extra_labels": [], "help": "Total header bytes delivered (alias for resp_header_bytes)."}, - {"field_name": "HitRespBodyBytesTotal", "type": "Counter", "metric_name": "hit_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes delivered for cache hits."}, - {"field_name": "HitsTimeTotal", "type": "Counter", "metric_name": "hits_time_total", "extra_labels": [], "help": "Total amount of time spent processing cache hits (in seconds)."}, - {"field_name": "HitsTotal", "type": "Counter", "metric_name": "hits_total", "extra_labels": [], "help": "Number of cache hits."}, - {"field_name": "HitSubCountTotal", "type": "Counter", "metric_name": "hit_sub_count_total", "extra_labels": [], "help": "Number of executions of the 'hit' Varnish subroutine."}, - {"field_name": "HitSubTimeTotal", "type": "Counter", "metric_name": "hit_sub_time_total", "extra_labels": [], "help": "Time spent inside the 'hit' Varnish subroutine (in seconds)."}, - {"field_name": "HTTP2Total", "type": "Counter", "metric_name": "http2_total", "extra_labels": [], "help": "Number of requests received over HTTP2."}, - {"field_name": "ImgOptoRespBodyBytesTotal", "type": "Counter", "metric_name": "imgopto_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes delivered from the Fastly Image Optimizer service."}, - {"field_name": "ImgOptoRespHeaderBytesTotal", "type": "Counter", "metric_name": "imgopto_resp_header_bytes_total", "extra_labels": [], "help": "Total header bytes delivered from the Fastly Image Optimizer service."}, - {"field_name": "ImgOptoShieldRespBodyBytesTotal", "type": "Counter", "metric_name": "imgopto_shield_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes delivered via a shield from the Fastly Image Optimizer service."}, - {"field_name": "ImgOptoShieldRespHeaderBytesTotal", "type": "Counter", "metric_name": "imgopto_shield_resp_header_bytes_total", "extra_labels": [], "help": "Total header bytes delivered via a shield from the Fastly Image Optimizer service."}, - {"field_name": "ImgOptoShieldTotal", "type": "Counter", "metric_name": "imgopto_shield_total", "extra_labels": [], "help": "Number of responses delivered via a shield from the Fastly Image Optimizer service."}, - {"field_name": "ImgOptoTotal", "type": "Counter", "metric_name": "imgopto_total", "extra_labels": [], "help": "Number of responses that came from the Fastly Image Optimizer service."}, - {"field_name": "ImgOptoTransformRespBodyBytesTotal", "type": "Counter", "metric_name": "imgopto_transform_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes of transforms delivered from the Fastly Image Optimizer service."}, - {"field_name": "ImgOptoTransformRespHeaderBytesTotal", "type": "Counter", "metric_name": "imgopto_transform_resp_header_bytes_total", "extra_labels": [], "help": "Total header bytes of transforms delivered from the Fastly Image Optimizer service."}, - {"field_name": "ImgOptoTransformTotal", "type": "Counter", "metric_name": "imgopto_transforms_total", "extra_labels": [], "help": "Total transforms performed by the Fastly Image Optimizer service."}, - {"field_name": "ImgVideoFramesTotal", "type": "Counter", "metric_name": "imgvideo_frames_total", "extra_labels": [], "help": "Number of video frames that came from the Fastly Image Optimizer service."}, - {"field_name": "ImgVideoRespBodyBytesTotal", "type": "Counter", "metric_name": "imgvideo_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes of video delivered from the Fastly Image Optimizer service."}, - {"field_name": "ImgVideoRespHeaderBytesTotal", "type": "Counter", "metric_name": "imgvideo_resp_header_bytes_total", "extra_labels": [], "help": "Total header bytes of video delivered from the Fastly Image Optimizer service."}, - {"field_name": "ImgVideoShieldFramesTotal", "type": "Counter", "metric_name": "imgvideo_shield_frames_total", "extra_labels": [], "help": "Number of video frames delivered via a shield from the Fastly Image Optimizer service."}, - {"field_name": "ImgVideoShieldRespBodyBytesTotal", "type": "Counter", "metric_name": "imgvideo_shield_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes of video delivered via a shield from the Fastly Image Optimizer service."}, - {"field_name": "ImgVideoShieldRespHeaderBytesTotal", "type": "Counter", "metric_name": "imgvideo_shield_resp_header_bytes_total", "extra_labels": [], "help": "Total header bytes of video delivered via a shield from the Fastly Image Optimizer service."}, - {"field_name": "ImgVideoShieldTotal", "type": "Counter", "metric_name": "imgvideo_shield_total", "extra_labels": [], "help": "Number of video responses that came via a shield from the Fastly Image Optimizer service."}, - {"field_name": "ImgVideoTotal", "type": "Counter", "metric_name": "imgvideo_total", "extra_labels": [], "help": "Number of video responses that came via a shield from the Fastly Image Optimizer service."}, - {"field_name": "IPv6Total", "type": "Counter", "metric_name": "ipv6_total", "extra_labels": [], "help": "Number of requests that were received over IPv6."}, - {"field_name": "LogBytesTotal", "type": "Counter", "metric_name": "log_bytes_total", "extra_labels": [], "help": "Total log bytes sent."}, - {"field_name": "LoggingTotal", "type": "Counter", "metric_name": "logging_total", "extra_labels": [], "help": "Number of log lines sent."}, - {"field_name": "MissDurationSeconds", "type": "Histogram", "metric_name": "miss_duration_seconds", "extra_labels": [], "help": "Histogram of time spent processing cache misses (in seconds).", "buckets": [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 4, 8, 16, 32, 60]}, - {"field_name": "MissesTotal", "type": "Counter", "metric_name": "miss_total", "extra_labels": [], "help": "Number of cache misses."}, - {"field_name": "MissRespBodyBytesTotal", "type": "Counter", "metric_name": "miss_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes delivered for cache misses."}, - {"field_name": "MissSubCountTotal", "type": "Counter", "metric_name": "miss_sub_count_total", "extra_labels": [], "help": "Number of executions of the 'miss' Varnish subroutine."}, - {"field_name": "MissSubTimeTotal", "type": "Counter", "metric_name": "miss_sub_time_total", "extra_labels": [], "help": "Time spent inside the 'miss' Varnish subroutine (in seconds)."}, - {"field_name": "MissTimeTotal", "type": "Counter", "metric_name": "miss_time_total", "extra_labels": [], "help": "Total amount of time spent processing cache misses (in seconds)."}, - {"field_name": "ObjectSizeBytes", "type": "Histogram", "metric_name": "object_size_bytes", "extra_labels": [], "help": "Histogram of count of objects served, bucketed by object size range.", "buckets": [1024, 10240, 102400, 1024000, 10240000, 102400000, 1024000000]}, - {"field_name": "OriginFetchBodyBytesTotal", "type": "Counter", "metric_name": "origin_fetch_body_bytes_total", "extra_labels": [], "help": "Total request body bytes sent to origin."}, - {"field_name": "OriginFetchesTotal", "type": "Counter", "metric_name": "origin_fetches_total", "extra_labels": [], "help": "Number of requests sent to origin."}, - {"field_name": "OriginFetchHeaderBytesTotal", "type": "Counter", "metric_name": "origin_fetch_header_bytes_total", "extra_labels": [], "help": "Total request header bytes sent to origin."}, - {"field_name": "OriginFetchRespBodyBytesTotal", "type": "Counter", "metric_name": "origin_fetch_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes received from origin."}, - {"field_name": "OriginFetchRespHeaderBytesTotal", "type": "Counter", "metric_name": "origin_fetch_resp_header_bytes_total", "extra_labels": [], "help": "Total header bytes received from origin."}, - {"field_name": "OriginRevalidationsTotal", "type": "Counter", "metric_name": "origin_revalidations_total", "extra_labels": [], "help": "Number of responses received from origin with a 304 status code in response to an If-Modified-Since or If-None-Match request. Under regular scenarios, a revalidation will imply a cache hit. However, if using Fastly Image Optimizer or segmented caching this may result in a cache miss."}, - {"field_name": "OTFPDeliverTimeTotal", "type": "Counter", "metric_name": "otfp_total", "extra_labels": [], "help": "Number of responses that came from the Fastly On-the-Fly Packager."}, - {"field_name": "OTFPManifestTotal", "type": "Counter", "metric_name": "otfp_deliver_time_total", "extra_labels": [], "help": "Total amount of time spent delivering a response from the Fastly On-the-Fly Packager (in seconds)."}, - {"field_name": "OTFPRespBodyBytesTotal", "type": "Counter", "metric_name": "otfp_manifests_total", "extra_labels": [], "help": "Number of responses that were manifest files from the Fastly On-the-Fly Packager."}, - {"field_name": "OTFPRespHeaderBytesTotal", "type": "Counter", "metric_name": "otfp_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes delivered from the Fastly On-the-Fly Packager."}, - {"field_name": "OTFPShieldRespBodyBytesTotal", "type": "Counter", "metric_name": "otfp_resp_header_bytes_total", "extra_labels": [], "help": "Total header bytes delivered from the Fastly On-the-Fly Packager."}, - {"field_name": "OTFPShieldRespHeaderBytesTotal", "type": "Counter", "metric_name": "otfp_shield_total", "extra_labels": [], "help": "Number of responses delivered from the Fastly On-the-Fly Packager"}, - {"field_name": "OTFPShieldTimeTotal", "type": "Counter", "metric_name": "otfp_shield_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes delivered via a shield for the Fastly On-the-Fly Packager."}, - {"field_name": "OTFPShieldTotal", "type": "Counter", "metric_name": "otfp_shield_resp_header_bytes_total", "extra_labels": [], "help": "Total header bytes delivered via a shield for the Fastly On-the-Fly Packager."}, - {"field_name": "OTFPTotal", "type": "Counter", "metric_name": "otfp_shield_time_total", "extra_labels": [], "help": "Total amount of time spent delivering a response via a shield from the Fastly On-the-Fly Packager (in seconds)."}, - {"field_name": "OTFPTransformRespBodyBytesTotal", "type": "Counter", "metric_name": "otfp_transforms_total", "extra_labels": [], "help": "Number of transforms performed by the Fastly On-the-Fly Packager."}, - {"field_name": "OTFPTransformRespHeaderBytesTotal", "type": "Counter", "metric_name": "otfp_transform_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes of transforms delivered from the Fastly On-the-Fly Packager."}, - {"field_name": "OTFPTransformTimeTotal", "type": "Counter", "metric_name": "otfp_transform_resp_header_bytes_total", "extra_labels": [], "help": "Total body bytes of transforms delivered from the Fastly On-the-Fly Packager."}, - {"field_name": "OTFPTransformTotal", "type": "Counter", "metric_name": "otfp_transform_time_total", "extra_labels": [], "help": "Total amount of time spent performing transforms from the Fastly On-the-Fly Packager."}, - {"field_name": "PassesTotal", "type": "Counter", "metric_name": "pass_total", "extra_labels": [], "help": "Number of requests that passed through the CDN without being cached."}, - {"field_name": "PassRespBodyBytesTotal", "type": "Counter", "metric_name": "pass_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes delivered for cache passes."}, - {"field_name": "PassSubCountTotal", "type": "Counter", "metric_name": "pass_sub_count_total", "extra_labels": [], "help": "Number of executions of the 'pass' Varnish subroutine."}, - {"field_name": "PassSubTimeTotal", "type": "Counter", "metric_name": "pass_sub_time_total", "extra_labels": [], "help": "Time spent inside the 'pass' Varnish subroutine (in seconds)."}, - {"field_name": "PassTimeTotal", "type": "Counter", "metric_name": "pass_time_total", "extra_labels": [], "help": "Total amount of time spent processing cache passes (in seconds)."}, - {"field_name": "PCITotal", "type": "Counter", "metric_name": "pci_total", "extra_labels": [], "help": "Number of responses with the PCI flag turned on."}, - {"field_name": "Pipe", "type": "Counter", "metric_name": "pipe", "extra_labels": [], "help": "Pipe operations performed."}, - {"field_name": "PipeSubCountTotal", "type": "Counter", "metric_name": "pipe_sub_count_total", "extra_labels": [], "help": "Number of executions of the 'pipe' Varnish subroutine."}, - {"field_name": "PipeSubTimeTotal", "type": "Counter", "metric_name": "pipe_sub_time_total", "extra_labels": [], "help": "Time spent inside the 'pipe' Varnish subroutine (in seconds)."}, - {"field_name": "PredeliverSubCountTotal", "type": "Counter", "metric_name": "predeliver_sub_count_total", "extra_labels": [], "help": "Number of executions of the 'predeliver' Varnish subroutine."}, - {"field_name": "PredeliverSubTimeTotal", "type": "Counter", "metric_name": "predeliver_sub_time_total", "extra_labels": [], "help": "Time spent inside the 'predeliver' Varnish subroutine (in seconds)."}, - {"field_name": "PrehashSubCountTotal", "type": "Counter", "metric_name": "prehash_sub_count_total", "extra_labels": [], "help": "Number of executions of the 'prehash' Varnish subroutine."}, - {"field_name": "PrehashSubTimeTotal", "type": "Counter", "metric_name": "prehash_sub_time_total", "extra_labels": [], "help": "Time spent inside the 'prehash' Varnish subroutine (in seconds)."}, - {"field_name": "RecvSubCountTotal", "type": "Counter", "metric_name": "recv_sub_count_total", "extra_labels": [], "help": "Number of executions of the 'recv' Varnish subroutine."}, - {"field_name": "RecvSubTimeTotal", "type": "Counter", "metric_name": "recv_sub_time_total", "extra_labels": [], "help": "Time spent inside the 'recv' Varnish subroutine (in seconds)."}, - {"field_name": "ReqBodyBytesTotal", "type": "Counter", "metric_name": "req_body_bytes_total", "extra_labels": [], "help": "Total body bytes received."}, - {"field_name": "ReqHeaderBytesTotal", "type": "Counter", "metric_name": "req_header_bytes_total", "extra_labels": [], "help": "Total header bytes received."}, - {"field_name": "RequestsTotal", "type": "Counter", "metric_name": "requests_total", "extra_labels": [], "help": "Number of requests processed."}, - {"field_name": "RespBodyBytesTotal", "type": "Counter", "metric_name": "resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes delivered."}, - {"field_name": "RespHeaderBytesTotal", "type": "Counter", "metric_name": "resp_header_bytes_total", "extra_labels": [], "help": "Total header bytes delivered."}, - {"field_name": "RestartTotal", "type": "Counter", "metric_name": "restarts_total", "extra_labels": [], "help": "Number of restarts performed."}, - {"field_name": "SegBlockOriginFetchesTotal", "type": "Counter", "metric_name": "segblock_origin_fetches_total", "extra_labels": [], "help": "Number of Range requests to origin for segments of resources when using segmented caching."}, - {"field_name": "SegBlockShieldFetchesTotal", "type": "Counter", "metric_name": "segblock_shield_fetches_total", "extra_labels": [], "help": "Number of Range requests to a shield for segments of resources when using segmented caching."}, - {"field_name": "ShieldFetchBodyBytesTotal", "type": "Counter", "metric_name": "shield_fetch_body_bytes_total", "extra_labels": [], "help": "Total request body bytes sent to a shield."}, - {"field_name": "ShieldFetchesTotal", "type": "Counter", "metric_name": "shield_fetches_total", "extra_labels": [], "help": "Number of requests made from one Fastly data center to another, as part of shielding."}, - {"field_name": "ShieldFetchHeaderBytesTotal", "type": "Counter", "metric_name": "shield_fetch_header_bytes_total", "extra_labels": [], "help": "Total request header bytes sent to a shield."}, - {"field_name": "ShieldFetchRespBodyBytesTotal", "type": "Counter", "metric_name": "shield_fetch_resp_body_bytes_total", "extra_labels": [], "help": "Total response body bytes sent from a shield to the edge."}, - {"field_name": "ShieldFetchRespHeaderBytesTotal", "type": "Counter", "metric_name": "shield_fetch_resp_header_bytes_total", "extra_labels": [], "help": "Total response header bytes sent from a shield to the edge."}, - {"field_name": "ShieldRespBodyBytesTotal", "type": "Counter", "metric_name": "shield_resp_body_bytes_total", "extra_labels": [], "help": "Total body bytes delivered via a shield."}, - {"field_name": "ShieldRespHeaderBytesTotal", "type": "Counter", "metric_name": "shield_resp_header_bytes_total", "extra_labels": [], "help": "Total header bytes delivered via a shield."}, - {"field_name": "ShieldRevalidationsTotal", "type": "Counter", "metric_name": "shield_revalidations_total", "extra_labels": [], "help": "Number of responses received from origin with a 304 status code, in response to an If-Modified-Since or If-None-Match request to a shield. Under regular scenarios, a revalidation will imply a cache hit. However, if using segmented caching this may result in a cache miss."}, - {"field_name": "ShieldTotal", "type": "Counter", "metric_name": "shield_total", "extra_labels": [], "help": "Number of requests from edge to the shield POP."}, - {"field_name": "StatusCodeTotal", "type": "Counter", "metric_name": "status_code_total", "extra_labels": ["status_code"], "help": "Number of responses sent with status code 500 (Internal Server Error)."}, - {"field_name": "StatusGroupTotal", "type": "Counter", "metric_name": "status_group_total", "extra_labels": ["status_group"], "help": "Number of 'Client Error' category status codes delivered."}, - {"field_name": "SynthsTotal", "type": "Counter", "metric_name": "synth_total", "extra_labels": [], "help": "TODO"}, - {"field_name": "TLSTotal", "type": "Counter", "metric_name": "tls_total", "extra_labels": ["tls_version"], "help": "Number of requests that were received over TLS."}, - {"field_name": "UncacheableTotal", "type": "Counter", "metric_name": "uncacheable_total", "extra_labels": [], "help": "Number of requests that were designated uncachable."}, - {"field_name": "VideoTotal", "type": "Counter", "metric_name": "video_total", "extra_labels": [], "help": "Number of responses with the video segment or video manifest MIME type (i.e., application/x-mpegurl, application/vnd.apple.mpegurl, application/f4m, application/dash+xml, application/vnd.ms-sstr+xml, ideo/mp2t, audio/aac, video/f4f, video/x-flv, video/mp4, audio/mp4)."}, - {"field_name": "WAFBlockedTotal", "type": "Counter", "metric_name": "waf_blocked_total", "extra_labels": [], "help": "Number of requests that triggered a WAF rule and were blocked."}, - {"field_name": "WAFLoggedTotal", "type": "Counter", "metric_name": "waf_logged_total", "extra_labels": [], "help": "Number of requests that triggered a WAF rule and were logged."}, - {"field_name": "WAFPassedTotal", "type": "Counter", "metric_name": "waf_passed_total", "extra_labels": [], "help": "Number of requests that triggered a WAF rule and were passed."} -] \ No newline at end of file diff --git a/cmd/fieldgen/main.go b/cmd/fieldgen/main.go deleted file mode 100644 index 06a18ef..0000000 --- a/cmd/fieldgen/main.go +++ /dev/null @@ -1,338 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "flag" - "fmt" - "go/format" - "io/ioutil" - "os" - "strings" -) - -func main() { - if err := exec(); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } -} - -func exec() error { - fs := flag.NewFlagSet("fieldgen", flag.ExitOnError) - var ( - metricsFile = fs.String("metrics", "exporter_metrics.json", "JSON file containing metric definitions") - fieldsFile = fs.String("fields", "api_fields.json", "JSON file containing API field definitions") - mappingsFile = fs.String("mappings", "mappings.json", "JSON file containing metric-to-field mappings") - invalid = fs.Bool("invalid", false, "skip validations") - ) - fs.Parse(os.Args[1:]) - - var metrics []exporterMetric - if err := readJSON(*metricsFile, &metrics); err != nil { - return err - } - - if !*invalid { - var ( - noHelp []string - todoHelp []string - ) - for _, m := range metrics { - if m.Help == "" { - noHelp = append(noHelp, m.FieldName) - } - if strings.Contains(m.Help, "TODO") { - todoHelp = append(todoHelp, m.FieldName) - } - } - if len(noHelp) > 0 { - return fmt.Errorf("metrics with undefined Help strings: %s", strings.Join(noHelp, ", ")) - } - if len(todoHelp) > 0 { - fmt.Fprintf(os.Stderr, "warning: metrics with TODO Help strings: %s\n", strings.Join(todoHelp, ", ")) - } - } - - var fields []apiField - if err := readJSON(*fieldsFile, &fields); err != nil { - return err - } - - var mappings []mapping - if err := readJSON(*mappingsFile, &mappings); err != nil { - return err - } - - if !*invalid { - unmappedAPIFields := map[string]bool{} - for _, f := range fields { - unmappedAPIFields[f.FieldName] = true - } - for _, m := range mappings { - delete(unmappedAPIFields, m.APIField) - for _, pair := range m.APIFieldLabels { - delete(unmappedAPIFields, pair[0]) - } - for _, f := range m.APIFieldSizes { - delete(unmappedAPIFields, f) - } - } - if len(unmappedAPIFields) > 0 { - var names []string - for f := range unmappedAPIFields { - names = append(names, f) - } - return fmt.Errorf("unmapped API fields: %s", strings.Join(names, ", ")) - } - } - - buf := bytes.NewBuffer(nil) - - fmt.Fprintln(buf, "// Code generated by fieldgen; DO NOT EDIT.") - fmt.Fprintln(buf) - fmt.Fprintln(buf, "package gen") - fmt.Fprintln(buf) - fmt.Fprintln(buf, importBlock) - fmt.Fprintln(buf) - fmt.Fprintln(buf, apiResponseBlock) - fmt.Fprintln(buf) - fmt.Fprintln(buf, "// Datacenter models the per-datacenter portion of the rt.fastly.com response.") - fmt.Fprintln(buf, "type Datacenter struct {") - for _, f := range fields { - fmt.Fprintf(buf, "\t%s %s `json:\"%s\"`\n", f.FieldName, f.Type, f.Key) - } - fmt.Fprintln(buf, "}") - fmt.Fprintln(buf) - fmt.Fprintln(buf, "// Metrics collects all of the Prometheus metrics exported by this service.") - fmt.Fprintln(buf, "type Metrics struct {") - fmt.Fprintln(buf, "\tRealtimeAPIRequestsTotal *prometheus.CounterVec") - fmt.Fprintln(buf, "\tServiceInfo *prometheus.GaugeVec") - fmt.Fprintln(buf, "\tLastSuccessfulResponse *prometheus.GaugeVec") - for _, m := range metrics { - fmt.Fprintf(buf, "\t%s *prometheus.%sVec\n", m.FieldName, m.Type) - } - fmt.Fprintln(buf, "}") - fmt.Fprintln(buf) - fmt.Fprintln(buf, "// NewMetrics returns a new set of metrics registered to the registerer.") - fmt.Fprintln(buf, "// Only metrics whose names pass the name filter are registered.") - fmt.Fprintln(buf, "func NewMetrics(namespace, subsystem string, nameFilter filter.Filter, r prometheus.Registerer) *Metrics {") - fmt.Fprintln(buf, "\tm := Metrics{") - fmt.Fprintln(buf, "\t\t"+`RealtimeAPIRequestsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "realtime_api_requests_total", Help: "Total requests made to the real-time stats API.", }, []string{"service_id", "service_name", "result"}),`) - fmt.Fprintln(buf, "\t\t"+`ServiceInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: namespace, Subsystem: subsystem, Name: "service_info", Help: "Static gauge with service ID, name, and version information.", }, []string{"service_id", "service_name", "service_version"}),`) - fmt.Fprintln(buf, "\t\t"+`LastSuccessfulResponse: prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: namespace, Subsystem: subsystem, Name: "last_successful_response", Help: "Unix timestamp of the last successful response received from the real-time stats API.", }, []string{"service_id", "service_name"}),`) - for _, m := range metrics { - fmt.Fprintf(buf, "\t\t%s: %s,\n", m.FieldName, m.create()) - } - fmt.Fprintln(buf, "\t}") - fmt.Fprintln(buf) - fmt.Fprintf(buf, "%s\n", registerBlock) - fmt.Fprintln(buf, "\treturn &m") - fmt.Fprintln(buf, "}") - fmt.Fprintln(buf) - fmt.Fprintln(buf, getNameBlock) - fmt.Fprintln(buf) - fmt.Fprintln(buf, "// Process updates the metrics with data from the API response.") - fmt.Fprintln(buf, "func Process(response *APIResponse, serviceID, serviceName, serviceVersion string, m *Metrics) {") - fmt.Fprintln(buf, "\tfor _, d := range response.Data {") - fmt.Fprintln(buf, "\t\tfor datacenter, stats := range d.Datacenter {") - for _, m := range mappings { - switch m.Kind { - case "Counter": - fmt.Fprintf(buf, "\t\t\tm.%s.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.%s))\n", m.ExporterMetric, m.APIField) - case "Counter1000": - fmt.Fprintf(buf, "\t\t\tm.%s.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.%s) / 10000.0)\n", m.ExporterMetric, m.APIField) - case "CounterLabels": - for _, pair := range m.APIFieldLabels { - fmt.Fprintf(buf, "\t\t\tm.%s.WithLabelValues(serviceID, serviceName, datacenter, \"%s\").Add(float64(stats.%s))\n", m.ExporterMetric, pair[1], pair[0]) - } - case "Histogram": - fmt.Fprintf(buf, "\t\t\tprocessHistogram(stats.%s, m.%s.WithLabelValues(serviceID, serviceName, datacenter))\n", m.APIField, m.ExporterMetric) - case "ObjectSize": - fmt.Fprintf(buf, "\t\t\tprocessObjectSizes(stats.ObjectSize1k, stats.ObjectSize10k, stats.ObjectSize100k, stats.ObjectSize1m, stats.ObjectSize10m, stats.ObjectSize100m, stats.ObjectSize1g, m.%s.WithLabelValues(serviceID, serviceName, datacenter))\n", m.ExporterMetric) // hacky hack - case "Ignored": - // - default: - fmt.Fprintf(buf, "\t\t\t// %s: unknown mapping kind %q\n", m.ExporterMetric, m.Kind) - } - } - - fmt.Fprintln(buf, "\t\t}") - fmt.Fprintln(buf, "\t}") - fmt.Fprintln(buf, "}") - fmt.Fprintln(buf) - fmt.Fprintln(buf, processBlock) - - return writeGoFile("gen.go", buf.Bytes()) -} - -func readJSON(filename string, data interface{}) error { - f, err := os.Open(filename) - if err != nil { - return err - } - defer f.Close() - - return json.NewDecoder(f).Decode(data) -} - -type exporterMetric struct { - FieldName string `json:"field_name"` - Type string `json:"type"` - MetricName string `json:"metric_name"` - ExtraLabels []string `json:"extra_labels"` - Help string `json:"help"` - Buckets []float64 `json:"buckets"` -} - -var standardLabels = []string{"service_id", "service_name", "datacenter"} - -func quoteList(a []string) string { - b := make([]string, len(a)) - for i, s := range a { - b[i] = `"` + s + `"` - } - return strings.Join(b, ", ") -} - -func (m exporterMetric) create() string { - var sb strings.Builder - - fmt.Fprintf(&sb, `prometheus.New%sVec(prometheus.%sOpts{`, m.Type, m.Type) - fmt.Fprintf(&sb, `Namespace: namespace`) - fmt.Fprintf(&sb, `, Subsystem: subsystem`) - fmt.Fprintf(&sb, `, Name: "%s"`, m.MetricName) - fmt.Fprintf(&sb, `, Help: "%s"`, m.Help) - if len(m.Buckets) > 0 { - fmt.Fprintf(&sb, `, Buckets: []float64{%s}`, renderFloats(m.Buckets)) - } - fmt.Fprintf(&sb, `}, []string{%s}`, quoteList(append(standardLabels, m.ExtraLabels...))) - fmt.Fprintf(&sb, `)`) - - return sb.String() -} - -type apiField struct { - FieldName string `json:"field_name"` - Type string `json:"type"` - Key string `json:"key"` -} - -type mapping struct { - ExporterMetric string `json:"exporter_metric"` - Kind string `json:"kind"` - APIField string `json:"api_field"` // kind=Counter, kind=Histogram - APIFieldLabels [][2]string `json:"api_field_labels"` // Kind=CounterLabels - APIFieldSizes []string `json:"api_field_sizes"` // kind=ObjectSize -} - -func renderFloats(a []float64) string { - s := make([]string, len(a)) - for i, f := range a { - s[i] = fmt.Sprint(f) - } - return strings.Join(s, ", ") -} - -// -// -// - -const importBlock = ` -import ( - "fmt" - "reflect" - "regexp" - "strconv" - - "github.com/fastly/fastly-exporter/pkg/filter" - "github.com/prometheus/client_golang/prometheus" -)` - -var apiResponseBlock = strings.Replace(` -// APIResponse models the response from rt.fastly.com. It can get quite large; -// when there are lots of services being monitored, unmarshaling to this type is -// the CPU bottleneck of the program. -type APIResponse struct { - Timestamp uint64 ·json:"Timestamp"· - AggregateDelay int64 ·json:"AggregateDelay"· - Data []struct { - Datacenter map[string]Datacenter ·json:"datacenter"· - Aggregated Datacenter ·json:"aggregated"· - Recorded uint64 ·json:"recorded"· - } ·json:"Data"· - Error string ·json:"error"· -} -`, "·", "`", -1) - -const getNameBlock = ` -var descNameRegex = regexp.MustCompile("fqName: \"([^\"]+)\"") - -func getName(c prometheus.Collector) string { - d := make(chan *prometheus.Desc, 1) - c.Describe(d) - desc := (<-d).String() - matches := descNameRegex.FindAllStringSubmatch(desc, -1) - if len(matches) == 1 && len(matches[0]) == 2 { - return matches[0][1] - } - return "" -}` - -const processBlock = ` -func processHistogram(src map[string]uint64, obs prometheus.Observer) { - for str, count := range src { - ms, err := strconv.Atoi(str) - if err != nil { - continue - } - s := float64(ms) / 1e3 - for i := 0; i < int(count); i++ { - obs.Observe(s) - } - } -} - -func processObjectSizes(n1k, n10k, n100k, n1m, n10m, n100m, n1g uint64, obs prometheus.Observer) { - for v, n := range map[uint64]uint64{ - 1 * 1024: n1k, - 10 * 1024: n10k, - 100 * 1024: n100k, - 1 * 1000 * 1024: n1m, - 10 * 1000 * 1024: n10m, - 100 * 1000 * 1024: n100m, - 1000 * 1000 * 1024: n1g, - } { - for i := uint64(0); i < n; i++ { - obs.Observe(float64(v)) - } - } -}` - -const registerBlock = ` -for i, v := 0, reflect.ValueOf(m); i < v.NumField(); i++ { - c, ok := v.Field(i).Interface().(prometheus.Collector) - if !ok { - panic(fmt.Errorf("field %d/%d in Metrics type isn't a prometheus.Collector", i+1, v.NumField())) - } - if name := getName(c); !nameFilter.Permit(name) { - continue - } - if err := r.Register(c); err != nil { - panic(fmt.Errorf("error registering metric %d/%d: %w", i+1, v.NumField(), err)) - } -} -` - -func writeGoFile(filename string, source []byte) error { - src, err := format.Source(source) - if err != nil { - return fmt.Errorf("could not format source: %s", err) - } - - if err := ioutil.WriteFile(filename, src, 0o644); err != nil { - return fmt.Errorf("could not write source: %s", err) - } - - return nil -} diff --git a/cmd/fieldgen/mappings.json b/cmd/fieldgen/mappings.json deleted file mode 100644 index 68bcb24..0000000 --- a/cmd/fieldgen/mappings.json +++ /dev/null @@ -1,146 +0,0 @@ -[ - {"exporter_metric": "-", "kind": "Ignored", "api_field": "ObjectSizeOther"}, - {"exporter_metric": "AttackBlockedReqBodyBytesTotal", "kind": "Counter", "api_field": "AttackBlockedReqBodyBytes"}, - {"exporter_metric": "AttackBlockedReqHeaderBytesTotal", "kind": "Counter", "api_field": "AttackBlockedReqHeaderBytes"}, - {"exporter_metric": "AttackLoggedReqBodyBytesTotal", "kind": "Counter", "api_field": "AttackLoggedReqBodyBytes"}, - {"exporter_metric": "AttackLoggedReqHeaderBytesTotal", "kind": "Counter", "api_field": "AttackLoggedReqHeaderBytes"}, - {"exporter_metric": "AttackPassedReqBodyBytesTotal", "kind": "Counter", "api_field": "AttackPassedReqBodyBytes"}, - {"exporter_metric": "AttackPassedReqHeaderBytesTotal", "kind": "Counter", "api_field": "AttackPassedReqHeaderBytes"}, - {"exporter_metric": "AttackReqBodyBytesTotal", "kind": "Counter", "api_field": "AttackReqBodyBytes"}, - {"exporter_metric": "AttackReqHeaderBytesTotal", "kind": "Counter", "api_field": "AttackReqHeaderBytes"}, - {"exporter_metric": "AttackRespSynthBytesTotal", "kind": "Counter", "api_field": "AttackRespSynthBytes"}, - {"exporter_metric": "BackendReqBodyBytesTotal", "kind": "Counter", "api_field": "BackendReqBodyBytes"}, - {"exporter_metric": "BackendReqHeaderBytesTotal", "kind": "Counter", "api_field": "BackendReqHeaderBytes"}, - {"exporter_metric": "BilledBodyBytesTotal", "kind": "Counter", "api_field": "BilledBodyBytes"}, - {"exporter_metric": "BilledHeaderBytesTotal", "kind": "Counter", "api_field": "BilledHeaderBytes"}, - {"exporter_metric": "BilledTotal", "kind": "Counter", "api_field": "Billed"}, - {"exporter_metric": "BlacklistedTotal", "kind": "Counter", "api_field": "Blacklisted"}, - {"exporter_metric": "BodySizeTotal", "kind": "Counter", "api_field": "BodySize"}, - {"exporter_metric": "ComputeBackendReqBodyBytesTotal", "kind": "Counter", "api_field": "ComputeBackendReqBodyBytesTotal"}, - {"exporter_metric": "ComputeBackendReqErrorsTotal", "kind": "Counter", "api_field": "ComputeBackendReqErrorsTotal"}, - {"exporter_metric": "ComputeBackendReqHeaderBytesTotal", "kind": "Counter", "api_field": "ComputeBackendReqHeaderBytesTotal"}, - {"exporter_metric": "ComputeBackendReqTotal", "kind": "Counter", "api_field": "ComputeBackendReqTotal"}, - {"exporter_metric": "ComputeBackendRespBodyBytesTotal", "kind": "Counter", "api_field": "ComputeBackendRespBodyBytesTotal"}, - {"exporter_metric": "ComputeBackendRespHeaderBytesTotal", "kind": "Counter", "api_field": "ComputeBackendRespHeaderBytesTotal"}, - {"exporter_metric": "ComputeExecutionTimeTotal", "kind": "Counter1000", "api_field": "ComputeExecutionTimeMilliseconds"}, - {"exporter_metric": "ComputeGlobalsLimitExceededTotal", "kind": "Counter", "api_field": "ComputeGlobalsLimitExceededTotal"}, - {"exporter_metric": "ComputeGuestErrorsTotal", "kind": "Counter", "api_field": "ComputeGuestErrorsTotal"}, - {"exporter_metric": "ComputeHeapLimitExceededTotal", "kind": "Counter", "api_field": "ComputeHeapLimitExceededTotal"}, - {"exporter_metric": "ComputeRAMUsedBytesTotal", "kind": "Counter", "api_field": "ComputeRAMUsed"}, - {"exporter_metric": "ComputeReqBodyBytesTotal", "kind": "Counter", "api_field": "ComputeReqBodyBytesTotal"}, - {"exporter_metric": "ComputeReqHeaderBytesTotal", "kind": "Counter", "api_field": "ComputeReqHeaderBytesTotal"}, - {"exporter_metric": "ComputeRequestsTotal", "kind": "Counter", "api_field": "ComputeRequests"}, - {"exporter_metric": "ComputeRequestTimeTotal", "kind": "Counter1000", "api_field": "ComputeRequestTimeMilliseconds"}, - {"exporter_metric": "ComputeResourceLimitExceedTotal", "kind": "Counter", "api_field": "ComputeResourceLimitExceedTotal"}, - {"exporter_metric": "ComputeRespBodyBytesTotal", "kind": "Counter", "api_field": "ComputeRespBodyBytesTotal"}, - {"exporter_metric": "ComputeRespHeaderBytesTotal", "kind": "Counter", "api_field": "ComputeRespHeaderBytesTotal"}, - {"exporter_metric": "ComputeRespStatusTotal", "kind": "CounterLabels", "api_field_labels": [["ComputeRespStatus1xx", "1xx"], ["ComputeRespStatus2xx", "2xx"], ["ComputeRespStatus3xx", "3xx"], ["ComputeRespStatus4xx", "4xx"], ["ComputeRespStatus5xx", "5xx"]]}, - {"exporter_metric": "ComputeRuntimeErrorsTotal", "kind": "Counter", "api_field": "ComputeRuntimeErrorsTotal"}, - {"exporter_metric": "ComputeStackLimitExceededTotal", "kind": "Counter", "api_field": "ComputeStackLimitExceededTotal"}, - {"exporter_metric": "DeliverSubCountTotal", "kind": "Counter", "api_field": "DeliverSubCount"}, - {"exporter_metric": "DeliverSubTimeTotal", "kind": "Counter", "api_field": "DeliverSubTime"}, - {"exporter_metric": "EdgeRespBodyBytesTotal", "kind": "Counter", "api_field": "EdgeRespBodyBytes"}, - {"exporter_metric": "EdgeRespHeaderBytesTotal", "kind": "Counter", "api_field": "EdgeRespHeaderBytes"}, - {"exporter_metric": "EdgeTotal", "kind": "Counter", "api_field": "Edge"}, - {"exporter_metric": "ErrorsTotal", "kind": "Counter", "api_field": "Errors"}, - {"exporter_metric": "ErrorSubCountTotal", "kind": "Counter", "api_field": "ErrorSubCount"}, - {"exporter_metric": "ErrorSubTimeTotal", "kind": "Counter", "api_field": "ErrorSubTime"}, - {"exporter_metric": "FetchSubCountTotal", "kind": "Counter", "api_field": "FetchSubCount"}, - {"exporter_metric": "FetchSubTimeTotal", "kind": "Counter", "api_field": "FetchSubTime"}, - {"exporter_metric": "HashSubCountTotal", "kind": "Counter", "api_field": "HashSubCount"}, - {"exporter_metric": "HashSubTimeTotal", "kind": "Counter", "api_field": "HashSubTime"}, - {"exporter_metric": "HeaderSizeTotal", "kind": "Counter", "api_field": "HeaderSize"}, - {"exporter_metric": "HitRespBodyBytesTotal", "kind": "Counter", "api_field": "HitRespBodyBytes"}, - {"exporter_metric": "HitsTimeTotal", "kind": "Counter", "api_field": "HitsTime"}, - {"exporter_metric": "HitsTotal", "kind": "Counter", "api_field": "Hits"}, - {"exporter_metric": "HitSubCountTotal", "kind": "Counter", "api_field": "HitSubCount"}, - {"exporter_metric": "HitSubTimeTotal", "kind": "Counter", "api_field": "HitSubTime"}, - {"exporter_metric": "HTTP2Total", "kind": "Counter", "api_field": "HTTP2"}, - {"exporter_metric": "ImgOptoRespBodyBytesTotal", "kind": "Counter", "api_field": "ImgOptoRespBodyBytes"}, - {"exporter_metric": "ImgOptoRespHeaderBytesTotal", "kind": "Counter", "api_field": "ImgOptoRespHeaderBytes"}, - {"exporter_metric": "ImgOptoShieldRespBodyBytesTotal", "kind": "Counter", "api_field": "ImgOptoShieldRespBodyBytes"}, - {"exporter_metric": "ImgOptoShieldRespHeaderBytesTotal", "kind": "Counter", "api_field": "ImgOptoShieldRespHeaderBytes"}, - {"exporter_metric": "ImgOptoShieldTotal", "kind": "Counter", "api_field": "ImgOptoShield"}, - {"exporter_metric": "ImgOptoTotal", "kind": "Counter", "api_field": "ImgOpto"}, - {"exporter_metric": "ImgOptoTransformRespBodyBytesTotal", "kind": "Counter", "api_field": "ImgOptoTransformRespBodyBytes"}, - {"exporter_metric": "ImgOptoTransformRespHeaderBytesTotal", "kind": "Counter", "api_field": "ImgOptoTransformRespHeaderBytes"}, - {"exporter_metric": "ImgOptoTransformTotal", "kind": "Counter", "api_field": "ImgOptoTransform"}, - {"exporter_metric": "ImgVideoFramesTotal", "kind": "Counter", "api_field": "ImgVideoFrames"}, - {"exporter_metric": "ImgVideoRespBodyBytesTotal", "kind": "Counter", "api_field": "ImgVideoRespBodyBytes"}, - {"exporter_metric": "ImgVideoRespHeaderBytesTotal", "kind": "Counter", "api_field": "ImgVideoRespHeaderBytes"}, - {"exporter_metric": "ImgVideoShieldFramesTotal", "kind": "Counter", "api_field": "ImgVideoShieldFrames"}, - {"exporter_metric": "ImgVideoShieldRespBodyBytesTotal", "kind": "Counter", "api_field": "ImgVideoShieldRespBodyBytes"}, - {"exporter_metric": "ImgVideoShieldRespHeaderBytesTotal", "kind": "Counter", "api_field": "ImgVideoShieldRespHeaderBytes"}, - {"exporter_metric": "ImgVideoShieldTotal", "kind": "Counter", "api_field": "ImgVideoShield"}, - {"exporter_metric": "ImgVideoTotal", "kind": "Counter", "api_field": "ImgVideo"}, - {"exporter_metric": "IPv6Total", "kind": "Counter", "api_field": "IPv6"}, - {"exporter_metric": "LogBytesTotal", "kind": "Counter", "api_field": "LogBytes"}, - {"exporter_metric": "LoggingTotal", "kind": "Counter", "api_field": "Logging"}, - {"exporter_metric": "MissDurationSeconds", "kind": "Histogram", "api_field": "MissHistogram"}, - {"exporter_metric": "MissesTotal", "kind": "Counter", "api_field": "Misses"}, - {"exporter_metric": "MissRespBodyBytesTotal", "kind": "Counter", "api_field": "MissRespBodyBytes"}, - {"exporter_metric": "MissSubCountTotal", "kind": "Counter", "api_field": "MissSubCount"}, - {"exporter_metric": "MissSubTimeTotal", "kind": "Counter", "api_field": "MissSubTime"}, - {"exporter_metric": "MissTimeTotal", "kind": "Counter", "api_field": "MissTime"}, - {"exporter_metric": "ObjectSizeBytes", "kind": "ObjectSize", "api_field_sizes": ["ObjectSize100k", "ObjectSize100m", "ObjectSize10k", "ObjectSize10m", "ObjectSize1g", "ObjectSize1k", "ObjectSize1m"]}, - {"exporter_metric": "OriginFetchBodyBytesTotal", "kind": "Counter", "api_field": "OriginFetchBodyBytes"}, - {"exporter_metric": "OriginFetchesTotal", "kind": "Counter", "api_field": "OriginFetches"}, - {"exporter_metric": "OriginFetchHeaderBytesTotal", "kind": "Counter", "api_field": "OriginFetchHeaderBytes"}, - {"exporter_metric": "OriginFetchRespBodyBytesTotal", "kind": "Counter", "api_field": "OriginFetchRespBodyBytes"}, - {"exporter_metric": "OriginFetchRespHeaderBytesTotal", "kind": "Counter", "api_field": "OriginFetchRespHeaderBytes"}, - {"exporter_metric": "OriginRevalidationsTotal", "kind": "Counter", "api_field": "OriginRevalidations"}, - {"exporter_metric": "OTFPDeliverTimeTotal", "kind": "Counter", "api_field": "OTFPDeliverTime"}, - {"exporter_metric": "OTFPManifestTotal", "kind": "Counter", "api_field": "OTFPManifest"}, - {"exporter_metric": "OTFPRespBodyBytesTotal", "kind": "Counter", "api_field": "OTFPRespBodyBytes"}, - {"exporter_metric": "OTFPRespHeaderBytesTotal", "kind": "Counter", "api_field": "OTFPRespHeaderBytes"}, - {"exporter_metric": "OTFPShieldRespBodyBytesTotal", "kind": "Counter", "api_field": "OTFPShieldRespBodyBytes"}, - {"exporter_metric": "OTFPShieldRespHeaderBytesTotal", "kind": "Counter", "api_field": "OTFPShieldRespHeaderBytes"}, - {"exporter_metric": "OTFPShieldTimeTotal", "kind": "Counter", "api_field": "OTFPShieldTime"}, - {"exporter_metric": "OTFPShieldTotal", "kind": "Counter", "api_field": "OTFPShield"}, - {"exporter_metric": "OTFPTotal", "kind": "Counter", "api_field": "OTFP"}, - {"exporter_metric": "OTFPTransformRespBodyBytesTotal", "kind": "Counter", "api_field": "OTFPTransformRespBodyBytes"}, - {"exporter_metric": "OTFPTransformRespHeaderBytesTotal", "kind": "Counter", "api_field": "OTFPTransformRespHeaderBytes"}, - {"exporter_metric": "OTFPTransformTimeTotal", "kind": "Counter", "api_field": "OTFPTransformTime"}, - {"exporter_metric": "OTFPTransformTotal", "kind": "Counter", "api_field": "OTFPTransform"}, - {"exporter_metric": "PassesTotal", "kind": "Counter", "api_field": "Passes"}, - {"exporter_metric": "PassRespBodyBytesTotal", "kind": "Counter", "api_field": "PassRespBodyBytes"}, - {"exporter_metric": "PassSubCountTotal", "kind": "Counter", "api_field": "PassSubCount"}, - {"exporter_metric": "PassSubTimeTotal", "kind": "Counter", "api_field": "PassSubTime"}, - {"exporter_metric": "PassTimeTotal", "kind": "Counter", "api_field": "PassTime"}, - {"exporter_metric": "PCITotal", "kind": "Counter", "api_field": "PCI"}, - {"exporter_metric": "Pipe", "kind": "Counter", "api_field": "Pipe"}, - {"exporter_metric": "PipeSubCountTotal", "kind": "Counter", "api_field": "PipeSubCount"}, - {"exporter_metric": "PipeSubTimeTotal", "kind": "Counter", "api_field": "PipeSubTime"}, - {"exporter_metric": "PredeliverSubCountTotal", "kind": "Counter", "api_field": "PredeliverSubCount"}, - {"exporter_metric": "PredeliverSubTimeTotal", "kind": "Counter", "api_field": "PredeliverSubTime"}, - {"exporter_metric": "PrehashSubCountTotal", "kind": "Counter", "api_field": "PrehashSubCount"}, - {"exporter_metric": "PrehashSubTimeTotal", "kind": "Counter", "api_field": "PrehashSubTime"}, - {"exporter_metric": "RecvSubCountTotal", "kind": "Counter", "api_field": "RecvSubCount"}, - {"exporter_metric": "RecvSubTimeTotal", "kind": "Counter", "api_field": "RecvSubTime"}, - {"exporter_metric": "ReqBodyBytesTotal", "kind": "Counter", "api_field": "ReqBodyBytes"}, - {"exporter_metric": "ReqHeaderBytesTotal", "kind": "Counter", "api_field": "ReqHeaderBytes"}, - {"exporter_metric": "RequestsTotal", "kind": "Counter", "api_field": "Requests"}, - {"exporter_metric": "RespBodyBytesTotal", "kind": "Counter", "api_field": "RespBodyBytes"}, - {"exporter_metric": "RespHeaderBytesTotal", "kind": "Counter", "api_field": "RespHeaderBytes"}, - {"exporter_metric": "RestartTotal", "kind": "Counter", "api_field": "Restart"}, - {"exporter_metric": "SegBlockOriginFetchesTotal", "kind": "Counter", "api_field": "SegBlockOriginFetches"}, - {"exporter_metric": "SegBlockShieldFetchesTotal", "kind": "Counter", "api_field": "SegBlockShieldFetches"}, - {"exporter_metric": "ShieldFetchBodyBytesTotal", "kind": "Counter", "api_field": "ShieldFetchBodyBytes"}, - {"exporter_metric": "ShieldFetchesTotal", "kind": "Counter", "api_field": "ShieldFetches"}, - {"exporter_metric": "ShieldFetchHeaderBytesTotal", "kind": "Counter", "api_field": "ShieldFetchHeaderBytes"}, - {"exporter_metric": "ShieldFetchRespBodyBytesTotal", "kind": "Counter", "api_field": "ShieldFetchRespBodyBytes"}, - {"exporter_metric": "ShieldFetchRespHeaderBytesTotal", "kind": "Counter", "api_field": "ShieldFetchRespHeaderBytes"}, - {"exporter_metric": "ShieldRespBodyBytesTotal", "kind": "Counter", "api_field": "ShieldRespBodyBytes"}, - {"exporter_metric": "ShieldRespHeaderBytesTotal", "kind": "Counter", "api_field": "ShieldRespHeaderBytes"}, - {"exporter_metric": "ShieldRevalidationsTotal", "kind": "Counter", "api_field": "ShieldRevalidations"}, - {"exporter_metric": "ShieldTotal", "kind": "Counter", "api_field": "Shield"}, - {"exporter_metric": "StatusCodeTotal", "kind": "CounterLabels", "api_field_labels": [["Status200", "200"], ["Status204", "204"], ["Status206", "206"], ["Status301", "301"], ["Status302", "302"], ["Status304", "304"], ["Status400", "400"], ["Status401", "401"], ["Status403", "403"], ["Status404", "404"], ["Status416", "416"], ["Status429", "429"], ["Status500", "500"], ["Status501", "501"], ["Status502", "502"], ["Status503", "503"], ["Status504", "504"], ["Status505", "505"]]}, - {"exporter_metric": "StatusGroupTotal", "kind": "CounterLabels", "api_field_labels": [["Status1xx", "1xx"], ["Status2xx", "2xx"], ["Status3xx", "3xx"], ["Status4xx", "4xx"], ["Status5xx", "5xx"]]}, - {"exporter_metric": "SynthsTotal", "kind": "Counter", "api_field": "Synths"}, - {"exporter_metric": "TLSTotal", "kind": "CounterLabels", "api_field_labels": [["TLS", "any"], ["TLSv10", "v10"], ["TLSv11", "v11"], ["TLSv12", "v12"], ["TLSv13", "v13"]]}, - {"exporter_metric": "UncacheableTotal", "kind": "Counter", "api_field": "Uncacheable"}, - {"exporter_metric": "VideoTotal", "kind": "Counter", "api_field": "Video"}, - {"exporter_metric": "WAFBlockedTotal", "kind": "Counter", "api_field": "WAFBlocked"}, - {"exporter_metric": "WAFLoggedTotal", "kind": "Counter", "api_field": "WAFLogged"}, - {"exporter_metric": "WAFPassedTotal", "kind": "Counter", "api_field": "WAFPassed"} -] diff --git a/pkg/gen/doc.go b/pkg/gen/doc.go deleted file mode 100644 index a268d76..0000000 --- a/pkg/gen/doc.go +++ /dev/null @@ -1,5 +0,0 @@ -// Package gen contains generated code that defines the rt.fastly.com API -// response, the Prometheus metrics we export, and the mapping between them. -package gen - -//go:generate go run ../../cmd/fieldgen/main.go -metrics ../../cmd/fieldgen/exporter_metrics.json -fields ../../cmd/fieldgen/api_fields.json -mappings ../../cmd/fieldgen/mappings.json diff --git a/pkg/origin/doc.go b/pkg/origin/doc.go new file mode 100644 index 0000000..9af8624 --- /dev/null +++ b/pkg/origin/doc.go @@ -0,0 +1,5 @@ +// Package origin translates Fastly Origin Inspector real-time stats to +// Prometheus metrics. +// +// https://developer.fastly.com/reference/api/metrics-stats/origin-inspector/real-time +package origin diff --git a/pkg/origin/metrics.go b/pkg/origin/metrics.go new file mode 100644 index 0000000..e2ad70d --- /dev/null +++ b/pkg/origin/metrics.go @@ -0,0 +1,62 @@ +package origin + +import ( + "fmt" + "reflect" + "regexp" + + "github.com/fastly/fastly-exporter/pkg/filter" + + "github.com/prometheus/client_golang/prometheus" +) + +// Metrics collects all of the origin inspector Prometheus metrics. +type Metrics struct { + RespBodyBytesTotal *prometheus.CounterVec + RespHeaderBytesTotal *prometheus.CounterVec + ResponsesTotal *prometheus.CounterVec + StatusCodeTotal *prometheus.CounterVec + StatusGroupTotal *prometheus.CounterVec + LatencySeconds *prometheus.HistogramVec +} + +// NewMetrics returns a new set of metrics registered to the Registerer. +// Only metrics whose names pass the name filter are registered. +func NewMetrics(namespace, subsystem string, nameFilter filter.Filter, r prometheus.Registerer) *Metrics { + m := Metrics{ + RespBodyBytesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "resp_body_bytes_total", Help: `Number of body bytes from origin.`}, []string{"service_id", "service_name", "datacenter", "origin"}), + RespHeaderBytesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "resp_header_bytes_total", Help: `Number of header bytes from origin.`}, []string{"service_id", "service_name", "datacenter", "origin"}), + ResponsesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "responses_total", Help: `Number of responses from origin.`}, []string{"service_id", "service_name", "datacenter", "origin"}), + StatusCodeTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "status_code_total", Help: `Number of responses from origin, by status code e.g. 200, 419.`}, []string{"service_id", "service_name", "datacenter", "origin", "status_code"}), + StatusGroupTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "status_group_total", Help: `Number of responses from origin, by status group e.g. 1xx, 2xx.`}, []string{"service_id", "service_name", "datacenter", "origin", "status_group"}), + LatencySeconds: prometheus.NewHistogramVec(prometheus.HistogramOpts{Namespace: namespace, Subsystem: subsystem, Name: "latency_seconds", Help: `Response time from origin in seconds.`, Buckets: []float64{0.001, 0.005, 0.010, 0.050, 0.100, 0.250, 0.500, 1.000, 5.000, 10.000, 60.000}}, []string{"service_id", "service_name", "datacenter", "origin"}), + } + + for i, v := 0, reflect.ValueOf(m); i < v.NumField(); i++ { + c, ok := v.Field(i).Interface().(prometheus.Collector) + if !ok { + panic(fmt.Errorf("field %d/%d isn't a prometheus.Collector", i+1, v.NumField())) + } + if name := getName(c); !nameFilter.Permit(name) { + continue + } + if err := r.Register(c); err != nil { + panic(fmt.Errorf("error registering metric %d/%d: %w", i+1, v.NumField(), err)) + } + } + + return &m +} + +var descNameRegex = regexp.MustCompile("fqName: \"([^\"]+)\"") + +func getName(c prometheus.Collector) string { + d := make(chan *prometheus.Desc, 1) + c.Describe(d) + desc := (<-d).String() + matches := descNameRegex.FindAllStringSubmatch(desc, -1) + if len(matches) == 1 && len(matches[0]) == 2 { + return matches[0][1] + } + return "" +} diff --git a/pkg/origin/process.go b/pkg/origin/process.go new file mode 100644 index 0000000..269b528 --- /dev/null +++ b/pkg/origin/process.go @@ -0,0 +1,62 @@ +package origin + +// Process updates the metrics with data from the API response. +func Process(response *Response, serviceID, serviceName, serviceVersion string, m *Metrics) { + for _, d := range response.Data { + for datacenter, byOrigin := range d.Datacenter { + for origin, stats := range byOrigin { + m.RespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin).Add(float64(stats.RespBodyBytes)) + m.RespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin).Add(float64(stats.RespHeaderBytes)) + m.ResponsesTotal.WithLabelValues(serviceID, serviceName, datacenter, origin).Add(float64(stats.Responses)) + m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "1xx").Add(float64(stats.Status1xx)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "200").Add(float64(stats.Status200)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "204").Add(float64(stats.Status204)) + m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "2xx").Add(float64(stats.Status2xx)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "301").Add(float64(stats.Status301)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "302").Add(float64(stats.Status302)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "304").Add(float64(stats.Status304)) + m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "3xx").Add(float64(stats.Status3xx)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "400").Add(float64(stats.Status400)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "401").Add(float64(stats.Status401)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "403").Add(float64(stats.Status403)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "404").Add(float64(stats.Status404)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "416").Add(float64(stats.Status416)) + m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "4xx").Add(float64(stats.Status4xx)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "500").Add(float64(stats.Status500)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "501").Add(float64(stats.Status501)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "502").Add(float64(stats.Status502)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "503").Add(float64(stats.Status503)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "504").Add(float64(stats.Status504)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "505").Add(float64(stats.Status505)) + m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, origin, "5xx").Add(float64(stats.Status5xx)) + + // Latency stats are clearly from xxx_bucket{le="v"} metrics, + // but I don't see a good way to re-populate a histogram from + // those numbers. (If I'm missing something, file an issue!) + // + // Our clue is the final bucket, which says it's observations + // "of 60s and above". Based on that we use the lower bound of + // each stat as the observed value, except for the first bucket + // which we yolo as 500us because 0 doesn't really make sense?? + for v, n := range map[float64]uint64{ + 60.00: stats.Latency60000plus, + 10.00: stats.Latency10000to60000, + 5.000: stats.Latency5000to10000, + 1.000: stats.Latency1000to5000, + 0.500: stats.Latency500to1000, + 0.250: stats.Latency250to500, + 0.100: stats.Latency100to250, + 0.050: stats.Latency50to100, + 0.010: stats.Latency10to50, + 0.005: stats.Latency5to10, + 0.001: stats.Latency1to5, + 0.0005: stats.Latency0to1, // yolo + } { + for i := uint64(0); i < n; i++ { + m.LatencySeconds.WithLabelValues(serviceID, serviceName, datacenter, origin).Observe(v) + } + } + } + } + } +} diff --git a/pkg/origin/response.go b/pkg/origin/response.go new file mode 100644 index 0000000..2a7f4ba --- /dev/null +++ b/pkg/origin/response.go @@ -0,0 +1,61 @@ +package origin + +// Response models the origin inspector real-time data from rt.fastly.com. +type Response struct { + Data []Data `json:"Data"` + Timestamp uint64 `json:"Timestamp"` + AggregateDelay int `json:"AggregateDelay"` + Error string `json:"Error"` +} + +// Data is the top-level grouping of real-time origin inspector stats. +type Data struct { + Datacenter ByDatacenter `json:"datacenter"` + Aggregated ByOrigin `json:"aggregated"` +} + +// ByDatacenter groups origin inspector stats by datacenter. +type ByDatacenter map[string]ByOrigin + +// ByOrigin groups origin inspector stats by origin. +type ByOrigin map[string]Stats + +// Stats for a specific datacenter and origin. +type Stats struct { + RespBodyBytes uint64 `json:"resp_body_bytes"` // Number of body bytes from origin. + RespHeaderBytes uint64 `json:"resp_header_bytes"` // Number of header bytes from origin. + Responses uint64 `json:"responses"` // Number of responses from origin. + Status1xx uint64 `json:"status_1xx"` // Number of 1xx "Informational" category status codes delivered from origin. + Status200 uint64 `json:"status_200"` // Number of responses received with status code 200 (Success) from origin. + Status204 uint64 `json:"status_204"` // Number of responses received with status code 204 (No Content) from origin. + Status2xx uint64 `json:"status_2xx"` // Number of 2xx "Success" status codes delivered from origin. + Status301 uint64 `json:"status_301"` // Number of responses received with status code 301 (Moved Permanently) from origin. + Status302 uint64 `json:"status_302"` // Number of responses received with status code 302 (Found) from origin. + Status304 uint64 `json:"status_304"` // Number of responses received with status code 304 (Not Modified) from origin. + Status3xx uint64 `json:"status_3xx"` // Number of 3xx "Redirection" codes delivered from origin. + Status400 uint64 `json:"status_400"` // Number of responses received with status code 400 (Bad Request) from origin. + Status401 uint64 `json:"status_401"` // Number of responses received with status code 401 (Unauthorized) from origin. + Status403 uint64 `json:"status_403"` // Number of responses received with status code 403 (Forbidden) from origin. + Status404 uint64 `json:"status_404"` // Number of responses received with status code 404 (Not Found) from origin. + Status416 uint64 `json:"status_416"` // Number of responses received with status code 416 (Range Not Satisfiable) from origin. + Status4xx uint64 `json:"status_4xx"` // Number of 4xx "Client Error" codes delivered from origin. + Status500 uint64 `json:"status_500"` // Number of responses received with status code 500 (Internal Server Error) from origin. + Status501 uint64 `json:"status_501"` // Number of responses received with status code 501 (Not Implemented) from origin. + Status502 uint64 `json:"status_502"` // Number of responses received with status code 502 (Bad Gateway) from origin. + Status503 uint64 `json:"status_503"` // Number of responses received with status code 503 (Service Unavailable) from origin. + Status504 uint64 `json:"status_504"` // Number of responses received with status code 504 (Gateway Timeout) from origin. + Status505 uint64 `json:"status_505"` // Number of responses received with status code 505 (HTTP Version Not Supported) from origin. + Status5xx uint64 `json:"status_5xx"` // Number of 5xx "Server Error" codes delivered from origin. + Latency0to1 uint64 `json:"latency_0_to_1ms"` // Number of responses from origin with latency between 0 and 1 millisecond. + Latency1to5 uint64 `json:"latency_1_to_5ms"` // Number of responses from origin with latency between 1 and 5 milliseconds. + Latency5to10 uint64 `json:"latency_5_to_10ms"` // Number of responses from origin with latency between 5 and 10 milliseconds. + Latency10to50 uint64 `json:"latency_10_to_50ms"` // Number of responses from origin with latency between 10 and 50 milliseconds. + Latency50to100 uint64 `json:"latency_50_to_100ms"` // Number of responses from origin with latency between 50 and 100 milliseconds. + Latency100to250 uint64 `json:"latency_100_to_250ms"` // Number of responses from origin with latency between 100 and 250 milliseconds. + Latency250to500 uint64 `json:"latency_250_to_500ms"` // Number of responses from origin with latency between 250 and 500 milliseconds. + Latency500to1000 uint64 `json:"latency_500_to_1000ms"` // Number of responses from origin with latency between 500 and 1,000 milliseconds. + Latency1000to5000 uint64 `json:"latency_1000_to_5000ms"` // Number of responses from origin with latency between 1,000 and 5,000 milliseconds. + Latency5000to10000 uint64 `json:"latency_5000_to_10000ms"` // Number of responses from origin with latency between 5,000 and 10,000 milliseconds. + Latency10000to60000 uint64 `json:"latency_10000_to_60000ms"` // Number of responses from origin with latency between 10,000 and 60,000 milliseconds. + Latency60000plus uint64 `json:"latency_60000ms"` // Number of responses from origin with latency of 60,000 milliseconds and above. +} diff --git a/pkg/prom/metrics.go b/pkg/prom/metrics.go new file mode 100644 index 0000000..74c8a8c --- /dev/null +++ b/pkg/prom/metrics.go @@ -0,0 +1,34 @@ +package prom + +import ( + "github.com/fastly/fastly-exporter/pkg/filter" + "github.com/fastly/fastly-exporter/pkg/origin" + "github.com/fastly/fastly-exporter/pkg/realtime" + + "github.com/prometheus/client_golang/prometheus" +) + +// Metrics is the top-level collection of Prometheus metrics provided by the +// exporter. Not all metrics may be updated, based on e.g. filter rules. +type Metrics struct { + ServiceInfo *prometheus.GaugeVec + LastSuccessfulResponse *prometheus.GaugeVec + Realtime *realtime.Metrics + Origin *origin.Metrics +} + +// NewMetrics returns a fresh Metrics with the provided parameters. +func NewMetrics(namespace, rtSubsystemWillBeDeprecated string, nameFilter filter.Filter, r prometheus.Registerer) *Metrics { + var ( + serviceInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: namespace, Subsystem: rtSubsystemWillBeDeprecated, Name: "service_info", Help: "Static gauge with service ID, name, and version information."}, []string{"service_id", "service_name", "service_version"}) + lastSuccessfulResponse = prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: namespace, Subsystem: rtSubsystemWillBeDeprecated, Name: "last_successful_response", Help: "Unix timestamp of the last successful response received from the real-time stats API."}, []string{"service_id", "service_name"}) + ) + r.MustRegister(serviceInfo, lastSuccessfulResponse) + + return &Metrics{ + ServiceInfo: serviceInfo, + LastSuccessfulResponse: lastSuccessfulResponse, + Realtime: realtime.NewMetrics(namespace, rtSubsystemWillBeDeprecated, nameFilter, r), // TODO(pb): change this to "rt" or "realtime" + Origin: origin.NewMetrics(namespace, "origin", nameFilter, r), + } +} diff --git a/pkg/prom/registry.go b/pkg/prom/registry.go index de8ec47..7e50e5a 100644 --- a/pkg/prom/registry.go +++ b/pkg/prom/registry.go @@ -11,7 +11,6 @@ import ( "sync" "github.com/fastly/fastly-exporter/pkg/filter" - "github.com/fastly/fastly-exporter/pkg/gen" "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -26,26 +25,32 @@ import ( // // https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config type Registry struct { - mtx sync.Mutex - version string - namespace string - subsystem string - metricNameFilter filter.Filter - byServiceID map[string]*metricsRegistry - defaultGatherers []prometheus.Gatherer + mtx sync.Mutex + version string + namespace string + rtSubsystemDeprecated string + metricNameFilter filter.Filter + byServiceID map[string]*metricsRegistry + defaultGatherers []prometheus.Gatherer http.Handler } -// NewRegistry returns a new and empty registry for Prometheus metrics. -func NewRegistry(version, namespace, subsystem string, metricNameFilter filter.Filter, defaultGatherers ...prometheus.Gatherer) *Registry { +// NewRegistry returns a new and empty registry for Prometheus metrics. The +// metric name filter restricts which metrics are made available for scrapes. +// The default gatherers can be used to provide additional arbitrary metrics. +// +// The rtSubsystemDeprecated param is used for default and real-time metrics +// only. In a future version, those metrics will have their subsystem fixed to +// "rt", and this parameter will be removed. +func NewRegistry(version, namespace, rtSubsystemDeprecated string, metricNameFilter filter.Filter, defaultGatherers ...prometheus.Gatherer) *Registry { r := &Registry{ - version: version, - namespace: namespace, - subsystem: subsystem, - metricNameFilter: metricNameFilter, - byServiceID: map[string]*metricsRegistry{}, - defaultGatherers: defaultGatherers, + version: version, + namespace: namespace, + rtSubsystemDeprecated: rtSubsystemDeprecated, + metricNameFilter: metricNameFilter, + byServiceID: map[string]*metricsRegistry{}, + defaultGatherers: defaultGatherers, } router := mux.NewRouter() @@ -63,21 +68,21 @@ func NewRegistry(version, namespace, subsystem string, metricNameFilter filter.F // with other registries and served as a single set of metrics via the // prometheus.Gatherers helper type. type metricsRegistry struct { - metrics *gen.Metrics + metrics *Metrics registry *prometheus.Registry } // MetricsFor returns a set of Prometheus metrics for a specific service, with // the expectation that callers will update those metrics with data retrieved // from the Fastly real-time stats API. -func (r *Registry) MetricsFor(serviceID string) *gen.Metrics { +func (r *Registry) MetricsFor(serviceID string) *Metrics { r.mtx.Lock() defer r.mtx.Unlock() mr, ok := r.byServiceID[serviceID] if !ok { registry := prometheus.NewRegistry() - metrics := gen.NewMetrics(r.namespace, r.subsystem, r.metricNameFilter, registry) + metrics := NewMetrics(r.namespace, r.rtSubsystemDeprecated, r.metricNameFilter, registry) mr = &metricsRegistry{metrics, registry} r.byServiceID[serviceID] = mr // TODO(pb): at some point, expire and remove? } diff --git a/pkg/prom/registry_test.go b/pkg/prom/registry_test.go index ce2ca27..3602ea0 100644 --- a/pkg/prom/registry_test.go +++ b/pkg/prom/registry_test.go @@ -25,11 +25,11 @@ func TestRegistryEndpoints(t *testing.T) { registry = prom.NewRegistry(version, namespace, subsystem, metricNameFilter) ) - registry.MetricsFor("AAA").RequestsTotal.With(prometheus.Labels{ + registry.MetricsFor("AAA").Realtime.RequestsTotal.With(prometheus.Labels{ "service_id": "AAA", "service_name": "Service One", "datacenter": "NYC", }).Add(1) - registry.MetricsFor("BBB").RequestsTotal.With(prometheus.Labels{ + registry.MetricsFor("BBB").Realtime.RequestsTotal.With(prometheus.Labels{ "service_id": "BBB", "service_name": "Service Two", "datacenter": "NYC", }).Add(2) diff --git a/pkg/realtime/doc.go b/pkg/realtime/doc.go new file mode 100644 index 0000000..6b2dc2e --- /dev/null +++ b/pkg/realtime/doc.go @@ -0,0 +1,4 @@ +// Package realtime translates Fastly Real-time analytics to Prometheus metrics. +// +// https://developer.fastly.com/reference/api/metrics-stats/realtime +package realtime diff --git a/pkg/gen/gen.go b/pkg/realtime/metrics.go similarity index 58% rename from pkg/gen/gen.go rename to pkg/realtime/metrics.go index b2ced42..9220c87 100644 --- a/pkg/gen/gen.go +++ b/pkg/realtime/metrics.go @@ -1,219 +1,17 @@ -// Code generated by fieldgen; DO NOT EDIT. - -package gen +package realtime import ( "fmt" "reflect" "regexp" - "strconv" "github.com/fastly/fastly-exporter/pkg/filter" "github.com/prometheus/client_golang/prometheus" ) -// APIResponse models the response from rt.fastly.com. It can get quite large; -// when there are lots of services being monitored, unmarshaling to this type is -// the CPU bottleneck of the program. -type APIResponse struct { - Timestamp uint64 `json:"Timestamp"` - AggregateDelay int64 `json:"AggregateDelay"` - Data []struct { - Datacenter map[string]Datacenter `json:"datacenter"` - Aggregated Datacenter `json:"aggregated"` - Recorded uint64 `json:"recorded"` - } `json:"Data"` - Error string `json:"error"` -} - -// Datacenter models the per-datacenter portion of the rt.fastly.com response. -type Datacenter struct { - AttackBlockedReqBodyBytes uint64 `json:"attack_blocked_req_body_bytes"` - AttackBlockedReqHeaderBytes uint64 `json:"attack_blocked_req_header_bytes"` - AttackLoggedReqBodyBytes uint64 `json:"attack_logged_req_body_bytes"` - AttackLoggedReqHeaderBytes uint64 `json:"attack_logged_req_header_bytes"` - AttackPassedReqBodyBytes uint64 `json:"attack_passed_req_body_bytes"` - AttackPassedReqHeaderBytes uint64 `json:"attack_passed_req_header_bytes"` - AttackReqBodyBytes uint64 `json:"attack_req_body_bytes"` - AttackReqHeaderBytes uint64 `json:"attack_req_header_bytes"` - AttackRespSynthBytes uint64 `json:"attack_resp_synth_bytes"` - BackendReqBodyBytes uint64 `json:"bereq_body_bytes"` - BackendReqHeaderBytes uint64 `json:"bereq_header_bytes"` - Billed uint64 `json:"billed"` - BilledBodyBytes uint64 `json:"billed_body_bytes"` - BilledHeaderBytes uint64 `json:"billed_header_bytes"` - Blacklisted uint64 `json:"blacklist"` - BodySize uint64 `json:"body_size"` - ComputeBackendReqBodyBytesTotal uint64 `json:"compute_bereq_body_bytes"` - ComputeBackendReqErrorsTotal uint64 `json:"compute_bereq_errors"` - ComputeBackendReqHeaderBytesTotal uint64 `json:"compute_bereq_header_bytes"` - ComputeBackendReqTotal uint64 `json:"compute_bereqs"` - ComputeBackendRespBodyBytesTotal uint64 `json:"compute_beresp_body_bytes"` - ComputeBackendRespHeaderBytesTotal uint64 `json:"compute_beresp_header_bytes"` - ComputeExecutionTimeMilliseconds uint64 `json:"compute_execution_time_ms"` - ComputeGlobalsLimitExceededTotal uint64 `json:"compute_globals_limit_exceeded"` - ComputeGuestErrorsTotal uint64 `json:"compute_guest_errors"` - ComputeHeapLimitExceededTotal uint64 `json:"compute_heap_limit_exceeded"` - ComputeRAMUsed uint64 `json:"compute_ram_used"` - ComputeReqBodyBytesTotal uint64 `json:"compute_req_body_bytes"` - ComputeReqHeaderBytesTotal uint64 `json:"compute_req_header_bytes"` - ComputeRequests uint64 `json:"compute_requests"` - ComputeRequestTimeMilliseconds uint64 `json:"compute_request_time_ms"` - ComputeResourceLimitExceedTotal uint64 `json:"compute_resource_limit_exceeded"` - ComputeRespBodyBytesTotal uint64 `json:"compute_resp_body_bytes"` - ComputeRespHeaderBytesTotal uint64 `json:"compute_resp_header_bytes"` - ComputeRespStatus1xx uint64 `json:"compute_resp_status_1xx"` - ComputeRespStatus2xx uint64 `json:"compute_resp_status_2xx"` - ComputeRespStatus3xx uint64 `json:"compute_resp_status_3xx"` - ComputeRespStatus4xx uint64 `json:"compute_resp_status_4xx"` - ComputeRespStatus5xx uint64 `json:"compute_resp_status_5xx"` - ComputeRuntimeErrorsTotal uint64 `json:"compute_runtime_errors"` - ComputeStackLimitExceededTotal uint64 `json:"compute_stack_limit_exceeded"` - DeliverSubCount uint64 `json:"deliver_sub_count"` - DeliverSubTime uint64 `json:"deliver_sub_time"` - Edge uint64 `json:"edge_requests"` - EdgeRespBodyBytes uint64 `json:"edge_resp_body_bytes"` - EdgeRespHeaderBytes uint64 `json:"edge_resp_header_bytes"` - Errors uint64 `json:"errors"` - ErrorSubCount uint64 `json:"error_sub_count"` - ErrorSubTime uint64 `json:"error_sub_time"` - FetchSubCount uint64 `json:"fetch_sub_count"` - FetchSubTime uint64 `json:"fetch_sub_time"` - HashSubCount uint64 `json:"hash_sub_count"` - HashSubTime uint64 `json:"hash_sub_time"` - HeaderSize uint64 `json:"header_size"` - HitRespBodyBytes uint64 `json:"hit_resp_body_bytes"` - Hits uint64 `json:"hits"` - HitsTime float64 `json:"hits_time"` - HitSubCount uint64 `json:"hit_sub_count"` - HitSubTime uint64 `json:"hit_sub_time"` - HTTP2 uint64 `json:"http2"` - ImgOpto uint64 `json:"imgopto"` - ImgOptoRespBodyBytes uint64 `json:"imgopto_resp_body_bytes"` - ImgOptoRespHeaderBytes uint64 `json:"imgopto_resp_header_bytes"` - ImgOptoShield uint64 `json:"imgopto_shield"` - ImgOptoShieldRespBodyBytes uint64 `json:"imgopto_shield_resp_body_bytes"` - ImgOptoShieldRespHeaderBytes uint64 `json:"imgopto_shield_resp_header_bytes"` - ImgOptoTransform uint64 `json:"imgopto_transforms"` - ImgOptoTransformRespBodyBytes uint64 `json:"imgopto_transform_resp_body_bytes"` - ImgOptoTransformRespHeaderBytes uint64 `json:"imgopto_transform_resp_header_bytes"` - ImgVideo uint64 `json:"imgvideo"` - ImgVideoFrames uint64 `json:"imgvideo_frames"` - ImgVideoRespBodyBytes uint64 `json:"imgvideo_resp_body_bytes"` - ImgVideoRespHeaderBytes uint64 `json:"imgvideo_resp_header_bytes"` - ImgVideoShield uint64 `json:"imgvideo_shield"` - ImgVideoShieldFrames uint64 `json:"imgvideo_shield_frames"` - ImgVideoShieldRespBodyBytes uint64 `json:"imgvideo_shield_resp_body_bytes"` - ImgVideoShieldRespHeaderBytes uint64 `json:"imgvideo_shield_resp_header_bytes"` - IPv6 uint64 `json:"ipv6"` - LogBytes uint64 `json:"log_bytes"` - Logging uint64 `json:"logging"` - Misses uint64 `json:"miss"` - MissHistogram map[string]uint64 `json:"miss_histogram"` - MissRespBodyBytes uint64 `json:"miss_resp_body_bytes"` - MissSubCount uint64 `json:"miss_sub_count"` - MissSubTime uint64 `json:"miss_sub_time"` - MissTime float64 `json:"miss_time"` - ObjectSize100k uint64 `json:"object_size_100k"` - ObjectSize100m uint64 `json:"object_size_100m"` - ObjectSize10k uint64 `json:"object_size_10k"` - ObjectSize10m uint64 `json:"object_size_10m"` - ObjectSize1g uint64 `json:"object_size_1g"` - ObjectSize1k uint64 `json:"object_size_1k"` - ObjectSize1m uint64 `json:"object_size_1m"` - ObjectSizeOther uint64 `json:"object_size_other"` - OriginFetchBodyBytes uint64 `json:"origin_fetch_body_bytes"` - OriginFetches uint64 `json:"origin_fetches"` - OriginFetchHeaderBytes uint64 `json:"origin_fetch_header_bytes"` - OriginFetchRespBodyBytes uint64 `json:"origin_fetch_resp_body_bytes"` - OriginFetchRespHeaderBytes uint64 `json:"origin_fetch_resp_header_bytes"` - OriginRevalidations uint64 `json:"origin_revalidations"` - OTFP uint64 `json:"otfp"` - OTFPDeliverTime uint64 `json:"otfp_deliver_time"` - OTFPManifest uint64 `json:"otfp_manifests"` - OTFPRespBodyBytes uint64 `json:"otfp_resp_body_bytes"` - OTFPRespHeaderBytes uint64 `json:"otfp_resp_header_bytes"` - OTFPShield uint64 `json:"otfp_shield"` - OTFPShieldRespBodyBytes uint64 `json:"otfp_shield_resp_body_bytes"` - OTFPShieldRespHeaderBytes uint64 `json:"otfp_shield_resp_header_bytes"` - OTFPShieldTime uint64 `json:"otfp_shield_time"` - OTFPTransform uint64 `json:"otfp_transforms"` - OTFPTransformRespBodyBytes uint64 `json:"otfp_transform_resp_body_bytes"` - OTFPTransformRespHeaderBytes uint64 `json:"otfp_transform_resp_header_bytes"` - OTFPTransformTime uint64 `json:"otfp_transform_time"` - Passes uint64 `json:"pass"` - PassRespBodyBytes uint64 `json:"pass_resp_body_bytes"` - PassSubCount uint64 `json:"pass_sub_count"` - PassSubTime uint64 `json:"pass_sub_time"` - PassTime float64 `json:"pass_time"` - PCI uint64 `json:"pci"` - Pipe uint64 `json:"pipe"` - PipeSubCount uint64 `json:"pipe_sub_count"` - PipeSubTime uint64 `json:"pipe_sub_time"` - PredeliverSubCount uint64 `json:"predeliver_sub_count"` - PredeliverSubTime uint64 `json:"predeliver_sub_time"` - PrehashSubCount uint64 `json:"prehash_sub_count"` - PrehashSubTime uint64 `json:"prehash_sub_time"` - RecvSubCount uint64 `json:"recv_sub_count"` - RecvSubTime uint64 `json:"recv_sub_time"` - ReqBodyBytes uint64 `json:"req_body_bytes"` - ReqHeaderBytes uint64 `json:"req_header_bytes"` - Requests uint64 `json:"requests"` - RespBodyBytes uint64 `json:"resp_body_bytes"` - RespHeaderBytes uint64 `json:"resp_header_bytes"` - Restart uint64 `json:"restarts"` - SegBlockOriginFetches uint64 `json:"segblock_origin_fetches"` - SegBlockShieldFetches uint64 `json:"segblock_shield_fetches"` - Shield uint64 `json:"shield"` - ShieldFetchBodyBytes uint64 `json:"shield_fetch_body_bytes"` - ShieldFetches uint64 `json:"shield_fetches"` - ShieldFetchHeaderBytes uint64 `json:"shield_fetch_header_bytes"` - ShieldFetchRespBodyBytes uint64 `json:"shield_fetch_resp_body_bytes"` - ShieldFetchRespHeaderBytes uint64 `json:"shield_fetch_resp_header_bytes"` - ShieldRespBodyBytes uint64 `json:"shield_resp_body_bytes"` - ShieldRespHeaderBytes uint64 `json:"shield_resp_header_bytes"` - ShieldRevalidations uint64 `json:"shield_revalidations"` - Status1xx uint64 `json:"status_1xx"` - Status200 uint64 `json:"status_200"` - Status204 uint64 `json:"status_204"` - Status206 uint64 `json:"status_206"` - Status2xx uint64 `json:"status_2xx"` - Status301 uint64 `json:"status_301"` - Status302 uint64 `json:"status_302"` - Status304 uint64 `json:"status_304"` - Status3xx uint64 `json:"status_3xx"` - Status400 uint64 `json:"status_400"` - Status401 uint64 `json:"status_401"` - Status403 uint64 `json:"status_403"` - Status404 uint64 `json:"status_404"` - Status416 uint64 `json:"status_416"` - Status429 uint64 `json:"status_429"` - Status4xx uint64 `json:"status_4xx"` - Status500 uint64 `json:"status_500"` - Status501 uint64 `json:"status_501"` - Status502 uint64 `json:"status_502"` - Status503 uint64 `json:"status_503"` - Status504 uint64 `json:"status_504"` - Status505 uint64 `json:"status_505"` - Status5xx uint64 `json:"status_5xx"` - Synths uint64 `json:"synth"` - TLS uint64 `json:"tls"` - TLSv10 uint64 `json:"tls_v10"` - TLSv11 uint64 `json:"tls_v11"` - TLSv12 uint64 `json:"tls_v12"` - TLSv13 uint64 `json:"tls_v13"` - Uncacheable uint64 `json:"uncacheable"` - Video uint64 `json:"video"` - WAFBlocked uint64 `json:"waf_blocked"` - WAFLogged uint64 `json:"waf_logged"` - WAFPassed uint64 `json:"waf_passed"` -} - -// Metrics collects all of the Prometheus metrics exported by this service. +// Metrics collects all of the Prometheus metrics that map to real-time stats. type Metrics struct { RealtimeAPIRequestsTotal *prometheus.CounterVec - ServiceInfo *prometheus.GaugeVec - LastSuccessfulResponse *prometheus.GaugeVec AttackBlockedReqBodyBytesTotal *prometheus.CounterVec AttackBlockedReqHeaderBytesTotal *prometheus.CounterVec AttackLoggedReqBodyBytesTotal *prometheus.CounterVec @@ -364,8 +162,6 @@ type Metrics struct { func NewMetrics(namespace, subsystem string, nameFilter filter.Filter, r prometheus.Registerer) *Metrics { m := Metrics{ RealtimeAPIRequestsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "realtime_api_requests_total", Help: "Total requests made to the real-time stats API."}, []string{"service_id", "service_name", "result"}), - ServiceInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: namespace, Subsystem: subsystem, Name: "service_info", Help: "Static gauge with service ID, name, and version information."}, []string{"service_id", "service_name", "service_version"}), - LastSuccessfulResponse: prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: namespace, Subsystem: subsystem, Name: "last_successful_response", Help: "Unix timestamp of the last successful response received from the real-time stats API."}, []string{"service_id", "service_name"}), AttackBlockedReqBodyBytesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "attack_blocked_req_body_bytes_total", Help: "Total body bytes received from requests that triggered a WAF rule that was blocked."}, []string{"service_id", "service_name", "datacenter"}), AttackBlockedReqHeaderBytesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "attack_blocked_req_header_bytes_total", Help: "Total header bytes received from requests that triggered a WAF rule that was blocked."}, []string{"service_id", "service_name", "datacenter"}), AttackLoggedReqBodyBytesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{Namespace: namespace, Subsystem: subsystem, Name: "attack_logged_req_body_bytes_total", Help: "Total body bytes received from requests that triggered a WAF rule that was logged."}, []string{"service_id", "service_name", "datacenter"}), @@ -539,212 +335,3 @@ func getName(c prometheus.Collector) string { } return "" } - -// Process updates the metrics with data from the API response. -func Process(response *APIResponse, serviceID, serviceName, serviceVersion string, m *Metrics) { - for _, d := range response.Data { - for datacenter, stats := range d.Datacenter { - m.AttackBlockedReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackBlockedReqBodyBytes)) - m.AttackBlockedReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackBlockedReqHeaderBytes)) - m.AttackLoggedReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackLoggedReqBodyBytes)) - m.AttackLoggedReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackLoggedReqHeaderBytes)) - m.AttackPassedReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackPassedReqBodyBytes)) - m.AttackPassedReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackPassedReqHeaderBytes)) - m.AttackReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackReqBodyBytes)) - m.AttackReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackReqHeaderBytes)) - m.AttackRespSynthBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackRespSynthBytes)) - m.BackendReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.BackendReqBodyBytes)) - m.BackendReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.BackendReqHeaderBytes)) - m.BilledBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.BilledBodyBytes)) - m.BilledHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.BilledHeaderBytes)) - m.BilledTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Billed)) - m.BlacklistedTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Blacklisted)) - m.BodySizeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.BodySize)) - m.ComputeBackendReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeBackendReqBodyBytesTotal)) - m.ComputeBackendReqErrorsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeBackendReqErrorsTotal)) - m.ComputeBackendReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeBackendReqHeaderBytesTotal)) - m.ComputeBackendReqTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeBackendReqTotal)) - m.ComputeBackendRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeBackendRespBodyBytesTotal)) - m.ComputeBackendRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeBackendRespHeaderBytesTotal)) - m.ComputeExecutionTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeExecutionTimeMilliseconds) / 10000.0) - m.ComputeGlobalsLimitExceededTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeGlobalsLimitExceededTotal)) - m.ComputeGuestErrorsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeGuestErrorsTotal)) - m.ComputeHeapLimitExceededTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeHeapLimitExceededTotal)) - m.ComputeRAMUsedBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeRAMUsed)) - m.ComputeReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeReqBodyBytesTotal)) - m.ComputeReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeReqHeaderBytesTotal)) - m.ComputeRequestsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeRequests)) - m.ComputeRequestTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeRequestTimeMilliseconds) / 10000.0) - m.ComputeResourceLimitExceedTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeResourceLimitExceedTotal)) - m.ComputeRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeRespBodyBytesTotal)) - m.ComputeRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeRespHeaderBytesTotal)) - m.ComputeRespStatusTotal.WithLabelValues(serviceID, serviceName, datacenter, "1xx").Add(float64(stats.ComputeRespStatus1xx)) - m.ComputeRespStatusTotal.WithLabelValues(serviceID, serviceName, datacenter, "2xx").Add(float64(stats.ComputeRespStatus2xx)) - m.ComputeRespStatusTotal.WithLabelValues(serviceID, serviceName, datacenter, "3xx").Add(float64(stats.ComputeRespStatus3xx)) - m.ComputeRespStatusTotal.WithLabelValues(serviceID, serviceName, datacenter, "4xx").Add(float64(stats.ComputeRespStatus4xx)) - m.ComputeRespStatusTotal.WithLabelValues(serviceID, serviceName, datacenter, "5xx").Add(float64(stats.ComputeRespStatus5xx)) - m.ComputeRuntimeErrorsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeRuntimeErrorsTotal)) - m.ComputeStackLimitExceededTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeStackLimitExceededTotal)) - m.DeliverSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.DeliverSubCount)) - m.DeliverSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.DeliverSubTime)) - m.EdgeRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.EdgeRespBodyBytes)) - m.EdgeRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.EdgeRespHeaderBytes)) - m.EdgeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Edge)) - m.ErrorsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Errors)) - m.ErrorSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ErrorSubCount)) - m.ErrorSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ErrorSubTime)) - m.FetchSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.FetchSubCount)) - m.FetchSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.FetchSubTime)) - m.HashSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HashSubCount)) - m.HashSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HashSubTime)) - m.HeaderSizeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HeaderSize)) - m.HitRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HitRespBodyBytes)) - m.HitsTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HitsTime)) - m.HitsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Hits)) - m.HitSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HitSubCount)) - m.HitSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HitSubTime)) - m.HTTP2Total.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HTTP2)) - m.ImgOptoRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoRespBodyBytes)) - m.ImgOptoRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoRespHeaderBytes)) - m.ImgOptoShieldRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoShieldRespBodyBytes)) - m.ImgOptoShieldRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoShieldRespHeaderBytes)) - m.ImgOptoShieldTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoShield)) - m.ImgOptoTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOpto)) - m.ImgOptoTransformRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoTransformRespBodyBytes)) - m.ImgOptoTransformRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoTransformRespHeaderBytes)) - m.ImgOptoTransformTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoTransform)) - m.ImgVideoFramesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoFrames)) - m.ImgVideoRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoRespBodyBytes)) - m.ImgVideoRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoRespHeaderBytes)) - m.ImgVideoShieldFramesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoShieldFrames)) - m.ImgVideoShieldRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoShieldRespBodyBytes)) - m.ImgVideoShieldRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoShieldRespHeaderBytes)) - m.ImgVideoShieldTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoShield)) - m.ImgVideoTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideo)) - m.IPv6Total.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.IPv6)) - m.LogBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.LogBytes)) - m.LoggingTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Logging)) - processHistogram(stats.MissHistogram, m.MissDurationSeconds.WithLabelValues(serviceID, serviceName, datacenter)) - m.MissesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Misses)) - m.MissRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.MissRespBodyBytes)) - m.MissSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.MissSubCount)) - m.MissSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.MissSubTime)) - m.MissTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.MissTime)) - processObjectSizes(stats.ObjectSize1k, stats.ObjectSize10k, stats.ObjectSize100k, stats.ObjectSize1m, stats.ObjectSize10m, stats.ObjectSize100m, stats.ObjectSize1g, m.ObjectSizeBytes.WithLabelValues(serviceID, serviceName, datacenter)) - m.OriginFetchBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OriginFetchBodyBytes)) - m.OriginFetchesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OriginFetches)) - m.OriginFetchHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OriginFetchHeaderBytes)) - m.OriginFetchRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OriginFetchRespBodyBytes)) - m.OriginFetchRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OriginFetchRespHeaderBytes)) - m.OriginRevalidationsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OriginRevalidations)) - m.OTFPDeliverTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPDeliverTime)) - m.OTFPManifestTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPManifest)) - m.OTFPRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPRespBodyBytes)) - m.OTFPRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPRespHeaderBytes)) - m.OTFPShieldRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPShieldRespBodyBytes)) - m.OTFPShieldRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPShieldRespHeaderBytes)) - m.OTFPShieldTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPShieldTime)) - m.OTFPShieldTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPShield)) - m.OTFPTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFP)) - m.OTFPTransformRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPTransformRespBodyBytes)) - m.OTFPTransformRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPTransformRespHeaderBytes)) - m.OTFPTransformTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPTransformTime)) - m.OTFPTransformTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPTransform)) - m.PassesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Passes)) - m.PassRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PassRespBodyBytes)) - m.PassSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PassSubCount)) - m.PassSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PassSubTime)) - m.PassTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PassTime)) - m.PCITotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PCI)) - m.Pipe.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Pipe)) - m.PipeSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PipeSubCount)) - m.PipeSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PipeSubTime)) - m.PredeliverSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PredeliverSubCount)) - m.PredeliverSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PredeliverSubTime)) - m.PrehashSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PrehashSubCount)) - m.PrehashSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PrehashSubTime)) - m.RecvSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.RecvSubCount)) - m.RecvSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.RecvSubTime)) - m.ReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ReqBodyBytes)) - m.ReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ReqHeaderBytes)) - m.RequestsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Requests)) - m.RespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.RespBodyBytes)) - m.RespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.RespHeaderBytes)) - m.RestartTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Restart)) - m.SegBlockOriginFetchesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.SegBlockOriginFetches)) - m.SegBlockShieldFetchesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.SegBlockShieldFetches)) - m.ShieldFetchBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldFetchBodyBytes)) - m.ShieldFetchesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldFetches)) - m.ShieldFetchHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldFetchHeaderBytes)) - m.ShieldFetchRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldFetchRespBodyBytes)) - m.ShieldFetchRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldFetchRespHeaderBytes)) - m.ShieldRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldRespBodyBytes)) - m.ShieldRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldRespHeaderBytes)) - m.ShieldRevalidationsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldRevalidations)) - m.ShieldTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Shield)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "200").Add(float64(stats.Status200)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "204").Add(float64(stats.Status204)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "206").Add(float64(stats.Status206)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "301").Add(float64(stats.Status301)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "302").Add(float64(stats.Status302)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "304").Add(float64(stats.Status304)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "400").Add(float64(stats.Status400)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "401").Add(float64(stats.Status401)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "403").Add(float64(stats.Status403)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "404").Add(float64(stats.Status404)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "416").Add(float64(stats.Status416)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "429").Add(float64(stats.Status429)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "500").Add(float64(stats.Status500)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "501").Add(float64(stats.Status501)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "502").Add(float64(stats.Status502)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "503").Add(float64(stats.Status503)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "504").Add(float64(stats.Status504)) - m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "505").Add(float64(stats.Status505)) - m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, "1xx").Add(float64(stats.Status1xx)) - m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, "2xx").Add(float64(stats.Status2xx)) - m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, "3xx").Add(float64(stats.Status3xx)) - m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, "4xx").Add(float64(stats.Status4xx)) - m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, "5xx").Add(float64(stats.Status5xx)) - m.SynthsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Synths)) - m.TLSTotal.WithLabelValues(serviceID, serviceName, datacenter, "any").Add(float64(stats.TLS)) - m.TLSTotal.WithLabelValues(serviceID, serviceName, datacenter, "v10").Add(float64(stats.TLSv10)) - m.TLSTotal.WithLabelValues(serviceID, serviceName, datacenter, "v11").Add(float64(stats.TLSv11)) - m.TLSTotal.WithLabelValues(serviceID, serviceName, datacenter, "v12").Add(float64(stats.TLSv12)) - m.TLSTotal.WithLabelValues(serviceID, serviceName, datacenter, "v13").Add(float64(stats.TLSv13)) - m.UncacheableTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Uncacheable)) - m.VideoTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Video)) - m.WAFBlockedTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.WAFBlocked)) - m.WAFLoggedTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.WAFLogged)) - m.WAFPassedTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.WAFPassed)) - } - } -} - -func processHistogram(src map[string]uint64, obs prometheus.Observer) { - for str, count := range src { - ms, err := strconv.Atoi(str) - if err != nil { - continue - } - s := float64(ms) / 1e3 - for i := 0; i < int(count); i++ { - obs.Observe(s) - } - } -} - -func processObjectSizes(n1k, n10k, n100k, n1m, n10m, n100m, n1g uint64, obs prometheus.Observer) { - for v, n := range map[uint64]uint64{ - 1 * 1024: n1k, - 10 * 1024: n10k, - 100 * 1024: n100k, - 1 * 1000 * 1024: n1m, - 10 * 1000 * 1024: n10m, - 100 * 1000 * 1024: n100m, - 1000 * 1000 * 1024: n1g, - } { - for i := uint64(0); i < n; i++ { - obs.Observe(float64(v)) - } - } -} diff --git a/pkg/realtime/process.go b/pkg/realtime/process.go new file mode 100644 index 0000000..d424348 --- /dev/null +++ b/pkg/realtime/process.go @@ -0,0 +1,216 @@ +package realtime + +import ( + "strconv" + + "github.com/prometheus/client_golang/prometheus" +) + +// Process updates the metrics with data from the API response. +func Process(response *Response, serviceID, serviceName, serviceVersion string, m *Metrics) { + for _, d := range response.Data { + for datacenter, stats := range d.Datacenter { + m.AttackBlockedReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackBlockedReqBodyBytes)) + m.AttackBlockedReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackBlockedReqHeaderBytes)) + m.AttackLoggedReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackLoggedReqBodyBytes)) + m.AttackLoggedReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackLoggedReqHeaderBytes)) + m.AttackPassedReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackPassedReqBodyBytes)) + m.AttackPassedReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackPassedReqHeaderBytes)) + m.AttackReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackReqBodyBytes)) + m.AttackReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackReqHeaderBytes)) + m.AttackRespSynthBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.AttackRespSynthBytes)) + m.BackendReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.BackendReqBodyBytes)) + m.BackendReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.BackendReqHeaderBytes)) + m.BilledBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.BilledBodyBytes)) + m.BilledHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.BilledHeaderBytes)) + m.BilledTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Billed)) + m.BlacklistedTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Blacklisted)) + m.BodySizeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.BodySize)) + m.ComputeBackendReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeBackendReqBodyBytesTotal)) + m.ComputeBackendReqErrorsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeBackendReqErrorsTotal)) + m.ComputeBackendReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeBackendReqHeaderBytesTotal)) + m.ComputeBackendReqTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeBackendReqTotal)) + m.ComputeBackendRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeBackendRespBodyBytesTotal)) + m.ComputeBackendRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeBackendRespHeaderBytesTotal)) + m.ComputeExecutionTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeExecutionTimeMilliseconds) / 10000.0) + m.ComputeGlobalsLimitExceededTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeGlobalsLimitExceededTotal)) + m.ComputeGuestErrorsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeGuestErrorsTotal)) + m.ComputeHeapLimitExceededTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeHeapLimitExceededTotal)) + m.ComputeRAMUsedBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeRAMUsed)) + m.ComputeReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeReqBodyBytesTotal)) + m.ComputeReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeReqHeaderBytesTotal)) + m.ComputeRequestsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeRequests)) + m.ComputeRequestTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeRequestTimeMilliseconds) / 10000.0) + m.ComputeResourceLimitExceedTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeResourceLimitExceedTotal)) + m.ComputeRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeRespBodyBytesTotal)) + m.ComputeRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeRespHeaderBytesTotal)) + m.ComputeRespStatusTotal.WithLabelValues(serviceID, serviceName, datacenter, "1xx").Add(float64(stats.ComputeRespStatus1xx)) + m.ComputeRespStatusTotal.WithLabelValues(serviceID, serviceName, datacenter, "2xx").Add(float64(stats.ComputeRespStatus2xx)) + m.ComputeRespStatusTotal.WithLabelValues(serviceID, serviceName, datacenter, "3xx").Add(float64(stats.ComputeRespStatus3xx)) + m.ComputeRespStatusTotal.WithLabelValues(serviceID, serviceName, datacenter, "4xx").Add(float64(stats.ComputeRespStatus4xx)) + m.ComputeRespStatusTotal.WithLabelValues(serviceID, serviceName, datacenter, "5xx").Add(float64(stats.ComputeRespStatus5xx)) + m.ComputeRuntimeErrorsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeRuntimeErrorsTotal)) + m.ComputeStackLimitExceededTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ComputeStackLimitExceededTotal)) + m.DeliverSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.DeliverSubCount)) + m.DeliverSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.DeliverSubTime)) + m.EdgeRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.EdgeRespBodyBytes)) + m.EdgeRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.EdgeRespHeaderBytes)) + m.EdgeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Edge)) + m.ErrorsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Errors)) + m.ErrorSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ErrorSubCount)) + m.ErrorSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ErrorSubTime)) + m.FetchSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.FetchSubCount)) + m.FetchSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.FetchSubTime)) + m.HashSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HashSubCount)) + m.HashSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HashSubTime)) + m.HeaderSizeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HeaderSize)) + m.HitRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HitRespBodyBytes)) + m.HitsTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HitsTime)) + m.HitsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Hits)) + m.HitSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HitSubCount)) + m.HitSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HitSubTime)) + m.HTTP2Total.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.HTTP2)) + m.ImgOptoRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoRespBodyBytes)) + m.ImgOptoRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoRespHeaderBytes)) + m.ImgOptoShieldRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoShieldRespBodyBytes)) + m.ImgOptoShieldRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoShieldRespHeaderBytes)) + m.ImgOptoShieldTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoShield)) + m.ImgOptoTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOpto)) + m.ImgOptoTransformRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoTransformRespBodyBytes)) + m.ImgOptoTransformRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoTransformRespHeaderBytes)) + m.ImgOptoTransformTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgOptoTransform)) + m.ImgVideoFramesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoFrames)) + m.ImgVideoRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoRespBodyBytes)) + m.ImgVideoRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoRespHeaderBytes)) + m.ImgVideoShieldFramesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoShieldFrames)) + m.ImgVideoShieldRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoShieldRespBodyBytes)) + m.ImgVideoShieldRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoShieldRespHeaderBytes)) + m.ImgVideoShieldTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideoShield)) + m.ImgVideoTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ImgVideo)) + m.IPv6Total.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.IPv6)) + m.LogBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.LogBytes)) + m.LoggingTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Logging)) + processHistogram(stats.MissHistogram, m.MissDurationSeconds.WithLabelValues(serviceID, serviceName, datacenter)) + m.MissesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Misses)) + m.MissRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.MissRespBodyBytes)) + m.MissSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.MissSubCount)) + m.MissSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.MissSubTime)) + m.MissTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.MissTime)) + processObjectSizes(stats.ObjectSize1k, stats.ObjectSize10k, stats.ObjectSize100k, stats.ObjectSize1m, stats.ObjectSize10m, stats.ObjectSize100m, stats.ObjectSize1g, m.ObjectSizeBytes.WithLabelValues(serviceID, serviceName, datacenter)) + m.OriginFetchBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OriginFetchBodyBytes)) + m.OriginFetchesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OriginFetches)) + m.OriginFetchHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OriginFetchHeaderBytes)) + m.OriginFetchRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OriginFetchRespBodyBytes)) + m.OriginFetchRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OriginFetchRespHeaderBytes)) + m.OriginRevalidationsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OriginRevalidations)) + m.OTFPDeliverTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPDeliverTime)) + m.OTFPManifestTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPManifest)) + m.OTFPRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPRespBodyBytes)) + m.OTFPRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPRespHeaderBytes)) + m.OTFPShieldRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPShieldRespBodyBytes)) + m.OTFPShieldRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPShieldRespHeaderBytes)) + m.OTFPShieldTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPShieldTime)) + m.OTFPShieldTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPShield)) + m.OTFPTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFP)) + m.OTFPTransformRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPTransformRespBodyBytes)) + m.OTFPTransformRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPTransformRespHeaderBytes)) + m.OTFPTransformTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPTransformTime)) + m.OTFPTransformTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.OTFPTransform)) + m.PassesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Passes)) + m.PassRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PassRespBodyBytes)) + m.PassSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PassSubCount)) + m.PassSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PassSubTime)) + m.PassTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PassTime)) + m.PCITotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PCI)) + m.Pipe.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Pipe)) + m.PipeSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PipeSubCount)) + m.PipeSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PipeSubTime)) + m.PredeliverSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PredeliverSubCount)) + m.PredeliverSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PredeliverSubTime)) + m.PrehashSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PrehashSubCount)) + m.PrehashSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.PrehashSubTime)) + m.RecvSubCountTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.RecvSubCount)) + m.RecvSubTimeTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.RecvSubTime)) + m.ReqBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ReqBodyBytes)) + m.ReqHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ReqHeaderBytes)) + m.RequestsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Requests)) + m.RespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.RespBodyBytes)) + m.RespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.RespHeaderBytes)) + m.RestartTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Restart)) + m.SegBlockOriginFetchesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.SegBlockOriginFetches)) + m.SegBlockShieldFetchesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.SegBlockShieldFetches)) + m.ShieldFetchBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldFetchBodyBytes)) + m.ShieldFetchesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldFetches)) + m.ShieldFetchHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldFetchHeaderBytes)) + m.ShieldFetchRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldFetchRespBodyBytes)) + m.ShieldFetchRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldFetchRespHeaderBytes)) + m.ShieldRespBodyBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldRespBodyBytes)) + m.ShieldRespHeaderBytesTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldRespHeaderBytes)) + m.ShieldRevalidationsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.ShieldRevalidations)) + m.ShieldTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Shield)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "200").Add(float64(stats.Status200)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "204").Add(float64(stats.Status204)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "206").Add(float64(stats.Status206)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "301").Add(float64(stats.Status301)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "302").Add(float64(stats.Status302)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "304").Add(float64(stats.Status304)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "400").Add(float64(stats.Status400)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "401").Add(float64(stats.Status401)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "403").Add(float64(stats.Status403)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "404").Add(float64(stats.Status404)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "416").Add(float64(stats.Status416)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "429").Add(float64(stats.Status429)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "500").Add(float64(stats.Status500)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "501").Add(float64(stats.Status501)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "502").Add(float64(stats.Status502)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "503").Add(float64(stats.Status503)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "504").Add(float64(stats.Status504)) + m.StatusCodeTotal.WithLabelValues(serviceID, serviceName, datacenter, "505").Add(float64(stats.Status505)) + m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, "1xx").Add(float64(stats.Status1xx)) + m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, "2xx").Add(float64(stats.Status2xx)) + m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, "3xx").Add(float64(stats.Status3xx)) + m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, "4xx").Add(float64(stats.Status4xx)) + m.StatusGroupTotal.WithLabelValues(serviceID, serviceName, datacenter, "5xx").Add(float64(stats.Status5xx)) + m.SynthsTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Synths)) + m.TLSTotal.WithLabelValues(serviceID, serviceName, datacenter, "any").Add(float64(stats.TLS)) + m.TLSTotal.WithLabelValues(serviceID, serviceName, datacenter, "v10").Add(float64(stats.TLSv10)) + m.TLSTotal.WithLabelValues(serviceID, serviceName, datacenter, "v11").Add(float64(stats.TLSv11)) + m.TLSTotal.WithLabelValues(serviceID, serviceName, datacenter, "v12").Add(float64(stats.TLSv12)) + m.TLSTotal.WithLabelValues(serviceID, serviceName, datacenter, "v13").Add(float64(stats.TLSv13)) + m.UncacheableTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Uncacheable)) + m.VideoTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.Video)) + m.WAFBlockedTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.WAFBlocked)) + m.WAFLoggedTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.WAFLogged)) + m.WAFPassedTotal.WithLabelValues(serviceID, serviceName, datacenter).Add(float64(stats.WAFPassed)) + } + } +} + +func processHistogram(src map[string]uint64, obs prometheus.Observer) { + for str, count := range src { + ms, err := strconv.Atoi(str) + if err != nil { + continue + } + s := float64(ms) / 1e3 + for i := 0; i < int(count); i++ { + obs.Observe(s) + } + } +} + +func processObjectSizes(n1k, n10k, n100k, n1m, n10m, n100m, n1g uint64, obs prometheus.Observer) { + for v, n := range map[uint64]uint64{ + 1 * 1024: n1k, + 10 * 1024: n10k, + 100 * 1024: n100k, + 1 * 1000 * 1024: n1m, + 10 * 1000 * 1024: n10m, + 100 * 1000 * 1024: n100m, + 1000 * 1000 * 1024: n1g, + } { + for i := uint64(0); i < n; i++ { + obs.Observe(float64(v)) + } + } +} diff --git a/pkg/realtime/response.go b/pkg/realtime/response.go new file mode 100644 index 0000000..2b04eba --- /dev/null +++ b/pkg/realtime/response.go @@ -0,0 +1,198 @@ +package realtime + +// Response models the response from rt.fastly.com. It can get quite large; when +// there are lots of services being monitored, unmarshaling to this type is the +// CPU bottleneck of the program. +type Response struct { + Timestamp uint64 `json:"Timestamp"` + AggregateDelay int64 `json:"AggregateDelay"` + Data []struct { + Datacenter map[string]Datacenter `json:"datacenter"` + Aggregated Datacenter `json:"aggregated"` + Recorded uint64 `json:"recorded"` + } `json:"Data"` + Error string `json:"error"` +} + +// Datacenter models the per-datacenter portion of the rt.fastly.com response. +type Datacenter struct { + AttackBlockedReqBodyBytes uint64 `json:"attack_blocked_req_body_bytes"` + AttackBlockedReqHeaderBytes uint64 `json:"attack_blocked_req_header_bytes"` + AttackLoggedReqBodyBytes uint64 `json:"attack_logged_req_body_bytes"` + AttackLoggedReqHeaderBytes uint64 `json:"attack_logged_req_header_bytes"` + AttackPassedReqBodyBytes uint64 `json:"attack_passed_req_body_bytes"` + AttackPassedReqHeaderBytes uint64 `json:"attack_passed_req_header_bytes"` + AttackReqBodyBytes uint64 `json:"attack_req_body_bytes"` + AttackReqHeaderBytes uint64 `json:"attack_req_header_bytes"` + AttackRespSynthBytes uint64 `json:"attack_resp_synth_bytes"` + BackendReqBodyBytes uint64 `json:"bereq_body_bytes"` + BackendReqHeaderBytes uint64 `json:"bereq_header_bytes"` + Billed uint64 `json:"billed"` + BilledBodyBytes uint64 `json:"billed_body_bytes"` + BilledHeaderBytes uint64 `json:"billed_header_bytes"` + Blacklisted uint64 `json:"blacklist"` + BodySize uint64 `json:"body_size"` + ComputeBackendReqBodyBytesTotal uint64 `json:"compute_bereq_body_bytes"` + ComputeBackendReqErrorsTotal uint64 `json:"compute_bereq_errors"` + ComputeBackendReqHeaderBytesTotal uint64 `json:"compute_bereq_header_bytes"` + ComputeBackendReqTotal uint64 `json:"compute_bereqs"` + ComputeBackendRespBodyBytesTotal uint64 `json:"compute_beresp_body_bytes"` + ComputeBackendRespHeaderBytesTotal uint64 `json:"compute_beresp_header_bytes"` + ComputeExecutionTimeMilliseconds uint64 `json:"compute_execution_time_ms"` + ComputeGlobalsLimitExceededTotal uint64 `json:"compute_globals_limit_exceeded"` + ComputeGuestErrorsTotal uint64 `json:"compute_guest_errors"` + ComputeHeapLimitExceededTotal uint64 `json:"compute_heap_limit_exceeded"` + ComputeRAMUsed uint64 `json:"compute_ram_used"` + ComputeReqBodyBytesTotal uint64 `json:"compute_req_body_bytes"` + ComputeReqHeaderBytesTotal uint64 `json:"compute_req_header_bytes"` + ComputeRequests uint64 `json:"compute_requests"` + ComputeRequestTimeMilliseconds uint64 `json:"compute_request_time_ms"` + ComputeResourceLimitExceedTotal uint64 `json:"compute_resource_limit_exceeded"` + ComputeRespBodyBytesTotal uint64 `json:"compute_resp_body_bytes"` + ComputeRespHeaderBytesTotal uint64 `json:"compute_resp_header_bytes"` + ComputeRespStatus1xx uint64 `json:"compute_resp_status_1xx"` + ComputeRespStatus2xx uint64 `json:"compute_resp_status_2xx"` + ComputeRespStatus3xx uint64 `json:"compute_resp_status_3xx"` + ComputeRespStatus4xx uint64 `json:"compute_resp_status_4xx"` + ComputeRespStatus5xx uint64 `json:"compute_resp_status_5xx"` + ComputeRuntimeErrorsTotal uint64 `json:"compute_runtime_errors"` + ComputeStackLimitExceededTotal uint64 `json:"compute_stack_limit_exceeded"` + DeliverSubCount uint64 `json:"deliver_sub_count"` + DeliverSubTime uint64 `json:"deliver_sub_time"` + Edge uint64 `json:"edge_requests"` + EdgeRespBodyBytes uint64 `json:"edge_resp_body_bytes"` + EdgeRespHeaderBytes uint64 `json:"edge_resp_header_bytes"` + Errors uint64 `json:"errors"` + ErrorSubCount uint64 `json:"error_sub_count"` + ErrorSubTime uint64 `json:"error_sub_time"` + FetchSubCount uint64 `json:"fetch_sub_count"` + FetchSubTime uint64 `json:"fetch_sub_time"` + HashSubCount uint64 `json:"hash_sub_count"` + HashSubTime uint64 `json:"hash_sub_time"` + HeaderSize uint64 `json:"header_size"` + HitRespBodyBytes uint64 `json:"hit_resp_body_bytes"` + Hits uint64 `json:"hits"` + HitsTime float64 `json:"hits_time"` + HitSubCount uint64 `json:"hit_sub_count"` + HitSubTime uint64 `json:"hit_sub_time"` + HTTP2 uint64 `json:"http2"` + ImgOpto uint64 `json:"imgopto"` + ImgOptoRespBodyBytes uint64 `json:"imgopto_resp_body_bytes"` + ImgOptoRespHeaderBytes uint64 `json:"imgopto_resp_header_bytes"` + ImgOptoShield uint64 `json:"imgopto_shield"` + ImgOptoShieldRespBodyBytes uint64 `json:"imgopto_shield_resp_body_bytes"` + ImgOptoShieldRespHeaderBytes uint64 `json:"imgopto_shield_resp_header_bytes"` + ImgOptoTransform uint64 `json:"imgopto_transforms"` + ImgOptoTransformRespBodyBytes uint64 `json:"imgopto_transform_resp_body_bytes"` + ImgOptoTransformRespHeaderBytes uint64 `json:"imgopto_transform_resp_header_bytes"` + ImgVideo uint64 `json:"imgvideo"` + ImgVideoFrames uint64 `json:"imgvideo_frames"` + ImgVideoRespBodyBytes uint64 `json:"imgvideo_resp_body_bytes"` + ImgVideoRespHeaderBytes uint64 `json:"imgvideo_resp_header_bytes"` + ImgVideoShield uint64 `json:"imgvideo_shield"` + ImgVideoShieldFrames uint64 `json:"imgvideo_shield_frames"` + ImgVideoShieldRespBodyBytes uint64 `json:"imgvideo_shield_resp_body_bytes"` + ImgVideoShieldRespHeaderBytes uint64 `json:"imgvideo_shield_resp_header_bytes"` + IPv6 uint64 `json:"ipv6"` + LogBytes uint64 `json:"log_bytes"` + Logging uint64 `json:"logging"` + Misses uint64 `json:"miss"` + MissHistogram map[string]uint64 `json:"miss_histogram"` + MissRespBodyBytes uint64 `json:"miss_resp_body_bytes"` + MissSubCount uint64 `json:"miss_sub_count"` + MissSubTime uint64 `json:"miss_sub_time"` + MissTime float64 `json:"miss_time"` + ObjectSize100k uint64 `json:"object_size_100k"` + ObjectSize100m uint64 `json:"object_size_100m"` + ObjectSize10k uint64 `json:"object_size_10k"` + ObjectSize10m uint64 `json:"object_size_10m"` + ObjectSize1g uint64 `json:"object_size_1g"` + ObjectSize1k uint64 `json:"object_size_1k"` + ObjectSize1m uint64 `json:"object_size_1m"` + ObjectSizeOther uint64 `json:"object_size_other"` + OriginFetchBodyBytes uint64 `json:"origin_fetch_body_bytes"` + OriginFetches uint64 `json:"origin_fetches"` + OriginFetchHeaderBytes uint64 `json:"origin_fetch_header_bytes"` + OriginFetchRespBodyBytes uint64 `json:"origin_fetch_resp_body_bytes"` + OriginFetchRespHeaderBytes uint64 `json:"origin_fetch_resp_header_bytes"` + OriginRevalidations uint64 `json:"origin_revalidations"` + OTFP uint64 `json:"otfp"` + OTFPDeliverTime uint64 `json:"otfp_deliver_time"` + OTFPManifest uint64 `json:"otfp_manifests"` + OTFPRespBodyBytes uint64 `json:"otfp_resp_body_bytes"` + OTFPRespHeaderBytes uint64 `json:"otfp_resp_header_bytes"` + OTFPShield uint64 `json:"otfp_shield"` + OTFPShieldRespBodyBytes uint64 `json:"otfp_shield_resp_body_bytes"` + OTFPShieldRespHeaderBytes uint64 `json:"otfp_shield_resp_header_bytes"` + OTFPShieldTime uint64 `json:"otfp_shield_time"` + OTFPTransform uint64 `json:"otfp_transforms"` + OTFPTransformRespBodyBytes uint64 `json:"otfp_transform_resp_body_bytes"` + OTFPTransformRespHeaderBytes uint64 `json:"otfp_transform_resp_header_bytes"` + OTFPTransformTime uint64 `json:"otfp_transform_time"` + Passes uint64 `json:"pass"` + PassRespBodyBytes uint64 `json:"pass_resp_body_bytes"` + PassSubCount uint64 `json:"pass_sub_count"` + PassSubTime uint64 `json:"pass_sub_time"` + PassTime float64 `json:"pass_time"` + PCI uint64 `json:"pci"` + Pipe uint64 `json:"pipe"` + PipeSubCount uint64 `json:"pipe_sub_count"` + PipeSubTime uint64 `json:"pipe_sub_time"` + PredeliverSubCount uint64 `json:"predeliver_sub_count"` + PredeliverSubTime uint64 `json:"predeliver_sub_time"` + PrehashSubCount uint64 `json:"prehash_sub_count"` + PrehashSubTime uint64 `json:"prehash_sub_time"` + RecvSubCount uint64 `json:"recv_sub_count"` + RecvSubTime uint64 `json:"recv_sub_time"` + ReqBodyBytes uint64 `json:"req_body_bytes"` + ReqHeaderBytes uint64 `json:"req_header_bytes"` + Requests uint64 `json:"requests"` + RespBodyBytes uint64 `json:"resp_body_bytes"` + RespHeaderBytes uint64 `json:"resp_header_bytes"` + Restart uint64 `json:"restarts"` + SegBlockOriginFetches uint64 `json:"segblock_origin_fetches"` + SegBlockShieldFetches uint64 `json:"segblock_shield_fetches"` + Shield uint64 `json:"shield"` + ShieldFetchBodyBytes uint64 `json:"shield_fetch_body_bytes"` + ShieldFetches uint64 `json:"shield_fetches"` + ShieldFetchHeaderBytes uint64 `json:"shield_fetch_header_bytes"` + ShieldFetchRespBodyBytes uint64 `json:"shield_fetch_resp_body_bytes"` + ShieldFetchRespHeaderBytes uint64 `json:"shield_fetch_resp_header_bytes"` + ShieldRespBodyBytes uint64 `json:"shield_resp_body_bytes"` + ShieldRespHeaderBytes uint64 `json:"shield_resp_header_bytes"` + ShieldRevalidations uint64 `json:"shield_revalidations"` + Status1xx uint64 `json:"status_1xx"` + Status200 uint64 `json:"status_200"` + Status204 uint64 `json:"status_204"` + Status206 uint64 `json:"status_206"` + Status2xx uint64 `json:"status_2xx"` + Status301 uint64 `json:"status_301"` + Status302 uint64 `json:"status_302"` + Status304 uint64 `json:"status_304"` + Status3xx uint64 `json:"status_3xx"` + Status400 uint64 `json:"status_400"` + Status401 uint64 `json:"status_401"` + Status403 uint64 `json:"status_403"` + Status404 uint64 `json:"status_404"` + Status416 uint64 `json:"status_416"` + Status429 uint64 `json:"status_429"` + Status4xx uint64 `json:"status_4xx"` + Status500 uint64 `json:"status_500"` + Status501 uint64 `json:"status_501"` + Status502 uint64 `json:"status_502"` + Status503 uint64 `json:"status_503"` + Status504 uint64 `json:"status_504"` + Status505 uint64 `json:"status_505"` + Status5xx uint64 `json:"status_5xx"` + Synths uint64 `json:"synth"` + TLS uint64 `json:"tls"` + TLSv10 uint64 `json:"tls_v10"` + TLSv11 uint64 `json:"tls_v11"` + TLSv12 uint64 `json:"tls_v12"` + TLSv13 uint64 `json:"tls_v13"` + Uncacheable uint64 `json:"uncacheable"` + Video uint64 `json:"video"` + WAFBlocked uint64 `json:"waf_blocked"` + WAFLogged uint64 `json:"waf_logged"` + WAFPassed uint64 `json:"waf_passed"` +} diff --git a/pkg/rt/common_test.go b/pkg/rt/common_test.go index 2ec5977..c2ae81d 100644 --- a/pkg/rt/common_test.go +++ b/pkg/rt/common_test.go @@ -59,10 +59,9 @@ func assertMetricOutput(t *testing.T, want, have map[string]float64) { } } - t.Log(cmp.Diff(want, have)) - if !cmp.Equal(want, have) { t.Error(cmp.Diff(want, have)) + t.Logf("metric output was different") } } diff --git a/pkg/rt/doc.go b/pkg/rt/doc.go index b987028..f20f54c 100644 --- a/pkg/rt/doc.go +++ b/pkg/rt/doc.go @@ -1,3 +1,4 @@ -// Package rt provides an opinionated interface to rt.fastly.com. -// It consumes real-time stats, and emits the data to Prometheus. +// Package rt provides an opinionated interface to rt.fastly.com. It consumes +// the realtime and origin inspector APIs, and emits the data to Prometheus +// metrics. package rt diff --git a/pkg/rt/manager.go b/pkg/rt/manager.go index 604fffd..98969d9 100644 --- a/pkg/rt/manager.go +++ b/pkg/rt/manager.go @@ -2,10 +2,11 @@ package rt import ( "context" + "fmt" "sort" "sync" - "github.com/fastly/fastly-exporter/pkg/gen" + "github.com/fastly/fastly-exporter/pkg/prom" "github.com/go-kit/log" "github.com/go-kit/log/level" ) @@ -20,12 +21,12 @@ type ServiceIdentifier interface { // the method of the prom.Registry which yields a set of Prometheus metrics for // a specific service ID. type MetricsProvider interface { - MetricsFor(serviceID string) *gen.Metrics + MetricsFor(serviceID string) *prom.Metrics } -// Manager owns a set of subscribers. When refreshed, it will ask a -// ServiceIdentifier for a set of service IDs that should be active, and manage -// the lifecycles of the corresponding subscribers. +// Manager owns a set of subscribers. On refresh, it asks the ServiceIdentifier +// for a set of service IDs that should be active, and manages the lifecycles of +// the corresponding subscribers. type Manager struct { ids ServiceIdentifier client HTTPClient @@ -116,9 +117,11 @@ func (m *Manager) StopAll() { level.Info(m.logger).Log("service_id", id, "subscriber", "stop") irq := m.managed[id] irq.cancel() - err := <-irq.done + for i := 0; i < cap(irq.done); i++ { + err := <-irq.done + level.Debug(m.logger).Log("service_id", id, "goroutine", i+1, "of", cap(irq.done), "interrupt", err) + } delete(m.managed, id) - level.Debug(m.logger).Log("service_id", id, "interrupt", err) } } @@ -126,11 +129,10 @@ func (m *Manager) spawn(serviceID string) interrupt { var ( subscriber = NewSubscriber(m.client, m.token, serviceID, m.metrics.MetricsFor(serviceID), m.subscriberOptions...) ctx, cancel = context.WithCancel(context.Background()) - done = make(chan error, 1) + done = make(chan error, 2) ) - go func() { - done <- subscriber.Run(ctx) - }() + go func() { done <- fmt.Errorf("realtime: %w", subscriber.RunRealtime(ctx)) }() + go func() { done <- fmt.Errorf("origins: %w", subscriber.RunOrigins(ctx)) }() return interrupt{cancel, done} } diff --git a/pkg/rt/subscriber.go b/pkg/rt/subscriber.go index 9c38b46..4404c66 100644 --- a/pkg/rt/subscriber.go +++ b/pkg/rt/subscriber.go @@ -10,7 +10,9 @@ import ( "strings" "time" - "github.com/fastly/fastly-exporter/pkg/gen" + "github.com/fastly/fastly-exporter/pkg/origin" + "github.com/fastly/fastly-exporter/pkg/prom" + "github.com/fastly/fastly-exporter/pkg/realtime" "github.com/go-kit/log" "github.com/go-kit/log/level" jsoniter "github.com/json-iterator/go" @@ -28,14 +30,14 @@ type MetadataProvider interface { Metadata(id string) (name string, version int, found bool) } -// Subscriber polls rt.fastly.com for a single service ID. -// It emits the received real-time stats data to Prometheus. +// Subscriber polls rt.fastly.com endpoints for a single service ID. It emits +// the received stats data to Prometheus metrics. type Subscriber struct { client HTTPClient token string serviceID string provider MetadataProvider - metrics *gen.Metrics + metrics *prom.Metrics postprocess func() logger log.Logger } @@ -58,15 +60,16 @@ func WithLogger(logger log.Logger) SubscriberOption { } // WithPostprocess sets the postprocess function for the subscriber, which is -// invoked after each successful call to the real-time stats API. By default, a -// no-op postprocess function is invoked. This option is only useful for tests. +// invoked after each successful call to any API endpoint. By default, a no-op +// postprocess function is invoked. This option is only useful for tests. func WithPostprocess(f func()) SubscriberOption { return func(s *Subscriber) { s.postprocess = f } } -// NewSubscriber returns a ready-to-use subscriber. -// Run must be called to update the metrics. -func NewSubscriber(client HTTPClient, token, serviceID string, metrics *gen.Metrics, options ...SubscriberOption) *Subscriber { +// NewSubscriber returns a ready-to-use subscriber. Callers must be sure to +// invoke the Run method of the returned subscriber in order to actually update +// any metrics. +func NewSubscriber(client HTTPClient, token, serviceID string, metrics *prom.Metrics, options ...SubscriberOption) *Subscriber { s := &Subscriber{ client: client, token: token, @@ -82,11 +85,11 @@ func NewSubscriber(client HTTPClient, token, serviceID string, metrics *gen.Metr return s } -// Run polls rt.fastly.com in a hot loop, collecting real-time stats information -// and emitting it to the Prometheus metrics provided to the constructor. The +// RunRealtime polls rt.fastly.com in a hot loop, collecting real-time stats and +// emitting them to the Prometheus metrics provided to the constructor. The // method returns when the context is canceled, or a non-recoverable error // occurs. -func (s *Subscriber) Run(ctx context.Context) error { +func (s *Subscriber) RunRealtime(ctx context.Context) error { var ts uint64 for { select { @@ -94,8 +97,8 @@ func (s *Subscriber) Run(ctx context.Context) error { return ctx.Err() default: - name, result, delay, newts, fatal := s.query(ctx, ts) - s.metrics.RealtimeAPIRequestsTotal.WithLabelValues(s.serviceID, name, string(result)).Inc() + name, result, delay, newts, fatal := s.queryRealtime(ctx, ts) + s.metrics.Realtime.RealtimeAPIRequestsTotal.WithLabelValues(s.serviceID, name, string(result)).Inc() if fatal != nil { return fatal } @@ -108,18 +111,43 @@ func (s *Subscriber) Run(ctx context.Context) error { } } -// query rt.fastly.com for the service ID represented by the subscriber, and -// with the provided starting timestamp. The function may block for several -// seconds; cancel the context to provoke early termination. On success, the -// received real-time data is processed, and the Prometheus metrics related to -// the Fastly service are updated. +// RunOrigins polls rt.fastly.com, collecting real-time origin stats and +// emitting them to the Prometheus metrics provided to the constructor. The +// method returns when the context is canceled, or a non-recoverable error +// occurs. +func (s *Subscriber) RunOrigins(ctx context.Context) error { + var ts uint64 + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + name, _, delay, newts, fatal := s.queryOrigins(ctx, ts) + if fatal != nil { + return fatal + } + s.metrics.LastSuccessfulResponse.WithLabelValues(s.serviceID, name).Set(float64(time.Now().Unix())) + if delay > 0 { + contextSleep(ctx, delay) + } + ts = newts + } + } +} + +// queryRealtime fetches real-time stats from rt.fastly.com for the service ID +// represented by the subscriber, and with the provided starting timestamp. The +// function may block for several seconds; cancel the context to provoke early +// termination. On success, the received real-time data is processed, and the +// Prometheus metrics related to the Fastly service are updated. // // Returns the current name of the service, the broad class of result of the API -// request, any delay that should pass before query is invoked again, the new -// timestamp that should be provided to the next call to query, and an error. -// Recoverable errors are logged internally and not returned, so any non-nil -// error returned by this method should be considered fatal to the subscriber. -func (s *Subscriber) query(ctx context.Context, ts uint64) (currentName string, result apiResult, delay time.Duration, newts uint64, fatal error) { +// request, any delay that should pass before queryRealtime is invoked again, +// the new timestamp that should be provided to the next call to queryRealtime, +// and an error. Recoverable errors are logged internally and not returned, so +// any non-nil error returned by this method should be considered fatal to the +// subscriber. +func (s *Subscriber) queryRealtime(ctx context.Context, ts uint64) (currentName string, result apiResult, delay time.Duration, newts uint64, fatal error) { name, ver, found := s.provider.Metadata(s.serviceID) version := strconv.Itoa(ver) if !found { @@ -143,7 +171,69 @@ func (s *Subscriber) query(ctx context.Context, ts uint64) (currentName string, return name, apiResultError, time.Second, ts, nil } - var response gen.APIResponse + var response realtime.Response + if err := jsoniterAPI.NewDecoder(resp.Body).Decode(&response); err != nil { + resp.Body.Close() + level.Error(s.logger).Log("during", "decode response", "err", err) + return name, apiResultError, time.Second, ts, nil + } + resp.Body.Close() + + apiErr := response.Error + if apiErr == "" { + apiErr = "" + } + + switch resp.StatusCode { + case http.StatusOK: + level.Debug(s.logger).Log("status_code", resp.StatusCode, "response_ts", response.Timestamp, "err", apiErr) + if strings.Contains(apiErr, "No data available") { + result = apiResultNoData + } else { + result = apiResultSuccess + } + realtime.Process(&response, s.serviceID, name, version, s.metrics.Realtime) + s.postprocess() + + case http.StatusUnauthorized, http.StatusForbidden: + result = apiResultError + level.Error(s.logger).Log("status_code", resp.StatusCode, "response_ts", response.Timestamp, "err", apiErr, "msg", "token may be invalid") + delay = 15 * time.Second + + default: + result = apiResultUnknown + level.Error(s.logger).Log("status_code", resp.StatusCode, "response_ts", response.Timestamp, "err", apiErr) + delay = 5 * time.Second + } + + return name, result, delay, response.Timestamp, nil +} + +func (s *Subscriber) queryOrigins(ctx context.Context, ts uint64) (currentName string, result apiResult, delay time.Duration, newts uint64, fatal error) { + name, ver, found := s.provider.Metadata(s.serviceID) + version := strconv.Itoa(ver) + if !found { + name, version = s.serviceID, "unknown" + } + s.metrics.ServiceInfo.WithLabelValues(s.serviceID, name, version).Set(1) + + // rt.fastly.com blocks until it has data to return. + // It's safe to call in a (single-threaded!) hot loop. + u := fmt.Sprintf("https://rt.fastly.com/v1/origins/%s/ts/%d", url.QueryEscape(s.serviceID), ts) + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return name, apiResultError, 0, ts, fmt.Errorf("error constructing origins API request: %w", err) + } + + req.Header.Set("Fastly-Key", s.token) + req.Header.Set("Accept", "application/json") + resp, err := s.client.Do(req.WithContext(ctx)) + if err != nil { + levelForError(s.logger, err).Log("during", "execute request", "err", err) + return name, apiResultError, time.Second, ts, nil + } + + var response origin.Response if err := jsoniterAPI.NewDecoder(resp.Body).Decode(&response); err != nil { resp.Body.Close() level.Error(s.logger).Log("during", "decode response", "err", err) @@ -164,7 +254,7 @@ func (s *Subscriber) query(ctx context.Context, ts uint64) (currentName string, } else { result = apiResultSuccess } - gen.Process(&response, s.serviceID, name, version, s.metrics) + origin.Process(&response, s.serviceID, name, version, s.metrics.Origin) s.postprocess() case http.StatusUnauthorized, http.StatusForbidden: diff --git a/pkg/rt/subscriber_test.go b/pkg/rt/subscriber_test.go index 4ea839a..e9019c0 100644 --- a/pkg/rt/subscriber_test.go +++ b/pkg/rt/subscriber_test.go @@ -2,13 +2,14 @@ package rt_test import ( "context" + "errors" "sync/atomic" "testing" "time" "github.com/fastly/fastly-exporter/pkg/api" "github.com/fastly/fastly-exporter/pkg/filter" - "github.com/fastly/fastly-exporter/pkg/gen" + "github.com/fastly/fastly-exporter/pkg/prom" "github.com/fastly/fastly-exporter/pkg/rt" "github.com/prometheus/client_golang/prometheus" ) @@ -19,9 +20,10 @@ func TestSubscriberFixture(t *testing.T) { subsystem = "testsystem" registry = prometheus.NewRegistry() nameFilter = filter.Filter{} - metrics = gen.NewMetrics(namespace, subsystem, nameFilter, registry) + metrics = prom.NewMetrics(namespace, subsystem, nameFilter, registry) ) + // Set up a subscriber. var ( client = newMockRealtimeClient(rtResponseFixture, `{}`) serviceID = "my-service-id" @@ -33,37 +35,44 @@ func TestSubscriberFixture(t *testing.T) { options = []rt.SubscriberOption{rt.WithMetadataProvider(cache), rt.WithPostprocess(postprocess)} subscriber = rt.NewSubscriber(client, "irrelevant token", serviceID, metrics, options...) ) + + // Prep the mock cache. cache.update([]api.Service{{ID: serviceID, Name: serviceName, Version: serviceVersion}}) - var ( - ctx, cancel = context.WithCancel(context.Background()) - done = make(chan struct{}) - ) - go func() { - subscriber.Run(ctx) - close(done) - }() + // Tell the subscriber to fetch real-time stats. + ctx, cancel := context.WithCancel(context.Background()) + errc := make(chan error, 1) + go func() { errc <- subscriber.RunRealtime(ctx) }() + // Block until the subscriber does finishes one fetch <-processed + // Assert the Prometheus metrics. output := prometheusOutput(t, registry, namespace+"_"+subsystem+"_") assertMetricOutput(t, expetedMetricsOutputMap, output) + // Kill the subscriber's goroutine, and wait for it to finish. cancel() - <-done + err := <-errc + switch { + case err == nil: + case errors.Is(err, context.Canceled): + case err != nil: + t.Fatal(err) + } } func TestSubscriberNoData(t *testing.T) { var ( client = newMockRealtimeClient(`{"Error": "No data available, please retry"}`, `{}`) registry = prometheus.NewRegistry() - metrics = gen.NewMetrics("ns", "ss", filter.Filter{}, registry) + metrics = prom.NewMetrics("ns", "ss", filter.Filter{}, registry) processed = make(chan struct{}, 100) postprocess = func() { processed <- struct{}{} } options = []rt.SubscriberOption{rt.WithPostprocess(postprocess)} subscriber = rt.NewSubscriber(client, "token", "service_id", metrics, options...) ) - go subscriber.Run(context.Background()) + go subscriber.RunRealtime(context.Background()) <-processed // No data client.advance() @@ -82,10 +91,10 @@ func TestSubscriberNoData(t *testing.T) { func TestBadTokenNoSpam(t *testing.T) { var ( client = &countingRealtimeClient{code: 403, response: `{"Error": "unauthorized"}`} - metrics = gen.NewMetrics("namespace", "subsystem", filter.Filter{}, prometheus.NewRegistry()) + metrics = prom.NewMetrics("namespace", "subsystem", filter.Filter{}, prometheus.NewRegistry()) subscriber = rt.NewSubscriber(client, "presumably bad token", "service ID", metrics) ) - go subscriber.Run(context.Background()) + go subscriber.RunRealtime(context.Background()) time.Sleep(time.Second)