generated from giantswarm/template-operator
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement handling of Alertmanager config (#180)
Co-authored-by: Taylor Bot <[email protected]> Co-authored-by: Quentin Bisson <[email protected]>
- Loading branch information
3 people
authored
Dec 16, 2024
1 parent
6be2b18
commit fbfd0f1
Showing
11 changed files
with
614 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
5 changes: 5 additions & 0 deletions
5
helm/observability-operator/templates/alertmanager/_helpers.tpl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| {{/* vim: set filetype=mustache: */}} | ||
|
|
||
| {{- define "alertmanager-secret.name" -}} | ||
| {{- include "resource.default.name" . -}}-alertmanager | ||
| {{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,150 @@ | ||
| package alertmanager | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "fmt" | ||
| "io" | ||
| "maps" | ||
| "net/http" | ||
| "path" | ||
| "slices" | ||
| "strings" | ||
|
|
||
| "github.com/pkg/errors" | ||
| "github.com/prometheus/alertmanager/config" | ||
| v1 "k8s.io/api/core/v1" | ||
| "sigs.k8s.io/controller-runtime/pkg/log" | ||
| "sigs.k8s.io/yaml" | ||
|
|
||
| common "github.com/giantswarm/observability-operator/pkg/common/monitoring" | ||
| pkgconfig "github.com/giantswarm/observability-operator/pkg/config" | ||
| ) | ||
|
|
||
| const ( | ||
| // Those values are used to retrieve the Alertmanager configuration from the secret named after conf.Monitoring.AlertmanagerSecretName | ||
| // alertmanagerConfigKey is the key to the alertmanager configuration in the secret | ||
| alertmanagerConfigKey = "alertmanager.yaml" | ||
| // templatesSuffix is the suffix used to identify the templates in the secret | ||
| templatesSuffix = ".tmpl" | ||
|
|
||
| alertmanagerAPIPath = "/api/v1/alerts" | ||
|
|
||
| //TODO: get this from somewhere | ||
| tenantID = "anonymous" | ||
| ) | ||
|
|
||
| type Service struct { | ||
| alertmanagerURL string | ||
| } | ||
|
|
||
| // configRequest is the structure used to send the configuration to Alertmanager's API | ||
| // json tags also applies yaml field names | ||
| type configRequest struct { | ||
| TemplateFiles map[string]string `json:"template_files"` | ||
| AlertmanagerConfig string `json:"alertmanager_config"` | ||
| } | ||
|
|
||
| func New(conf pkgconfig.Config) Service { | ||
| service := Service{ | ||
| alertmanagerURL: strings.TrimSuffix(conf.Monitoring.AlertmanagerURL, "/"), | ||
| } | ||
|
|
||
| return service | ||
| } | ||
|
|
||
| func (s Service) Configure(ctx context.Context, secret *v1.Secret) error { | ||
| logger := log.FromContext(ctx) | ||
|
|
||
| logger.Info("Alertmanager: configuring") | ||
|
|
||
| if secret == nil { | ||
| return errors.WithStack(fmt.Errorf("alertmanager: failed to get secret")) | ||
| } | ||
|
|
||
| // Retrieve Alertmanager configuration from secret | ||
| alertmanagerConfigContent, ok := secret.Data[alertmanagerConfigKey] | ||
| if !ok { | ||
| return errors.WithStack(fmt.Errorf("alertmanager: config not found")) | ||
| } | ||
|
|
||
| // Retrieve all alertmanager templates from secret | ||
| templates := make(map[string]string) | ||
| for key, value := range secret.Data { | ||
| if strings.HasSuffix(key, templatesSuffix) { | ||
| // Template key/name should not be a path otherwise the request will fail with: | ||
| // > error validating Alertmanager config: invalid template name "/etc/dummy.tmpl": the template name cannot contain any path | ||
| baseKey := path.Base(key) | ||
| templates[baseKey] = string(value) | ||
| } | ||
| } | ||
|
|
||
| err := s.configure(ctx, alertmanagerConfigContent, templates, tenantID) | ||
| if err != nil { | ||
| return errors.WithStack(fmt.Errorf("alertmanager: failed to configure: %w", err)) | ||
| } | ||
|
|
||
| logger.Info("Alertmanager: configured") | ||
| return nil | ||
| } | ||
|
|
||
| // configure sends the configuration and templates to Mimir Alertmanager's API | ||
| // https://grafana.com/docs/mimir/latest/references/http-api/#set-alertmanager-configuration | ||
| func (s Service) configure(ctx context.Context, alertmanagerConfigContent []byte, templates map[string]string, tenantID string) error { | ||
| logger := log.FromContext(ctx) | ||
|
|
||
| // Load alertmanager configuration | ||
| alertmanagerConfig, err := config.Load(string(alertmanagerConfigContent)) | ||
| if err != nil { | ||
| return errors.WithStack(fmt.Errorf("alertmanager: failed to load configuration: %w", err)) | ||
| } | ||
|
|
||
| // Set template names | ||
| // Values set here must match the keys set in requestData.TemplateFiles | ||
| alertmanagerConfig.Templates = slices.Collect(maps.Keys(templates)) | ||
| alertmanagerConfigString := alertmanagerConfig.String() | ||
|
|
||
| // Prepare request for Alertmanager API | ||
| requestData := configRequest{ | ||
| AlertmanagerConfig: alertmanagerConfigString, | ||
| TemplateFiles: templates, | ||
| } | ||
| data, err := yaml.Marshal(requestData) | ||
| if err != nil { | ||
| return errors.WithStack(fmt.Errorf("alertmanager: failed to marshal yaml: %w", err)) | ||
| } | ||
|
|
||
| url := s.alertmanagerURL + alertmanagerAPIPath | ||
| logger.WithValues("url", url, "data_size", len(data), "config_size", len(alertmanagerConfigString), "templates_count", len(templates)).Info("Alertmanager: sending configuration") | ||
|
|
||
| // Send request to Alertmanager's API | ||
| req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(data)) | ||
| if err != nil { | ||
| return errors.WithStack(fmt.Errorf("alertmanager: failed to create request: %w", err)) | ||
| } | ||
| req.Header.Set(common.OrgIDHeader, tenantID) | ||
|
|
||
| resp, err := http.DefaultClient.Do(req) | ||
| if err != nil { | ||
| return errors.WithStack(fmt.Errorf("alertmanager: failed to send request: %w", err)) | ||
| } | ||
| defer resp.Body.Close() // nolint: errcheck | ||
|
|
||
| logger.WithValues("status_code", resp.StatusCode).Info("Alertmanager: configuration sent") | ||
|
|
||
| if resp.StatusCode != http.StatusCreated { | ||
| respBody, err := io.ReadAll(resp.Body) | ||
| if err != nil { | ||
| return errors.WithStack(fmt.Errorf("alertmanager: failed to read response: %w", err)) | ||
| } | ||
|
|
||
| e := APIError{ | ||
| Code: resp.StatusCode, | ||
| Message: string(respBody), | ||
| } | ||
|
|
||
| return errors.WithStack(fmt.Errorf("alertmanager: failed to send configuration: %w", e)) | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| package alertmanager | ||
|
|
||
| import "fmt" | ||
|
|
||
| type APIError struct { | ||
| Code int | ||
| Message string | ||
| } | ||
|
|
||
| func (e APIError) Error() string { | ||
| return fmt.Sprintf("%d: %s", e.Code, e.Message) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters