From 65fafdf5b8490d0a37334385d210c5e68f769d75 Mon Sep 17 00:00:00 2001 From: Aleksandr Ganin Date: Fri, 10 Jul 2026 21:28:21 +0300 Subject: [PATCH 1/3] feat(metrics): serve /metrics endpoint with prometheus-save-to-file Adds an experimental HTTP endpoint that exposes a profile's prometheus-save-to-file together with the Go runtime metrics on /metrics, so a sidecar Prometheus can scrape the last backup status from either the schedule command or a stand-alone sidecar. Two integration points, both opt-in via the hidden --metrics-port flag (or RESTICPROFILE_METRICS_PORT): * resticprofile --metrics-port schedule --all schedules cron jobs as usual, then blocks on SIGINT/SIGTERM serving /metrics for every profile and group. Textfile paths are de-duplicated across profiles that share one. * resticprofile --metrics-port serve-metrics long-running single-profile sidecar. Response contract: * 200 OK runtime metrics + textfile body * 503 Service textfile not yet created (first backup has not run). Unavailable Prometheus treats this as up == 0 instead of a 404 every scrape. * 500 Internal other I/O errors. The endpoint binds on 0.0.0.0: for network_mode: host containers. The serve command no longer mounts a companion metrics server; the two endpoints above cover the documented use cases without coupling the metrics feature to the experimental client/server mode. --- commands.go | 12 ++ commands_schedule.go | 86 ++++++++++ commands_schedule_test.go | 95 +++++++++++ docs/content/changelog.md | 23 +++ docs/content/monitoring/prometheus.md | 44 ++++- flags.go | 5 + metrics.go | 192 ++++++++++++++++++++++ metrics_test.go | 224 ++++++++++++++++++++++++++ serve_metrics.go | 48 ++++++ 9 files changed, 728 insertions(+), 1 deletion(-) create mode 100644 metrics.go create mode 100644 metrics_test.go create mode 100644 serve_metrics.go diff --git a/commands.go b/commands.go index 1aaf13c19..9baa50c01 100644 --- a/commands.go +++ b/commands.go @@ -189,6 +189,18 @@ func getOwnCommands() []ownCommand { hide: true, experimental: true, }, + { + name: "serve-metrics", + description: "serve the profile's prometheus-save-to-file (and Go runtime metrics) on /metrics", + action: serveMetricsCommand, + needConfiguration: true, + noProfile: false, + hide: true, + experimental: true, + flags: map[string]string{ + "--metrics-port": "port to expose /metrics on (overrides RESTICPROFILE_METRICS_PORT)", + }, + }, } } diff --git a/commands_schedule.go b/commands_schedule.go index fbe2d0ec4..e541bc0f5 100644 --- a/commands_schedule.go +++ b/commands_schedule.go @@ -4,8 +4,11 @@ import ( "errors" "fmt" "maps" + "os" + "os/signal" "slices" "strings" + "syscall" "github.com/creativeprojects/clog" "github.com/creativeprojects/resticprofile/config" @@ -72,9 +75,92 @@ func createSchedule(ctx commandContext) error { } } + // Optional: if --metrics-port is set, keep the process alive and serve + // the profiles' prometheus-save-to-file on /metrics. This lets the schedule + // command double as a metrics sidecar when running under crond in a container. + if err := blockForMetricsServer(ctx); err != nil { + return err + } + return nil } +// blockForMetricsServer starts a /metrics HTTP server for the prometheus-save-to-file +// of the selected profiles, then blocks until SIGINT/SIGTERM. Returns nil immediately +// (and the function returns) when --metrics-port is not set or no profile has a +// prometheus-save-to-file configured. A warning is logged in the latter case so the +// user knows metrics were requested but no file is available. +func blockForMetricsServer(ctx commandContext) error { + if ctx.flags.metricsPort <= 0 { + return nil + } + + metricFiles := collectPrometheusSaveToFiles(ctx) + if len(metricFiles) == 0 { + clog.Warningf("metrics server not started: no profile has prometheus-save-to-file set") + return nil + } + + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(quit) + + errCh := make(chan error, len(metricFiles)) + for _, file := range metricFiles { + file := file + go func() { + if err := newMetricsServer(ctx.flags.metricsPort, file).run(quit); err != nil { + clog.Errorf("metrics server failed for %q: %v", file, err) + errCh <- err + } + }() + } + + clog.Infof("schedule command will keep the process alive for the metrics server; send SIGINT/SIGTERM to stop") + select { + case <-quit: + clog.Info("shutting down metrics server (schedule)") + return nil + case err := <-errCh: + // best-effort: close quit so all running goroutines shut down + close(quit) + return err + } +} + +// collectPrometheusSaveToFiles returns the unique non-empty prometheus-save-to-file +// paths of all profiles selected by the schedule command (--all or a specific profile). +// If a group is selected, the prometheus-save-to-file of each member profile is used. +func collectPrometheusSaveToFiles(ctx commandContext) []string { + seen := make(map[string]struct{}) + var files []string + args := ctx.request.arguments + add := func(name string) { + profile, err := ctx.config.GetProfile(name) + if err != nil || profile == nil || profile.PrometheusSaveToFile == "" { + return + } + if _, ok := seen[profile.PrometheusSaveToFile]; ok { + return + } + seen[profile.PrometheusSaveToFile] = struct{}{} + files = append(files, profile.PrometheusSaveToFile) + } + + for _, name := range selectProfilesAndGroups(ctx.config, ctx.request.profile, args) { + if ctx.config.HasProfile(name) { + add(name) + continue + } + if group, err := ctx.config.GetProfileGroup(name); err == nil && group != nil { + for _, member := range group.Profiles { + add(member) + } + } + } + return files +} + func removeSchedule(ctx commandContext) error { var err error c := ctx.config diff --git a/commands_schedule_test.go b/commands_schedule_test.go index 4d5914070..c2b02a80b 100644 --- a/commands_schedule_test.go +++ b/commands_schedule_test.go @@ -298,3 +298,98 @@ func TestCreateScheduleOverwriteExistingIntegrationUsingCrontab(t *testing.T) { assert.NotContains(t, output.String(), "Original form: *-*-* *:10,40:00") // previous one t.Log(output.String()) } + +func TestCollectPrometheusSaveToFiles(t *testing.T) { + const configYAML = ` +version: "2" + +groups: + group-a: + profiles: + - alpha + - beta + +profiles: + alpha: + prometheus-save-to-file: /var/metrics/alpha.prom + beta: + prometheus-save-to-file: /var/metrics/beta.prom + gamma: + prometheus-save-to-file: /var/metrics/shared.prom + delta: + prometheus-save-to-file: /var/metrics/shared.prom + epsilon: {} +` + testCases := []struct { + name string + arguments []string + profile string + expected []string + matcher func(t assert.TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool + }{ + { + name: "single profile", + arguments: []string{"schedule"}, + profile: "alpha", + expected: []string{"/var/metrics/alpha.prom"}, + matcher: assert.Equal, + }, + { + name: "all profiles dedups shared file", + arguments: []string{"schedule", "--all"}, + profile: "", + // Order is non-deterministic (viper map iteration); assert set + // membership via ElementsMatch so the test isn't order-sensitive. + expected: []string{"/var/metrics/shared.prom", "/var/metrics/alpha.prom", "/var/metrics/beta.prom"}, + matcher: assert.ElementsMatch, + }, + { + name: "group aggregates member files", + arguments: []string{"schedule"}, + profile: "group-a", + expected: []string{"/var/metrics/alpha.prom", "/var/metrics/beta.prom"}, + matcher: assert.ElementsMatch, + }, + { + name: "profile without prometheus-save-to-file yields nothing", + arguments: []string{"schedule"}, + profile: "epsilon", + expected: nil, + matcher: assert.Equal, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cfg, err := config.Load(bytes.NewBufferString(configYAML), config.FormatYAML, config.WithConfigFile("config.yaml")) + require.NoError(t, err) + + ctx := commandContext{ + Context: Context{ + config: cfg, + request: Request{ + profile: tc.profile, + arguments: tc.arguments, + }, + }, + } + + got := collectPrometheusSaveToFiles(ctx) + tc.matcher(t, tc.expected, got) + }) + } +} + +func TestBlockForMetricsServerDisabledByDefault(t *testing.T) { + cfg, err := config.Load(bytes.NewBufferString("version: \"2\"\nprofiles:\n alpha:\n prometheus-save-to-file: /tmp/x.prom\n"), config.FormatYAML, config.WithConfigFile("config.yaml")) + require.NoError(t, err) + + ctx := commandContext{ + Context: Context{ + config: cfg, + request: Request{profile: "alpha"}, + flags: commandLineFlags{metricsPort: 0}, // explicitly disabled + }, + } + assert.NoError(t, blockForMetricsServer(ctx)) +} diff --git a/docs/content/changelog.md b/docs/content/changelog.md index 11573cb3a..887ef0119 100644 --- a/docs/content/changelog.md +++ b/docs/content/changelog.md @@ -5,6 +5,29 @@ title: Release Notes weight: 8 --- +# v0.34.0 (unreleased) + +## New Features + +### Prometheus metrics endpoint [EXPERIMENTAL] + +resticprofile can now expose the contents of a profile's `prometheus-save-to-file` on an HTTP endpoint, alongside the Go runtime metrics, so a sidecar Prometheus can scrape the last backup status from `schedule` and `serve-metrics` commands. + +Two ways to use it: + +- `resticprofile --metrics-port schedule --all` — run `schedule` as usual; once the cron entries are installed the process blocks on SIGINT/SIGTERM, serving `/metrics` for every selected profile and group (textfile paths are de-duplicated). +- `resticprofile --metrics-port serve-metrics` — long-running sidecar serving a single profile's textfile. Useful when you can't or don't want to pass `--metrics-port` to the scheduler. + +Both honour `RESTICPROFILE_METRICS_PORT` and the `--metrics-port` flag is hidden from `--help` until the feature is stabilized. + +Response contract: + +- `200 OK` — runtime metrics + textfile body +- `503 Service Unavailable` — textfile not yet created (e.g. first backup has not run). Prometheus treats this as `up == 0` rather than logging a 404 every scrape. +- `500 Internal Server Error` — other I/O errors + +The `/metrics` endpoint binds on `0.0.0.0:` for compatibility with container environments using `network_mode: host`. + # v0.33.0 (2026-04-04) ## 🌸 Easter release 🥚 diff --git a/docs/content/monitoring/prometheus.md b/docs/content/monitoring/prometheus.md index bde4cf03f..f76538740 100644 --- a/docs/content/monitoring/prometheus.md +++ b/docs/content/monitoring/prometheus.md @@ -229,4 +229,46 @@ root: {{< /tabs >}} -This adds the `host` label to all your metrics. \ No newline at end of file +This adds the `host` label to all your metrics. + +## Exposing metrics over HTTP [EXPERIMENTAL] + +In addition to the file-based and push-based options, resticprofile can serve the latest metrics on an HTTP endpoint. The endpoint combines the Go runtime collector with the profile's `prometheus-save-to-file` body so a single sidecar can be scraped by Prometheus, Node Exporter's `--collector.textfile.directory`, or anything else that understands the Prometheus text exposition format. + +The feature is opt-in and hidden from `--help` until it stabilises. Enable it via the `--metrics-port` flag (or the `RESTICPROFILE_METRICS_PORT` environment variable) and select one of the following integration points: + +### `schedule` keeps the process alive and serves `/metrics` + +When `resticprofile` is used as the scheduler under `crond` (e.g. inside a container), it can double as a metrics sidecar. The scheduler registers all cron jobs as usual, then blocks on SIGINT/SIGTERM while serving `/metrics` for each profile that has `prometheus-save-to-file` set. Textfile paths are de-duplicated across profiles and groups, so several profiles writing to the same file share a single scrape target. + +```sh +resticprofile --metrics-port 19302 schedule --all & +crond -f +``` + +### `serve-metrics` as a stand-alone sidecar + +If you don't use `schedule` (or already have a separate scheduler), run a dedicated process: + +```sh +resticprofile --metrics-port 19302 default serve-metrics +``` + +### Response contract + +- `200 OK` — Go runtime metrics followed by the textfile body. +- `503 Service Unavailable` — the textfile does not exist yet (e.g. the first backup has not run). Prometheus reports the target as `up == 0` without logging a 404 on every scrape. +- `500 Internal Server Error` — other I/O errors. + +The server binds on `0.0.0.0:` for `network_mode: host` containers. + +### Scrape configuration + +```yaml +scrape_configs: + - job_name: resticprofile + static_configs: + - targets: [':19302'] + labels: + host: backup-server +``` \ No newline at end of file diff --git a/flags.go b/flags.go index 1b51c7fde..ac75e7692 100644 --- a/flags.go +++ b/flags.go @@ -39,6 +39,7 @@ type commandLineFlags struct { ignoreOnBattery int usagesHelp string remote string // url of the remote server to download configuration files from + metricsPort int // port to expose /metrics on (0 = disabled) } func envValueOverride[T any](defaultValue T, keys ...string) T { @@ -94,6 +95,7 @@ func loadFlags(args []string) (*pflag.FlagSet, commandLineFlags, error) { wait: envValueOverride(false, "RESTICPROFILE_WAIT"), ignoreOnBattery: envValueOverride(0, "RESTICPROFILE_IGNORE_ON_BATTERY"), remote: envValueOverride("", "RESTICPROFILE_REMOTE"), + metricsPort: envValueOverride(0, "RESTICPROFILE_METRICS_PORT"), } flagset.BoolVarP(&flags.help, "help", "h", flags.help, "display this help") @@ -119,6 +121,9 @@ func loadFlags(args []string) (*pflag.FlagSet, commandLineFlags, error) { // keep the "remote" flag hidden for now _ = flagset.MarkHidden("remote") + flagset.IntVar(&flags.metricsPort, "metrics-port", flags.metricsPort, "port to expose /metrics on (overrides RESTICPROFILE_METRICS_PORT; 0 disables)") + _ = flagset.MarkHidden("metrics-port") + flagset.SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName { switch name { case "profile": diff --git a/metrics.go b/metrics.go new file mode 100644 index 000000000..c55504fd8 --- /dev/null +++ b/metrics.go @@ -0,0 +1,192 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "time" + + "github.com/creativeprojects/clog" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// metricsServer exposes Go runtime metrics together with the Prometheus textfile +// written by a profile's prometheus-save-to-file on a single /metrics endpoint. +// +// Response contract: +// - 200 OK with the runtime collector followed by the textfile body when the +// file exists and is readable. +// - 503 Service Unavailable when the file does not exist (e.g. the first +// backup has not yet populated it). Prometheus treats 503 as a target +// failure (up == 0) without flooding logs. +// - 500 Internal Server Error for any other I/O error. +// +// Exposing the runtime collector and the textfile on the same /metrics +// endpoint matches what `node_exporter --collector.textfile` does, so the +// resulting file is also a drop-in for `prom.SaveTo`. +type metricsServer struct { + textfile string + listen string + timeouts metricsTimeouts +} + +type metricsTimeouts struct { + readHeader time.Duration + read time.Duration + write time.Duration + idle time.Duration + gracefulMax time.Duration +} + +func defaultMetricsTimeouts() metricsTimeouts { + return metricsTimeouts{ + readHeader: 5 * time.Second, + read: 30 * time.Second, + write: 30 * time.Second, + idle: 120 * time.Second, + gracefulMax: 30 * time.Second, + } +} + +// newMetricsServer returns a server bound on 0.0.0.0:port serving the +// textfile at textfile. The textfile path is operator-controlled via the +// `prometheus-save-to-file` profile setting. +func newMetricsServer(port int, textfile string) *metricsServer { + return &metricsServer{ + textfile: textfile, + listen: fmt.Sprintf(":%d", port), + timeouts: defaultMetricsTimeouts(), + } +} + +// handler returns the /metrics handler. Promhttp always emits valid +// exposition-format bodies terminated with a newline, so we can append +// the textfile body without a separator. +func (m *metricsServer) handler() http.Handler { + runtime := promhttp.Handler() + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + data, err := os.ReadFile(m.textfile) //nolint:gosec // textfile path is operator-supplied + if err != nil { + if errors.Is(err, os.ErrNotExist) { + clog.Debugf("metrics textfile %q not found yet", m.textfile) + http.Error(resp, fmt.Sprintf("metrics file %q not found", m.textfile), http.StatusServiceUnavailable) + return + } + http.Error(resp, fmt.Sprintf("reading metrics file: %v", err), http.StatusInternalServerError) + return + } + + // Let promhttp own the response headers, status and body prefix. + // wrapWriter tees every Write through `resp`, so anything promhttp + // writes reaches the wire and we can append our textfile body + // afterwards without having to know whether promhttp will gzip, + // negotiate content-type, or pick a non-200 status. + ww := &wrapWriter{ResponseWriter: resp} + runtime.ServeHTTP(ww, req) + + if ww.status >= 400 { + // Runtime collector reported an error (or panic in user code). + // Do not append the textfile body — the exposition format would + // be invalid and Prometheus would log a parse error. + return + } + + if len(data) == 0 { + return + } + if data[len(data)-1] != '\n' { + if _, err := ww.Write([]byte{'\n'}); err != nil { + clog.Debugf("failed to write newline before textfile body: %v", err) + return + } + } + if _, err := ww.Write(data); err != nil { + clog.Debugf("failed to write textfile body: %v", err) + } + }) +} + +// wrapWriter is an http.ResponseWriter that forwards every call to the +// underlying writer while keeping track of the status code, so the handler +// can decide after-the-fact whether it is safe to append more bytes. +type wrapWriter struct { + http.ResponseWriter + status int + wroteHeader bool +} + +func (w *wrapWriter) WriteHeader(code int) { + if w.wroteHeader { + return // ignore the second call, like net/http does + } + w.status = code + w.wroteHeader = true + w.ResponseWriter.WriteHeader(code) +} + +func (w *wrapWriter) Write(b []byte) (int, error) { + if !w.wroteHeader { + // Mirroring net/http default behaviour: a Write without an explicit + // WriteHeader is treated as 200 OK. + w.status = http.StatusOK + w.wroteHeader = true + } + return w.ResponseWriter.Write(b) +} + +// Flush forwards to the underlying writer when it supports http.Flusher, +// which promhttp uses for streaming large metric sets. +func (w *wrapWriter) Flush() { + if f, ok := w.ResponseWriter.(http.Flusher); ok { + f.Flush() + } +} + +// run starts the HTTP server and blocks until quit is closed. On shutdown it +// gives the server up to m.timeouts.gracefulMax to drain in-flight requests. +// Returns nil on clean shutdown (including http.ErrServerClosed), the bind +// or runtime error otherwise. +// +// The returned error can come from either ListenAndServe (e.g. EADDRINUSE) or +// from the shutdown goroutine; we surface whichever fires first so the caller +// can react without waiting for the process to exit. +func (m *metricsServer) run(quit <-chan os.Signal) error { + server := &http.Server{ + Addr: m.listen, + Handler: m.handler(), + ReadHeaderTimeout: m.timeouts.readHeader, + ReadTimeout: m.timeouts.read, + WriteTimeout: m.timeouts.write, + IdleTimeout: m.timeouts.idle, + } + + errCh := make(chan error, 2) + + go func() { + <-quit + clog.Info("shutting down the metrics server") + ctx, cancel := context.WithTimeout(context.Background(), m.timeouts.gracefulMax) + defer cancel() + if err := server.Shutdown(ctx); err != nil && !errors.Is(err, http.ErrServerClosed) { + clog.Errorf("error while shutting down the metrics server: %v", err) + errCh <- err + } + }() + + go func() { + err := server.ListenAndServe() + if err == nil || errors.Is(err, http.ErrServerClosed) { + errCh <- nil + } else { + errCh <- err + } + }() + + clog.Infof("metrics server listening on %s (textfile=%s)", server.Addr, m.textfile) + if err := <-errCh; err != nil { + return err + } + return nil +} diff --git a/metrics_test.go b/metrics_test.go new file mode 100644 index 000000000..5f221cba4 --- /dev/null +++ b/metrics_test.go @@ -0,0 +1,224 @@ +package main + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestMetricsHandlerMissingFile confirms that calling /metrics on a server +// whose textfile does not exist returns 503 Service Unavailable, with a +// message that names the missing path. Prometheus treats this as +// `up == 0` rather than logging a 404 every scrape. +func TestMetricsHandlerMissingFile(t *testing.T) { + dir := t.TempDir() + textfile := filepath.Join(dir, "does-not-exist.prom") + srv := httptestMetricsServer(t, textfile) + resp := mustGetMetricsIdentity(t, srv.URL) + body := mustReadBody(t, resp) + resp.Body.Close() + + assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) + assert.Contains(t, body, "not found") + assert.Contains(t, body, textfile) +} + +// TestMetricsHandlerEmitsRuntimeAndTextfile confirms that, when the textfile +// exists, the response contains both the runtime collector output +// (`go_goroutines` is a stable signal we can grep for) and the body of the +// textfile appended verbatim after the runtime block. +func TestMetricsHandlerEmitsRuntimeAndTextfile(t *testing.T) { + dir := t.TempDir() + textfile := filepath.Join(dir, "self.prom") + require.NoError(t, os.WriteFile(textfile, []byte("# HELP x test counter\nx_total 7\n"), 0o644)) + + srv := httptestMetricsServer(t, textfile) + resp := mustGetMetricsIdentity(t, srv.URL) + body := mustReadBody(t, resp) + resp.Body.Close() + + require.Equal(t, http.StatusOK, resp.StatusCode) + assert.Contains(t, body, "go_goroutines", "runtime metrics should be present") + assert.Contains(t, body, "x_total 7", "textfile body should be appended verbatim") + // The textfile itself ends with a newline; we append it as-is after + // promhttp's last line. The response must therefore end with the + // textfile's final line. + assert.True(t, strings.HasSuffix(body, "x_total 7\n"), + "textfile body must be the last segment and end with a newline") +} + +// TestMetricsHandlerTrailingNewlineGuard confirms the handler appends the +// textfile body with at least one separating newline, even when the file +// body itself does not end with a newline. +func TestMetricsHandlerTrailingNewlineGuard(t *testing.T) { + dir := t.TempDir() + textfile := filepath.Join(dir, "no-trailing-newline.prom") + require.NoError(t, os.WriteFile(textfile, []byte("missing_newline_at_end 1"), 0o644)) + + srv := httptestMetricsServer(t, textfile) + resp := mustGetMetricsIdentity(t, srv.URL) + body := mustReadBody(t, resp) + resp.Body.Close() + + require.Equal(t, http.StatusOK, resp.StatusCode) + assert.Contains(t, body, "go_goroutines") + // The runtime collector always ends its body with a newline, so our + // appended content is on a fresh line regardless of the file's tail. + assert.Contains(t, body, "missing_newline_at_end 1") + assert.True(t, + strings.Contains(body, "\nmissing_newline_at_end 1") || + strings.HasSuffix(body, "\nmissing_newline_at_end 1\n"), + "textfile line must start at a newline boundary, not glued to a prior partial line") +} + +// TestMetricsServerGracefulShutdown starts the metrics server on a free port, +// scrapes /metrics once to make sure it's listening, and verifies that +// closing quit leads to a clean shutdown within the configured graceful window. +func TestMetricsServerGracefulShutdown(t *testing.T) { + dir := t.TempDir() + textfile := filepath.Join(dir, "shutdown.prom") + require.NoError(t, os.WriteFile(textfile, []byte("# shutdown\n"), 0o644)) + + srv := newMetricsServer(getFreePort(t), textfile) + srv.timeouts.gracefulMax = 2 * time.Second // tighter for tests + + quit := make(chan os.Signal, 1) + done := make(chan error, 1) + go func() { done <- srv.run(quit) }() + + port := portFromAddr(t, srv.listen) + baseURL := fmt.Sprintf("http://127.0.0.1:%d", port) + waitForServer(t, baseURL+"/metrics") + + resp := mustGetMetricsIdentity(t, baseURL) + resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + quit <- os.Interrupt + select { + case err := <-done: + assert.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("server did not shut down in time after quit was closed") + } +} + +// TestMetricsServerPortInUse confirms that two servers racing for the same +// port produce a non-nil error from ListenAndServe (we surface it via .run). +// This is the realistic failure mode under `docker-compose scale`. +// +// Both servers bind on ":N" (all interfaces), so the second one must hit +// EADDRINUSE regardless of which interface the OS hands out. +func TestMetricsServerPortInUse(t *testing.T) { + dir := t.TempDir() + textfile := filepath.Join(dir, "occupied.prom") + require.NoError(t, os.WriteFile(textfile, []byte("# x\n"), 0o644)) + + port := getAllInterfacesFreePort(t) + + first := newMetricsServer(port, textfile) + first.timeouts.gracefulMax = 1 * time.Second + quit1 := make(chan os.Signal, 1) + done1 := make(chan error, 1) + go func() { done1 <- first.run(quit1) }() + + baseURL := fmt.Sprintf("http://127.0.0.1:%d", port) + waitForServer(t, baseURL+"/metrics") + + // Second server on the same port must fail to bind. + second := newMetricsServer(port, textfile) + second.timeouts.gracefulMax = 1 * time.Second + quit2 := make(chan os.Signal, 1) + done2 := make(chan error, 1) + go func() { done2 <- second.run(quit2) }() + + select { + case err := <-done2: + require.Error(t, err, "expected the second server to fail to bind") + assert.NotContains(t, err.Error(), http.ErrServerClosed.Error()) + case <-time.After(3 * time.Second): + t.Fatal("second server did not surface a bind error in time") + } + + // Tear down the first server. + close(quit1) + select { + case <-done1: + case <-time.After(5 * time.Second): + t.Fatal("first server did not shut down in time after quit was closed") + } + + // second.run already returned its bind error above and is no longer + // listening — nothing else to drain for it. +} + +// getAllInterfacesFreePort returns a port number that is currently free on +// 0.0.0.0 (all interfaces), matching how the production metrics server binds. +func getAllInterfacesFreePort(t *testing.T) int { + t.Helper() + lc := net.ListenConfig{} + l, err := lc.Listen(context.Background(), "tcp", ":0") + require.NoError(t, err) + port := l.Addr().(*net.TCPAddr).Port + require.NoError(t, l.Close()) + return port +} + +// httptestMetricsServer wires a *metricsServer.handler behind httptest.Server +// on 127.0.0.1:, so the tests above get a real HTTP roundtrip +// without depending on the package-level ListenAndServe path. +func httptestMetricsServer(t *testing.T, textfile string) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.Handle("GET /metrics", newMetricsServer(0, textfile).handler()) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +// portFromAddr pulls the numeric port from an ":NNNN" or "host:NNNN" address. +// The metrics server always uses ":NNNN" (all interfaces), so the host part +// is irrelevant for tests. +func portFromAddr(t *testing.T, addr string) int { + t.Helper() + _, portStr, err := net.SplitHostPort(addr) + require.NoError(t, err) + port, err := strconv.Atoi(portStr) + require.NoError(t, err) + return port +} + +// mustGetMetricsIdentity issues GET /metrics with Accept-Encoding: identity +// so the response body is plain text (promhttp auto-compresses otherwise and +// Go's default http.Client sets Accept-Encoding: gzip). +func mustGetMetricsIdentity(t *testing.T, baseURL string) *http.Response { + t.Helper() + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, baseURL+"/metrics", nil) + require.NoError(t, err) + req.Header.Set("Accept-Encoding", "identity") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + return resp +} + +// mustReadBody reads the entire HTTP response body and returns it as a string. +// Test helper — fatal on read errors so callers can keep the test body short. +func mustReadBody(t *testing.T, resp *http.Response) string { + t.Helper() + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + return string(data) +} diff --git a/serve_metrics.go b/serve_metrics.go new file mode 100644 index 000000000..807d574de --- /dev/null +++ b/serve_metrics.go @@ -0,0 +1,48 @@ +package main + +import ( + "fmt" + "os" + "os/signal" + + "github.com/creativeprojects/clog" +) + +// serveMetricsCommand exposes the active profile's prometheus-save-to-file +// together with Go runtime metrics on /metrics. It is intended for two +// scenarios: +// +// 1. As a long-running sidecar next to `resticprofile schedule`, sharing +// the same prometheus-save-to-file so Prometheus can scrape the latest +// backup status without polling per-profile sidecars. +// 2. As a stand-alone helper to debug what's actually written to the +// textfile by `resticprofile backup`. +// +// The command is registered as experimental/hidden; the recommended path is +// `resticprofile --metrics-port schedule --all`, which keeps the +// process alive and blocks on SIGINT/SIGTERM. +func serveMetricsCommand(ctx commandContext) error { + port := ctx.flags.metricsPort + if port <= 0 { + return fmt.Errorf("missing metrics port: set --metrics-port or RESTICPROFILE_METRICS_PORT") + } + + profileName := ctx.request.profile + if profileName == "" { + return fmt.Errorf("missing profile: pass --name or use the .%s form", ctx.request.command) + } + profile, err := ctx.config.GetProfile(profileName) + if err != nil { + return fmt.Errorf("loading profile %q: %w", profileName, err) + } + if profile.PrometheusSaveToFile == "" { + return fmt.Errorf("profile %q has no prometheus-save-to-file configured", profileName) + } + + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt) + defer signal.Stop(quit) + + clog.Infof("serving metrics for profile %q", profileName) + return newMetricsServer(port, profile.PrometheusSaveToFile).run(quit) +} From b01a75729c27673e91cad321f22758a616e8322d Mon Sep 17 00:00:00 2001 From: Aleksandr Ganin Date: Wed, 22 Jul 2026 16:58:02 +0300 Subject: [PATCH 2/3] =?UTF-8?q?=D0=A1=D0=BE=D0=B1=D0=B8=D1=80=D0=B0=D0=B5?= =?UTF-8?q?=D0=BC=20=D1=81=D0=B0=D0=BC=D0=B8,=20=D0=BF=D0=BE=D0=BA=D0=B0?= =?UTF-8?q?=20PR=20=D0=BD=D0=B5=20=D0=BF=D1=80=D0=B8=D0=BD=D1=8F=D1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .goreleaser-docker-only.yml | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/.goreleaser-docker-only.yml b/.goreleaser-docker-only.yml index ca30f2366..ef1486f58 100644 --- a/.goreleaser-docker-only.yml +++ b/.goreleaser-docker-only.yml @@ -29,10 +29,8 @@ archives: dockers: - image_templates: - - "creativeprojects/resticprofile:latest-amd64" - - "creativeprojects/resticprofile:{{ .RawVersion }}-amd64" - - "ghcr.io/creativeprojects/resticprofile:latest-amd64" - - "ghcr.io/creativeprojects/resticprofile:{{ .RawVersion }}-amd64" + - "ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:latest-amd64" + - "ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:{{ .RawVersion }}-amd64" ids: - resticprofile_targz use: buildx @@ -52,10 +50,8 @@ dockers: - "--build-arg=ARCH=amd64" - image_templates: - - "creativeprojects/resticprofile:latest-arm64v8" - - "creativeprojects/resticprofile:{{ .RawVersion }}-arm64v8" - - "ghcr.io/creativeprojects/resticprofile:latest-arm64v8" - - "ghcr.io/creativeprojects/resticprofile:{{ .RawVersion }}-arm64v8" + - "ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:latest-arm64v8" + - "ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:{{ .RawVersion }}-arm64v8" ids: - resticprofile_targz use: buildx @@ -75,25 +71,15 @@ dockers: - "--build-arg=ARCH=arm64" docker_manifests: - - name_template: creativeprojects/resticprofile:{{ .RawVersion }} + - name_template: ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:{{ .RawVersion }} image_templates: - - creativeprojects/resticprofile:{{ .RawVersion }}-amd64 - - creativeprojects/resticprofile:{{ .RawVersion }}-arm64v8 + - ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:{{ .RawVersion }}-amd64 + - ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:{{ .RawVersion }}-arm64v8 - - name_template: creativeprojects/resticprofile:latest + - name_template: ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:latest image_templates: - - creativeprojects/resticprofile:latest-amd64 - - creativeprojects/resticprofile:latest-arm64v8 - - - name_template: ghcr.io/creativeprojects/resticprofile:{{ .RawVersion }} - image_templates: - - ghcr.io/creativeprojects/resticprofile:{{ .RawVersion }}-amd64 - - ghcr.io/creativeprojects/resticprofile:{{ .RawVersion }}-arm64v8 - - - name_template: ghcr.io/creativeprojects/resticprofile:latest - image_templates: - - ghcr.io/creativeprojects/resticprofile:latest-amd64 - - ghcr.io/creativeprojects/resticprofile:latest-arm64v8 + - ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:latest-amd64 + - ghcr.io/{{ .Env.GITHUB_REPOSITORY }}:latest-arm64v8 release: disable: true From 008bfba59813dfd264f1c8dbaf6e938c3aa0d982 Mon Sep 17 00:00:00 2001 From: Aleksandr Ganin Date: Thu, 23 Jul 2026 13:26:02 +0300 Subject: [PATCH 3/3] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=20gzip,=20prometheus=20=D0=BF=D0=BE=20=D1=83=D0=BC=D0=BE?= =?UTF-8?q?=D0=BB=D1=87=D0=B0=D0=BD=D0=B8=D1=8E=20=D0=B6=D0=B4=D0=B5=D1=82?= =?UTF-8?q?=20=D1=81=D0=B6=D0=B0=D1=82=D1=8B=D0=B5=20=D0=B4=D0=B0=D0=BD?= =?UTF-8?q?=D0=BD=D1=8B=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics.go | 56 +++++++++++++++++++++++++++++++++++++++++++------ metrics_test.go | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/metrics.go b/metrics.go index c55504fd8..577b45347 100644 --- a/metrics.go +++ b/metrics.go @@ -1,14 +1,18 @@ package main import ( + "compress/gzip" "context" "errors" "fmt" + "io" "net/http" "os" + "strings" "time" "github.com/creativeprojects/clog" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -65,7 +69,14 @@ func newMetricsServer(port int, textfile string) *metricsServer { // exposition-format bodies terminated with a newline, so we can append // the textfile body without a separator. func (m *metricsServer) handler() http.Handler { - runtime := promhttp.Handler() + // Disable promhttp's own gzip: it would compress only its runtime block, + // and the raw textfile we append afterwards would land past the end of + // that gzip member — every scraper then fails with "gzip: invalid header". + // We compress the whole response (runtime + textfile) as one gzip stream + // below instead. + runtime := promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{ + DisableCompression: true, + }) return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { data, err := os.ReadFile(m.textfile) //nolint:gosec // textfile path is operator-supplied if err != nil { @@ -79,11 +90,17 @@ func (m *metricsServer) handler() http.Handler { } // Let promhttp own the response headers, status and body prefix. - // wrapWriter tees every Write through `resp`, so anything promhttp - // writes reaches the wire and we can append our textfile body - // afterwards without having to know whether promhttp will gzip, - // negotiate content-type, or pick a non-200 status. + // wrapWriter tees every body Write through ww.body — the gzip writer + // when the client accepts it, the raw connection otherwise — so we can + // append our textfile body afterwards without having to know whether + // promhttp negotiated content-type or picked a non-200 status. ww := &wrapWriter{ResponseWriter: resp} + if acceptsGzip(req) { + resp.Header().Set("Content-Encoding", "gzip") + gz := gzip.NewWriter(resp) + defer gz.Close() + ww.body = gz + } runtime.ServeHTTP(ww, req) if ww.status >= 400 { @@ -113,6 +130,7 @@ func (m *metricsServer) handler() http.Handler { // can decide after-the-fact whether it is safe to append more bytes. type wrapWriter struct { http.ResponseWriter + body io.Writer // body sink; when nil, writes go straight to ResponseWriter status int wroteHeader bool } @@ -133,17 +151,43 @@ func (w *wrapWriter) Write(b []byte) (int, error) { w.status = http.StatusOK w.wroteHeader = true } + if w.body != nil { + return w.body.Write(b) + } return w.ResponseWriter.Write(b) } // Flush forwards to the underlying writer when it supports http.Flusher, -// which promhttp uses for streaming large metric sets. +// which promhttp uses for streaming large metric sets. When gzipping, the +// gzip writer is flushed first so its buffered bytes reach the connection. func (w *wrapWriter) Flush() { + if gz, ok := w.body.(*gzip.Writer); ok { + _ = gz.Flush() + } if f, ok := w.ResponseWriter.(http.Flusher); ok { f.Flush() } } +// acceptsGzip reports whether the client offered gzip in Accept-Encoding. +// An absent header, a bare "identity" (as the tests send), or an explicit +// "gzip;q=0" all mean no. +func acceptsGzip(req *http.Request) bool { + for _, part := range strings.Split(req.Header.Get("Accept-Encoding"), ",") { + fields := strings.Split(strings.TrimSpace(part), ";") + if !strings.EqualFold(strings.TrimSpace(fields[0]), "gzip") { + continue + } + for _, p := range fields[1:] { + if strings.EqualFold(strings.TrimSpace(p), "q=0") { + return false + } + } + return true + } + return false +} + // run starts the HTTP server and blocks until quit is closed. On shutdown it // gives the server up to m.timeouts.gracefulMax to drain in-flight requests. // Returns nil on clean shutdown (including http.ErrServerClosed), the bind diff --git a/metrics_test.go b/metrics_test.go index 5f221cba4..ddda4ac65 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -1,6 +1,7 @@ package main import ( + "compress/gzip" "context" "fmt" "io" @@ -83,6 +84,43 @@ func TestMetricsHandlerTrailingNewlineGuard(t *testing.T) { "textfile line must start at a newline boundary, not glued to a prior partial line") } +// TestMetricsHandlerGzipRoundTrip confirms that a scraper sending +// Accept-Encoding: gzip (Prometheus and Go's default client do) gets a single, +// valid gzip stream covering both the runtime block and the appended textfile. +// Before the fix promhttp gzipped only its own body and the raw textfile was +// appended past the end of that gzip member, so decoding failed with +// "gzip: invalid header". +func TestMetricsHandlerGzipRoundTrip(t *testing.T) { + dir := t.TempDir() + textfile := filepath.Join(dir, "self.prom") + require.NoError(t, os.WriteFile(textfile, []byte("# HELP x test counter\nx_total 7\n"), 0o644)) + + srv := httptestMetricsServer(t, textfile) + + // Disable the transport's transparent gzip so we decode the body ourselves + // and would observe a corrupt stream if the response were malformed. + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, srv.URL+"/metrics", nil) + require.NoError(t, err) + req.Header.Set("Accept-Encoding", "gzip") + resp, err := (&http.Client{Transport: &http.Transport{DisableCompression: true}}).Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, "gzip", resp.Header.Get("Content-Encoding")) + + gz, err := gzip.NewReader(resp.Body) + require.NoError(t, err, "response must be a single valid gzip stream") + data, err := io.ReadAll(gz) + require.NoError(t, err, "decoding the whole body must not hit invalid gzip data") + require.NoError(t, gz.Close()) + + body := string(data) + assert.Contains(t, body, "go_goroutines", "runtime metrics should be present") + assert.True(t, strings.HasSuffix(body, "x_total 7\n"), + "textfile body must be the last segment inside the same gzip stream") +} + // TestMetricsServerGracefulShutdown starts the metrics server on a free port, // scrapes /metrics once to make sure it's listening, and verifies that // closing quit leads to a clean shutdown within the configured graceful window.