From 261497388837a6f2b0ffeeb1cce86b0c02a52db1 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 31 Aug 2026 18:37:18 +0000 Subject: [PATCH 01/13] Add local image tagging API --- cmd/api/api/images.go | 33 + cmd/api/api/images_test.go | 119 +++- cmd/api/main_test.go | 44 ++ lib/builds/manager_test.go | 13 + lib/images/errors.go | 1 + lib/images/manager.go | 22 +- lib/images/storage.go | 34 +- lib/images/tag.go | 106 +++ lib/images/tag_test.go | 207 ++++++ lib/images/testseed.go | 111 +++ lib/images/testutil/testutil.go | 19 + lib/middleware/resolve.go | 7 + lib/middleware/resolve_test.go | 21 + lib/oapi/oapi.go | 1142 +++++++++++++++++++------------ lib/scopes/scopes.go | 9 +- openapi.yaml | 66 ++ stainless.yaml | 2 + 17 files changed, 1479 insertions(+), 477 deletions(-) create mode 100644 lib/images/tag.go create mode 100644 lib/images/tag_test.go create mode 100644 lib/images/testseed.go create mode 100644 lib/images/testutil/testutil.go diff --git a/cmd/api/api/images.go b/cmd/api/api/images.go index dc003370d..812134ab0 100644 --- a/cmd/api/api/images.go +++ b/cmd/api/api/images.go @@ -94,6 +94,39 @@ func (s *ApiService) CreateImage(ctx context.Context, request oapi.CreateImageRe return oapi.CreateImage202JSONResponse(imageToOAPI(*img)), nil } +// TagImage handles POST /images/{name}/tag. +// Note: ResolveResource skips POSTs under /images/{name}, so the source is +// resolved by ImageManager.TagImage and a missing source gets the specific +// 404 body below. +func (s *ApiService) TagImage(ctx context.Context, request oapi.TagImageRequestObject) (oapi.TagImageResponseObject, error) { + if request.Body == nil { + return oapi.TagImage400JSONResponse{ + Code: "invalid_request", + Message: "request body is required", + }, nil + } + + img, err := s.ImageManager.TagImage(ctx, request.Name, request.Body.Target) + if err != nil { + return tagImageErrorResponse(ctx, err, request.Name, request.Body.Target), nil + } + return oapi.TagImage200JSONResponse(imageToOAPI(*img)), nil +} + +func tagImageErrorResponse(ctx context.Context, err error, source, target string) oapi.TagImageResponseObject { + switch { + case errors.Is(err, images.ErrInvalidName): + return oapi.TagImage400JSONResponse{Code: "invalid_name", Message: err.Error()} + case errors.Is(err, images.ErrNotFound): + return oapi.TagImage404JSONResponse{Code: "not_found", Message: "source image not found"} + case errors.Is(err, images.ErrImageNotReady): + return oapi.TagImage409JSONResponse{Code: "image_not_ready", Message: err.Error()} + default: + logger.FromContext(ctx).ErrorContext(ctx, "failed to tag image", "error", err, "source", source, "target", target) + return oapi.TagImage500JSONResponse{Code: "internal_error", Message: "failed to tag image"} + } +} + // GetImage gets image details by name // Note: Resolution is handled by ResolveResource middleware func (s *ApiService) GetImage(ctx context.Context, request oapi.GetImageRequestObject) (oapi.GetImageResponseObject, error) { diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go index bb13a9c02..b6a1857b5 100644 --- a/cmd/api/api/images_test.go +++ b/cmd/api/api/images_test.go @@ -2,13 +2,12 @@ package api import ( "context" - "encoding/json" "fmt" - "os" "testing" "time" "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/images/testutil" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" @@ -37,6 +36,15 @@ func (m *captureCreateImageManager) CreateImage(_ context.Context, req images.Cr return &images.Image{Name: req.Name, Digest: "sha256:test", Status: images.StatusPending, CreatedAt: time.Now()}, nil } +type tagImageErrManager struct { + images.Manager + err error +} + +func (m tagImageErrManager) TagImage(context.Context, string, string) (*images.Image, error) { + return nil, m.err +} + func TestCreateImage_MapsBorrowedCredentials(t *testing.T) { t.Parallel() @@ -505,28 +513,95 @@ func seedReadyDigestOnlyImage(t *testing.T, svc *ApiService, imageRef string, im require.NoError(t, err) require.True(t, ref.IsDigest(), "test helper expects a digest reference") - p := paths.New(svc.Config.DataDir) - digestDir := p.ImageDigestDir(ref.Repository(), ref.DigestHex()) - require.NoError(t, os.MkdirAll(digestDir, 0o755)) - require.NoError(t, os.WriteFile(p.ImageDigestPath(ref.Repository(), ref.DigestHex()), []byte("rootfs"), 0o644)) - - meta := struct { - Name string `json:"name"` - Digest string `json:"digest"` - Status string `json:"status"` - SizeBytes int64 `json:"size_bytes"` - Tags map[string]string `json:"tags,omitempty"` - CreatedAt time.Time `json:"created_at"` + testutil.SeedReadyImage(t, paths.New(svc.Config.DataDir), testutil.Seed{ + Repository: ref.Repository(), + DigestHex: ref.DigestHex(), + Name: imageRef, + Tags: imageTags, + }) +} + +func TestTagImage_ErrorStatusMapping(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + err error + want oapi.TagImageResponseObject }{ - Name: imageRef, - Digest: "sha256:" + ref.DigestHex(), - Status: "ready", - SizeBytes: int64(len("rootfs")), - Tags: imageTags, - CreatedAt: time.Now().UTC(), + { + name: "invalid name -> 400", + err: fmt.Errorf("tag: %w", images.ErrInvalidName), + want: oapi.TagImage400JSONResponse{Code: "invalid_name", Message: "tag: invalid image name"}, + }, + { + name: "not found -> 404", + err: fmt.Errorf("tag: %w", images.ErrNotFound), + want: oapi.TagImage404JSONResponse{Code: "not_found", Message: "source image not found"}, + }, + { + name: "not ready -> 409", + err: fmt.Errorf("tag: %w", images.ErrImageNotReady), + want: oapi.TagImage409JSONResponse{Code: "image_not_ready", Message: "tag: image is not ready"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + svc := &ApiService{ImageManager: tagImageErrManager{err: tc.err}} + resp, err := svc.TagImage(ctx(), oapi.TagImageRequestObject{ + Name: "docker.io/library/alpine:latest", + Body: &oapi.TagImageRequest{Target: "docker.io/library/alpine:stable"}, + }) + require.NoError(t, err) + require.Equal(t, tc.want, resp) + }) } +} + +func TestTagImage_MissingBody(t *testing.T) { + t.Parallel() + + svc := &ApiService{ImageManager: tagImageErrManager{}} + resp, err := svc.TagImage(ctx(), oapi.TagImageRequestObject{ + Name: "docker.io/library/alpine:latest", + }) + require.NoError(t, err) + require.IsType(t, oapi.TagImage400JSONResponse{}, resp) +} + +// seedReadyContentImage writes a ready image into the shared content layout +// plus a repository tag reference, without pulling from a registry. +func seedReadyContentImage(t *testing.T, svc *ApiService, repository, tag, digestHex string) { + t.Helper() + testutil.SeedReadyImage(t, paths.New(svc.Config.DataDir), testutil.Seed{ + Repository: repository, + Tag: tag, + DigestHex: digestHex, + Content: true, + }) +} + +func TestTagImage_Success(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + const repository = "docker.io/library/alpine" + const digestHex = "ab12cd34ab12cd34ab12cd34ab12cd34ab12cd34ab12cd34ab12cd34ab12cd34" + seedReadyContentImage(t, svc, repository, "latest", digestHex) + + resp, err := svc.TagImage(ctx(), oapi.TagImageRequestObject{ + Name: repository + ":latest", + Body: &oapi.TagImageRequest{Target: "registry.example/apps/alpine:v1"}, + }) + require.NoError(t, err) + okResp, ok := resp.(oapi.TagImage200JSONResponse) + require.True(t, ok, "expected 200 response, got %T", resp) + require.Equal(t, "registry.example/apps/alpine:v1", okResp.Name) + require.Equal(t, "sha256:"+digestHex, okResp.Digest) - data, err := json.Marshal(meta) + // The new tag resolves through GetImage. + img, err := svc.ImageManager.GetImage(ctx(), "registry.example/apps/alpine:v1") require.NoError(t, err) - require.NoError(t, os.WriteFile(p.ImageMetadata(ref.Repository(), ref.DigestHex()), data, 0o644)) + require.Equal(t, images.StatusReady, img.Status) } diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 34dbba428..ef88b586b 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -252,6 +252,50 @@ func (h *testImageHandler) GetImage(w http.ResponseWriter, r *http.Request, name } } +func TestTagImageRoute_URLDecodedName(t *testing.T) { + // Verifies the generated POST /images/{name}/tag route exists and that the + // URL-encoded source name is decoded before reaching the handler. + r := chi.NewRouter() + + var receivedName string + handler := &testTagImageHandler{ + tagImage: func(w http.ResponseWriter, r *http.Request, name string) { + receivedName = name + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"name":"` + name + `","digest":"sha256:test","status":"ready","created_at":"2026-01-01T00:00:00Z"}`)) + }, + } + + oapi.HandlerFromMux(handler, r) + + token, err := generateValidJWT("user-123") + require.NoError(t, err) + + body := bytes.NewReader([]byte(`{"target":"docker.io/library/alpine:stable"}`)) + req := httptest.NewRequest(http.MethodPost, "/images/docker.io%2Flibrary%2Falpine%3Alatest/tag", body) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, "docker.io/library/alpine:latest", receivedName) +} + +// testTagImageHandler implements oapi.ServerInterface with just TagImage for testing +type testTagImageHandler struct { + oapi.Unimplemented + tagImage func(w http.ResponseWriter, r *http.Request, name string) +} + +func (h *testTagImageHandler) TagImage(w http.ResponseWriter, r *http.Request, name string) { + if h.tagImage != nil { + h.tagImage(w, r, name) + } +} + func TestImageNameWithSlashes_URLEncoding(t *testing.T) { // This test verifies how chi router handles image names with slashes. // Image names like "docker.io/onkernel/chromium-headful:latest" contain slashes diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index a137edc66..05015bb98 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -373,6 +373,19 @@ func (m *mockImageManager) DeleteImage(ctx context.Context, name string) error { return nil } +func (m *mockImageManager) TagImage(ctx context.Context, source, target string) (*images.Image, error) { + m.mu.Lock() + defer m.mu.Unlock() + img, ok := m.images[source] + if !ok { + return nil, images.ErrNotFound + } + tagged := *img + tagged.Name = target + m.images[target] = &tagged + return &tagged, nil +} + func (m *mockImageManager) RecoverInterruptedBuilds() {} func (m *mockImageManager) TotalImageBytes(ctx context.Context) (int64, error) { diff --git a/lib/images/errors.go b/lib/images/errors.go index 6a63741f0..47be5032c 100644 --- a/lib/images/errors.go +++ b/lib/images/errors.go @@ -17,6 +17,7 @@ var ( ErrNotFound = errors.New("image not found") ErrInvalidName = errors.New("invalid image name") ErrInvalidPlatform = errors.New("invalid platform") + ErrImageNotReady = errors.New("image is not ready") // ErrPlatformNotAvailable means the requested platform is well-formed but the // image's manifest index does not publish a matching variant. Unlike // ErrInvalidPlatform (bad user syntax), the platform itself is valid; the diff --git a/lib/images/manager.go b/lib/images/manager.go index f89fb6678..06e6cbbe0 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -46,6 +46,10 @@ type Manager interface { // Unlike CreateImage, it does not resolve from a remote registry. ImportLocalImage(ctx context.Context, repo, reference, digest string) (*Image, error) GetImage(ctx context.Context, name string) (*Image, error) + // TagImage creates or updates a local tag pointing at an existing ready + // image, without pulling or reconverting. Source and target may be in + // different repositories. + TagImage(ctx context.Context, source, target string) (*Image, error) DeleteImage(ctx context.Context, name string) error RecoverInterruptedBuilds() // TotalImageBytes returns the total size of all ready images on disk. @@ -768,23 +772,7 @@ func (m *manager) GetImage(ctx context.Context, name string) (*Image, error) { return nil, fmt.Errorf("%w: %s", ErrInvalidName, err.Error()) } - repository := ref.Repository() - - var digestHex string - if ref.IsDigest() { - // Direct digest lookup - digestHex = ref.DigestHex() - } else { - // Tag lookup - resolve symlink - tag := ref.Tag() - d, err := resolveTag(m.paths, repository, tag) - if err != nil { - return nil, err - } - digestHex = d - } - - meta, err := readMetadata(m.paths, repository, digestHex) + _, meta, err := resolveRefMetadata(m.paths, ref) if err != nil { return nil, err } diff --git a/lib/images/storage.go b/lib/images/storage.go index 2e3430851..65d433c47 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -240,6 +240,24 @@ func readMetadata(p *paths.Paths, repository, digestHex string) (*imageMetadata, return readMetadataAt(resolveImageLayout(p, repository, digestHex)) } +// resolveRefMetadata resolves a digest reference directly or a tag reference +// through its symlink, then reads the metadata it points at. +func resolveRefMetadata(p *paths.Paths, ref *NormalizedRef) (string, *imageMetadata, error) { + digestHex := ref.DigestHex() + if !ref.IsDigest() { + var err error + digestHex, err = resolveTag(p, ref.Repository(), ref.Tag()) + if err != nil { + return "", nil, err + } + } + meta, err := readMetadata(p, ref.Repository(), digestHex) + if err != nil { + return "", nil, err + } + return digestHex, meta, nil +} + func readContentMetadata(p *paths.Paths, digestHex string) (*imageMetadata, error) { return readMetadataAt(contentLayout(p, digestHex)) } @@ -322,6 +340,12 @@ func promoteLegacyTags(p *paths.Paths, repository, digestHex string) error { if err != nil || target != digestHex { continue } + // resolveTag validated that content-relative links resolve to this + // digest's content dir, so only legacy links (bare digest target) + // still need restaging. + if raw, err := os.Readlink(tagSymlinkPath(p, repository, tag)); err == nil && raw != digestHex { + continue + } ref, err := stageTagSymlink(p, repository, tag, digestHex) if err != nil { return fmt.Errorf("stage legacy tag %s: %w", tag, err) @@ -470,6 +494,10 @@ func createTagSymlink(p *paths.Paths, repository, tag, digestHex string) error { return nil } +// errInvalidSymlinkTarget marks a tag symlink that does not resolve to a +// digest or the shared content directory. +var errInvalidSymlinkTarget = errors.New("invalid symlink target") + func resolveTag(p *paths.Paths, repository, tag string) (string, error) { linkPath := tagSymlinkPath(p, repository, tag) @@ -484,16 +512,16 @@ func resolveTag(p *paths.Paths, repository, tag string) (string, error) { // Legacy links contain only the digest. New links point relatively into the // shared content directory; validate that they resolve to that digest only. if filepath.IsAbs(target) { - return "", fmt.Errorf("invalid symlink target: %s", target) + return "", fmt.Errorf("%w: %s", errInvalidSymlinkTarget, target) } digestHex := filepath.Base(target) if digestHex == "." || digestHex == string(filepath.Separator) { - return "", fmt.Errorf("invalid symlink target: %s", target) + return "", fmt.Errorf("%w: %s", errInvalidSymlinkTarget, target) } if target != digestHex { resolved := filepath.Clean(filepath.Join(filepath.Dir(linkPath), target)) if resolved != filepath.Clean(p.ImageContentDir(digestHex)) { - return "", fmt.Errorf("invalid symlink target: %s", target) + return "", fmt.Errorf("%w: %s", errInvalidSymlinkTarget, target) } } diff --git a/lib/images/tag.go b/lib/images/tag.go new file mode 100644 index 000000000..181176952 --- /dev/null +++ b/lib/images/tag.go @@ -0,0 +1,106 @@ +package images + +import ( + "context" + "errors" + "fmt" + "log/slog" +) + +// TagImage creates a ready-image tag without pulling or converting content. +// Cross-repository tags promote legacy content into the shared layout. A +// failed call leaves no side effects: the target tag's generation is only +// bumped after the new tag is on disk, so pending pulls that claimed the +// target tag keep their claim. When the target previously pointed at +// different content, that digest is collected after the new tag is live; +// cleanup failures are logged and do not fail the call, since the tag is +// already installed at that point. +// +// Promotion deliberately runs before the symlink install, so a symlink +// failure after a cross-repo promotion leaves the content promoted with no +// target tag. That state is gc-consistent (unreferenced content is +// collected) and retry is idempotent (promoteImageToContent short-circuits +// on ready content), which beats rolling back a completed promotion. +func (m *manager) TagImage(ctx context.Context, source, target string) (*Image, error) { + sourceRef, targetRef, err := parseTagReferences(source, target) + if err != nil { + return nil, err + } + + m.createMu.Lock() + defer m.createMu.Unlock() + + digestHex, meta, err := m.readyTagImage(sourceRef) + if err != nil { + return nil, err + } + + // A dangling or malformed target symlink is treated like a missing tag so + // the retag self-heals; createTagSymlink replaces the link either way. + previousDigest, err := resolveTag(m.paths, targetRef.Repository(), targetRef.Tag()) + if err != nil && !errors.Is(err, ErrNotFound) && !errors.Is(err, errInvalidSymlinkTarget) { + return nil, fmt.Errorf("resolve existing target tag: %w", err) + } + if sourceRef.Repository() != targetRef.Repository() { + if err := promoteImageToContent(m.paths, sourceRef.Repository(), digestHex, meta); err != nil { + return nil, fmt.Errorf("promote image to content: %w", err) + } + } + if err := createTagSymlink(m.paths, targetRef.Repository(), targetRef.Tag(), digestHex); err != nil { + return nil, fmt.Errorf("create image tag: %w", err) + } + // Unlike updateExistingReference, which bumps the generation before + // installing the symlink, the bump happens after the install here so a + // failed tag call cannot invalidate a pending pull's claim on the target + // tag (pinned by TestTagImageFailureLeavesNoSideEffects). + m.nextTagGeneration(targetRef.Repository(), targetRef.Tag()) + m.cleanupReplacedTag(targetRef, previousDigest, digestHex) + + return meta.toImageFor(targetRef.String()), nil +} + +func (m *manager) cleanupReplacedTag(ref *NormalizedRef, previousDigest, digestHex string) { + if previousDigest == "" || previousDigest == digestHex { + return + } + // Sibling tags in this repository may still reference the previous + // digest; only collect when this was the last reference. + count, err := countTagsForDigest(m.paths, ref.Repository(), previousDigest) + if err != nil { + slog.Warn("failed to count tags for replaced image", "repository", ref.Repository(), "digest", previousDigest, "error", err) + return + } + if count > 0 { + return + } + if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), previousDigest, true); err != nil { + slog.Warn("failed to collect replaced image content", "repository", ref.Repository(), "digest", previousDigest, "error", err) + } + m.refreshDiskUsageTotals() +} + +func parseTagReferences(source, target string) (*NormalizedRef, *NormalizedRef, error) { + sourceRef, err := ParseNormalizedRef(source) + if err != nil { + return nil, nil, fmt.Errorf("%w: invalid source reference: %s", ErrInvalidName, err) + } + targetRef, err := ParseNormalizedRef(target) + if err != nil { + return nil, nil, fmt.Errorf("%w: invalid target reference: %s", ErrInvalidName, err) + } + if targetRef.IsDigest() { + return nil, nil, fmt.Errorf("%w: target must be a tag reference, not a digest", ErrInvalidName) + } + return sourceRef, targetRef, nil +} + +func (m *manager) readyTagImage(ref *NormalizedRef) (string, *imageMetadata, error) { + digestHex, meta, err := resolveRefMetadata(m.paths, ref) + if err != nil { + return "", nil, err + } + if meta.Status != StatusReady { + return "", nil, fmt.Errorf("%w: %s", ErrImageNotReady, meta.Status) + } + return digestHex, meta, nil +} diff --git a/lib/images/tag_test.go b/lib/images/tag_test.go new file mode 100644 index 000000000..9c6462085 --- /dev/null +++ b/lib/images/tag_test.go @@ -0,0 +1,207 @@ +package images + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func seedContent(t *testing.T, p *paths.Paths, repository, tag, digestHex string) { + t.Helper() + require.NoError(t, SeedTestImage(p, TestSeed{Repository: repository, Tag: tag, DigestHex: digestHex, Content: true})) +} + +func seedLegacy(t *testing.T, p *paths.Paths, repository, tag, digestHex string) { + t.Helper() + require.NoError(t, SeedTestImage(p, TestSeed{Repository: repository, Tag: tag, DigestHex: digestHex})) +} + +func newTagTestCase(t *testing.T) (*paths.Paths, *manager, string) { + t.Helper() + p := paths.New(t.TempDir()) + return p, &manager{paths: p, tagGenerations: make(map[string]uint64)}, "docker.io/library/alpine" +} + +func requireTagResolvesTo(t *testing.T, p *paths.Paths, repository, tag, digest string) { + t.Helper() + resolved, err := resolveTag(p, repository, tag) + require.NoError(t, err) + require.Equal(t, digest, resolved) +} + +func tagImage(t *testing.T, m *manager, source, target, digest string) { + t.Helper() + img, err := m.TagImage(context.Background(), source, target) + require.NoError(t, err) + require.Equal(t, target, img.Name) + require.Equal(t, "sha256:"+digest, img.Digest) +} + +func TestTagImageAliasesReadyImage(t *testing.T) { + cases := []struct { + name, source, target, digest string + }{ + {"same repository", "docker.io/library/alpine:latest", "docker.io/library/alpine:stable", "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"}, + {"digest source", "docker.io/library/alpine@sha256:b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2", "docker.io/library/alpine:pinned", "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p, m, repository := newTagTestCase(t) + seedContent(t, p, repository, "latest", tc.digest) + tagImage(t, m, tc.source, tc.target, tc.digest) + targetRef, err := ParseNormalizedRef(tc.target) + require.NoError(t, err) + requireTagResolvesTo(t, p, targetRef.Repository(), targetRef.Tag(), tc.digest) + }) + } +} + +func TestTagImageSameRepositoryDeletesContent(t *testing.T) { + p, m, repository := newTagTestCase(t) + digest := "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" + seedContent(t, p, repository, "latest", digest) + + tagImage(t, m, repository+":latest", repository+":stable", digest) + requireTagResolvesTo(t, p, repository, "stable", digest) + + require.NoError(t, m.DeleteImage(context.Background(), repository+":latest")) + _, err := os.Stat(p.ImageContentDir(digest)) + require.NoError(t, err) + require.NoError(t, m.DeleteImage(context.Background(), repository+":stable")) + _, err = os.Stat(p.ImageContentDir(digest)) + require.True(t, os.IsNotExist(err), "content must be removed once unreferenced") +} + +func TestTagImageLegacyLayoutSource(t *testing.T) { + p, m, repository := newTagTestCase(t) + digest := "d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8d8" + seedLegacy(t, p, repository, "latest", digest) + + tagImage(t, m, repository+":latest", repository+":v2", digest) + requireTagResolvesTo(t, p, repository, "v2", digest) +} + +func TestTagImageCrossRepository(t *testing.T) { + p, m, sourceRepo := newTagTestCase(t) + targetRepo := "registry.example/apps/alpine" + digest := "c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3" + seedLegacy(t, p, sourceRepo, "latest", digest) + + tagImage(t, m, sourceRepo+":latest", targetRepo+":v1", digest) + data, err := os.ReadFile(p.ImageContentPath(digest)) + require.NoError(t, err) + require.Equal(t, "rootfs!", string(data)) + require.DirExists(t, p.ImageDigestDir(sourceRepo, digest)) + + for repo, tag := range map[string]string{sourceRepo: "latest", targetRepo: "v1"} { + requireTagResolvesTo(t, p, repo, tag, digest) + } + require.NoError(t, m.DeleteImage(context.Background(), sourceRepo+":latest")) + _, err = os.Stat(p.ImageContentDir(digest)) + require.NoError(t, err) + require.NoError(t, m.DeleteImage(context.Background(), targetRepo+":v1")) + _, err = os.Stat(p.ImageContentDir(digest)) + require.True(t, os.IsNotExist(err), "content must be removed once unreferenced") +} + +func TestTagImageRejectsNotReady(t *testing.T) { + p, m, repository := newTagTestCase(t) + digest := "d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4" + require.NoError(t, os.MkdirAll(p.ImageContentDir(digest), 0o755)) + require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digest), &imageMetadata{ + Name: repository + ":latest", Digest: "sha256:" + digest, + Status: StatusConverting, CreatedAt: time.Now().UTC(), + })) + require.NoError(t, createTagSymlink(p, repository, "latest", digest)) + + targetRepository := "registry.example/apps/alpine" + _, err := m.TagImage(context.Background(), repository+":latest", targetRepository+":stable") + require.ErrorIs(t, err, ErrImageNotReady) + _, err = resolveTag(p, targetRepository, "stable") + require.ErrorIs(t, err, ErrNotFound) +} + +func TestTagImageSourceNotFound(t *testing.T) { + _, m, repository := newTagTestCase(t) + _, err := m.TagImage(context.Background(), repository+":missing", repository+":stable") + require.ErrorIs(t, err, ErrNotFound) +} + +func TestTagImageFailureLeavesNoSideEffects(t *testing.T) { + p, m, repository := newTagTestCase(t) + digest := "d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5" + seedContent(t, p, repository, "latest", digest) + + // Failed calls must not create the target tag or bump its generation, + // which would invalidate a pending pull's claim on the target tag. + _, err := m.TagImage(context.Background(), repository+":missing", repository+":stable") + require.ErrorIs(t, err, ErrNotFound) + _, err = resolveTag(p, repository, "stable") + require.ErrorIs(t, err, ErrNotFound) + require.Equal(t, uint64(0), m.tagGenerations[repository+":stable"]) + + tagImage(t, m, repository+":latest", repository+":stable", digest) + requireTagResolvesTo(t, p, repository, "stable", digest) +} + +func TestTagImageRejectsDigestTarget(t *testing.T) { + p, m, repository := newTagTestCase(t) + digest := "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5" + seedContent(t, p, repository, "latest", digest) + _, err := m.TagImage(context.Background(), repository+":latest", repository+"@sha256:"+digest) + require.ErrorIs(t, err, ErrInvalidName) +} + +func TestTagImageReplacesExistingTagAndCollectsOldContent(t *testing.T) { + p, m, repository := newTagTestCase(t) + first := "f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6f6" + second := "a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7" + seedContent(t, p, repository, "latest", first) + seedContent(t, p, repository, "stable", second) + + readyBytes, err := m.TotalImageBytes(context.Background()) + require.NoError(t, err) + require.Equal(t, int64(14), readyBytes) + + tagImage(t, m, repository+":latest", repository+":stable", first) + requireTagResolvesTo(t, p, repository, "stable", first) + _, err = os.Stat(p.ImageContentDir(second)) + require.ErrorIs(t, err, os.ErrNotExist) + + readyBytes, err = m.TotalImageBytes(context.Background()) + require.NoError(t, err) + require.Equal(t, int64(7), readyBytes) +} + +func TestTagImageKeepsLegacySiblingTags(t *testing.T) { + p, m, repository := newTagTestCase(t) + first := "c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3" + second := "d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4" + seedLegacy(t, p, repository, "latest", first) + seedLegacy(t, p, repository, "stable", first) + seedLegacy(t, p, repository, "v2", second) + + tagImage(t, m, repository+":v2", repository+":stable", second) + requireTagResolvesTo(t, p, repository, "stable", second) + requireTagResolvesTo(t, p, repository, "latest", first) + _, err := os.Stat(p.ImageDigestDir(repository, first)) + require.NoError(t, err, "legacy content for a sibling tag must not be collected") +} + +func TestTagImageSelfHealsDanglingTargetSymlink(t *testing.T) { + p, m, repository := newTagTestCase(t) + digest := "e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6" + seedContent(t, p, repository, "latest", digest) + + linkPath := p.ImageRepositoryTagSymlink(repository, "stable") + require.NoError(t, os.MkdirAll(filepath.Dir(linkPath), 0o755)) + require.NoError(t, os.Symlink("already-collected", linkPath)) + + tagImage(t, m, repository+":latest", repository+":stable", digest) + requireTagResolvesTo(t, p, repository, "stable", digest) +} diff --git a/lib/images/testseed.go b/lib/images/testseed.go new file mode 100644 index 000000000..565a3a65d --- /dev/null +++ b/lib/images/testseed.go @@ -0,0 +1,111 @@ +package images + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/kernel/hypeman/lib/paths" +) + +const seedImageContent = "rootfs!" + +// TestSeed describes a ready image to write directly to disk, bypassing pulls, +// builds, and conversion. +type TestSeed struct { + Repository string + DigestHex string + // Tag optionally creates a tag symlink pointing at the image. + Tag string + // Name overrides the metadata "name" field; it defaults to + // repository:tag, or repository@sha256:digest when Tag is empty. + Name string + // Tags records resource tags in the metadata. + Tags map[string]string + // Content writes the image into the shared content layout instead of + // the legacy per-repository digest layout. + Content bool +} + +// seedImagePaths holds the on-disk locations for one seeded image, in either +// the shared content layout or the legacy per-repository digest layout. +type seedImagePaths struct { + dir string + disk string + metadata string +} + +func seedLayout(p *paths.Paths, s TestSeed) seedImagePaths { + if s.Content { + return seedImagePaths{ + dir: p.ImageContentDir(s.DigestHex), + disk: p.ImageContentPath(s.DigestHex), + metadata: p.ImageContentMetadata(s.DigestHex), + } + } + return seedImagePaths{ + dir: p.ImageDigestDir(s.Repository, s.DigestHex), + disk: p.ImageDigestPath(s.Repository, s.DigestHex), + metadata: p.ImageMetadata(s.Repository, s.DigestHex), + } +} + +// SeedTestImage writes a ready image to disk per s. It is exported for +// lib/images/testutil so test seeds marshal imageMetadata directly instead of +// re-declaring the schema; production code does not use it. +func SeedTestImage(p *paths.Paths, s TestSeed) error { + layout := seedLayout(p, s) + if err := os.MkdirAll(layout.dir, 0o755); err != nil { + return fmt.Errorf("create image dir: %w", err) + } + if err := os.WriteFile(layout.disk, []byte(seedImageContent), 0o644); err != nil { + return fmt.Errorf("write disk image: %w", err) + } + + name := s.Name + if name == "" && s.Tag != "" { + name = s.Repository + ":" + s.Tag + } else if name == "" { + name = s.Repository + "@sha256:" + s.DigestHex + } + data, err := json.Marshal(imageMetadata{ + Name: name, + Digest: "sha256:" + s.DigestHex, + Status: StatusReady, + SizeBytes: int64(len(seedImageContent)), + Tags: s.Tags, + CreatedAt: time.Now().UTC(), + }) + if err != nil { + return fmt.Errorf("marshal metadata: %w", err) + } + if err := os.WriteFile(layout.metadata, data, 0o644); err != nil { + return fmt.Errorf("write metadata: %w", err) + } + + return seedTagSymlink(p, s) +} + +// seedTagSymlink creates the tag symlink when s.Tag is set, pointing at the +// shared content directory or the bare digest per s.Content. +func seedTagSymlink(p *paths.Paths, s TestSeed) error { + if s.Tag == "" { + return nil + } + linkPath := p.ImageTagSymlink(s.Repository, s.Tag) + target := s.DigestHex + if s.Content { + linkPath = p.ImageRepositoryTagSymlink(s.Repository, s.Tag) + rel, err := filepath.Rel(filepath.Dir(linkPath), p.ImageContentDir(s.DigestHex)) + if err != nil { + return fmt.Errorf("rel content symlink target: %w", err) + } + target = rel + } + if err := os.MkdirAll(filepath.Dir(linkPath), 0o755); err != nil { + return fmt.Errorf("create tag dir: %w", err) + } + return os.Symlink(target, linkPath) +} diff --git a/lib/images/testutil/testutil.go b/lib/images/testutil/testutil.go new file mode 100644 index 000000000..e9e11a5fb --- /dev/null +++ b/lib/images/testutil/testutil.go @@ -0,0 +1,19 @@ +// Package testutil provides helpers for seeding on-disk image state in tests. +package testutil + +import ( + "testing" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +// Seed describes a ready image to seed; see images.TestSeed for the fields. +type Seed = images.TestSeed + +// SeedReadyImage writes a ready image to disk per s. +func SeedReadyImage(t testing.TB, p *paths.Paths, s Seed) { + t.Helper() + require.NoError(t, images.SeedTestImage(p, s)) +} diff --git a/lib/middleware/resolve.go b/lib/middleware/resolve.go index ab8507c62..80209f627 100644 --- a/lib/middleware/resolve.go +++ b/lib/middleware/resolve.go @@ -83,6 +83,13 @@ func ResolveResource(resolvers Resolvers, errResponder ErrorResponder) func(http resourceType = "ingress" paramName = "id" case strings.HasPrefix(path, "/images/"): + // POSTs under /images/{name} (currently only /tag) resolve the + // source in the handler, which returns a specific 404 body for + // a missing source; don't intercept with the generic one. + if r.Method == http.MethodPost { + next.ServeHTTP(w, r) + return + } resolver = resolvers.Image resourceType = "image" paramName = "name" diff --git a/lib/middleware/resolve_test.go b/lib/middleware/resolve_test.go index 5536c540f..810818465 100644 --- a/lib/middleware/resolve_test.go +++ b/lib/middleware/resolve_test.go @@ -127,3 +127,24 @@ func TestResolveResource_ResolvesBuilderByID(t *testing.T) { assert.Equal(t, "bld_123", resolver.receivedName, "Builder resolver was not invoked with the path ID") } + +func TestResolveResource_SkipsImagePosts(t *testing.T) { + resolver := &mockResolver{} + + middleware := ResolveResource(Resolvers{Image: resolver}, func(w http.ResponseWriter, err error, lookup string) { + w.WriteHeader(http.StatusNotFound) + }) + + r := chi.NewRouter() + r.With(middleware).Post("/images/{name}/tag", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + req := httptest.NewRequest(http.MethodPost, "/images/docker.io%2Flibrary%2Falpine:latest/tag", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + assert.Empty(t, resolver.receivedName, + "image POSTs must not be intercepted by the resolver") +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index eb43eb45c..bda3c0909 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1841,6 +1841,12 @@ type StandbyInstanceRequest struct { CompressionDelay *string `json:"compression_delay,omitempty"` } +// TagImageRequest defines model for TagImageRequest. +type TagImageRequest struct { + // Target Target OCI image reference with a tag. The local tag points to the source image without pulling it again. + Target string `json:"target"` +} + // Tags User-defined key-value tags. type Tags map[string]string @@ -2037,12 +2043,6 @@ type ListInstancesParams struct { Tags *Tags `json:"tags,omitempty"` } -// DeleteInstanceParams defines parameters for DeleteInstance. -type DeleteInstanceParams struct { - // GracefulShutdown Whether to attempt graceful guest shutdown before deleting the instance - GracefulShutdown *bool `form:"graceful_shutdown,omitempty" json:"graceful_shutdown,omitempty"` -} - // GetInstanceLogsParams defines parameters for GetInstanceLogs. type GetInstanceLogsParams struct { // Tail Number of lines to return from end @@ -2137,6 +2137,9 @@ type CreateDeviceJSONRequestBody = CreateDeviceRequest // CreateImageJSONRequestBody defines body for CreateImage for application/json ContentType. type CreateImageJSONRequestBody = CreateImageRequest +// TagImageJSONRequestBody defines body for TagImage for application/json ContentType. +type TagImageJSONRequestBody = TagImageRequest + // CreateIngressJSONRequestBody defines body for CreateIngress for application/json ContentType. type CreateIngressJSONRequestBody = CreateIngressRequest @@ -2321,6 +2324,11 @@ type ClientInterface interface { // GetImage request GetImage(ctx context.Context, name string, reqEditors ...RequestEditorFn) (*http.Response, error) + // TagImageWithBody request with any body + TagImageWithBody(ctx context.Context, name string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + TagImage(ctx context.Context, name string, body TagImageJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // ListIngresses request ListIngresses(ctx context.Context, params *ListIngressesParams, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -2344,7 +2352,7 @@ type ClientInterface interface { CreateInstance(ctx context.Context, body CreateInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) // DeleteInstance request - DeleteInstance(ctx context.Context, id string, params *DeleteInstanceParams, reqEditors ...RequestEditorFn) (*http.Response, error) + DeleteInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) // GetInstance request GetInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -2761,6 +2769,30 @@ func (c *Client) GetImage(ctx context.Context, name string, reqEditors ...Reques return c.Client.Do(req) } +func (c *Client) TagImageWithBody(ctx context.Context, name string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewTagImageRequestWithBody(c.Server, name, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) TagImage(ctx context.Context, name string, body TagImageJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewTagImageRequest(c.Server, name, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) ListIngresses(ctx context.Context, params *ListIngressesParams, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewListIngressesRequest(c.Server, params) if err != nil { @@ -2857,8 +2889,8 @@ func (c *Client) CreateInstance(ctx context.Context, body CreateInstanceJSONRequ return c.Client.Do(req) } -func (c *Client) DeleteInstance(ctx context.Context, id string, params *DeleteInstanceParams, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewDeleteInstanceRequest(c.Server, id, params) +func (c *Client) DeleteInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewDeleteInstanceRequest(c.Server, id) if err != nil { return nil, err } @@ -4209,6 +4241,53 @@ func NewGetImageRequest(server string, name string) (*http.Request, error) { return req, nil } +// NewTagImageRequest calls the generic TagImage builder with application/json body +func NewTagImageRequest(server string, name string, body TagImageJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewTagImageRequestWithBody(server, name, "application/json", bodyReader) +} + +// NewTagImageRequestWithBody generates requests for TagImage with any type of body +func NewTagImageRequestWithBody(server string, name string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "name", runtime.ParamLocationPath, name) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/images/%s/tag", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewListIngressesRequest generates requests for ListIngresses func NewListIngressesRequest(server string, params *ListIngressesParams) (*http.Request, error) { var err error @@ -4472,7 +4551,7 @@ func NewCreateInstanceRequestWithBody(server string, contentType string, body io } // NewDeleteInstanceRequest generates requests for DeleteInstance -func NewDeleteInstanceRequest(server string, id string, params *DeleteInstanceParams) (*http.Request, error) { +func NewDeleteInstanceRequest(server string, id string) (*http.Request, error) { var err error var pathParam0 string @@ -4497,28 +4576,6 @@ func NewDeleteInstanceRequest(server string, id string, params *DeleteInstancePa return nil, err } - if params != nil { - queryValues := queryURL.Query() - - if params.GracefulShutdown != nil { - - if queryFrag, err := runtime.StyleParamWithLocation("form", true, "graceful_shutdown", runtime.ParamLocationQuery, *params.GracefulShutdown); err != nil { - return nil, err - } else if parsed, err := url.ParseQuery(queryFrag); err != nil { - return nil, err - } else { - for k, v := range parsed { - for _, v2 := range v { - queryValues.Add(k, v2) - } - } - } - - } - - queryURL.RawQuery = queryValues.Encode() - } - req, err := http.NewRequest("DELETE", queryURL.String(), nil) if err != nil { return nil, err @@ -6194,6 +6251,11 @@ type ClientWithResponsesInterface interface { // GetImageWithResponse request GetImageWithResponse(ctx context.Context, name string, reqEditors ...RequestEditorFn) (*GetImageResponse, error) + // TagImageWithBodyWithResponse request with any body + TagImageWithBodyWithResponse(ctx context.Context, name string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*TagImageResponse, error) + + TagImageWithResponse(ctx context.Context, name string, body TagImageJSONRequestBody, reqEditors ...RequestEditorFn) (*TagImageResponse, error) + // ListIngressesWithResponse request ListIngressesWithResponse(ctx context.Context, params *ListIngressesParams, reqEditors ...RequestEditorFn) (*ListIngressesResponse, error) @@ -6217,7 +6279,7 @@ type ClientWithResponsesInterface interface { CreateInstanceWithResponse(ctx context.Context, body CreateInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateInstanceResponse, error) // DeleteInstanceWithResponse request - DeleteInstanceWithResponse(ctx context.Context, id string, params *DeleteInstanceParams, reqEditors ...RequestEditorFn) (*DeleteInstanceResponse, error) + DeleteInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*DeleteInstanceResponse, error) // GetInstanceWithResponse request GetInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetInstanceResponse, error) @@ -6861,6 +6923,32 @@ func (r GetImageResponse) StatusCode() int { return 0 } +type TagImageResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *Image + JSON400 *Error + JSON404 *Error + JSON409 *Error + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r TagImageResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r TagImageResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type ListIngressesResponse struct { Body []byte HTTPResponse *http.Response @@ -8093,6 +8181,23 @@ func (c *ClientWithResponses) GetImageWithResponse(ctx context.Context, name str return ParseGetImageResponse(rsp) } +// TagImageWithBodyWithResponse request with arbitrary body returning *TagImageResponse +func (c *ClientWithResponses) TagImageWithBodyWithResponse(ctx context.Context, name string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*TagImageResponse, error) { + rsp, err := c.TagImageWithBody(ctx, name, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseTagImageResponse(rsp) +} + +func (c *ClientWithResponses) TagImageWithResponse(ctx context.Context, name string, body TagImageJSONRequestBody, reqEditors ...RequestEditorFn) (*TagImageResponse, error) { + rsp, err := c.TagImage(ctx, name, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseTagImageResponse(rsp) +} + // ListIngressesWithResponse request returning *ListIngressesResponse func (c *ClientWithResponses) ListIngressesWithResponse(ctx context.Context, params *ListIngressesParams, reqEditors ...RequestEditorFn) (*ListIngressesResponse, error) { rsp, err := c.ListIngresses(ctx, params, reqEditors...) @@ -8164,8 +8269,8 @@ func (c *ClientWithResponses) CreateInstanceWithResponse(ctx context.Context, bo } // DeleteInstanceWithResponse request returning *DeleteInstanceResponse -func (c *ClientWithResponses) DeleteInstanceWithResponse(ctx context.Context, id string, params *DeleteInstanceParams, reqEditors ...RequestEditorFn) (*DeleteInstanceResponse, error) { - rsp, err := c.DeleteInstance(ctx, id, params, reqEditors...) +func (c *ClientWithResponses) DeleteInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*DeleteInstanceResponse, error) { + rsp, err := c.DeleteInstance(ctx, id, reqEditors...) if err != nil { return nil, err } @@ -9491,6 +9596,60 @@ func ParseGetImageResponse(rsp *http.Response) (*GetImageResponse, error) { return response, nil } +// ParseTagImageResponse parses an HTTP response from a TagImageWithResponse call +func ParseTagImageResponse(rsp *http.Response) (*TagImageResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &TagImageResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest Image + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 409: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON409 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseListIngressesResponse parses an HTTP response from a ListIngressesWithResponse call func ParseListIngressesResponse(rsp *http.Response) (*ListIngressesResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -11441,6 +11600,9 @@ type ServerInterface interface { // Get image details // (GET /images/{name}) GetImage(w http.ResponseWriter, r *http.Request, name string) + // Create or update a local image tag + // (POST /images/{name}/tag) + TagImage(w http.ResponseWriter, r *http.Request, name string) // List ingresses // (GET /ingresses) ListIngresses(w http.ResponseWriter, r *http.Request, params ListIngressesParams) @@ -11461,7 +11623,7 @@ type ServerInterface interface { CreateInstance(w http.ResponseWriter, r *http.Request) // Stop and delete instance // (DELETE /instances/{id}) - DeleteInstance(w http.ResponseWriter, r *http.Request, id string, params DeleteInstanceParams) + DeleteInstance(w http.ResponseWriter, r *http.Request, id string) // Get instance details // (GET /instances/{id}) GetInstance(w http.ResponseWriter, r *http.Request, id string) @@ -11696,6 +11858,12 @@ func (_ Unimplemented) GetImage(w http.ResponseWriter, r *http.Request, name str w.WriteHeader(http.StatusNotImplemented) } +// Create or update a local image tag +// (POST /images/{name}/tag) +func (_ Unimplemented) TagImage(w http.ResponseWriter, r *http.Request, name string) { + w.WriteHeader(http.StatusNotImplemented) +} + // List ingresses // (GET /ingresses) func (_ Unimplemented) ListIngresses(w http.ResponseWriter, r *http.Request, params ListIngressesParams) { @@ -11734,7 +11902,7 @@ func (_ Unimplemented) CreateInstance(w http.ResponseWriter, r *http.Request) { // Stop and delete instance // (DELETE /instances/{id}) -func (_ Unimplemented) DeleteInstance(w http.ResponseWriter, r *http.Request, id string, params DeleteInstanceParams) { +func (_ Unimplemented) DeleteInstance(w http.ResponseWriter, r *http.Request, id string) { w.WriteHeader(http.StatusNotImplemented) } @@ -12538,6 +12706,37 @@ func (siw *ServerInterfaceWrapper) GetImage(w http.ResponseWriter, r *http.Reque handler.ServeHTTP(w, r) } +// TagImage operation middleware +func (siw *ServerInterfaceWrapper) TagImage(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "name" ------------- + var name string + + err = runtime.BindStyledParameterWithOptions("simple", "name", chi.URLParam(r, "name"), &name, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "name", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.TagImage(w, r, name) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // ListIngresses operation middleware func (siw *ServerInterfaceWrapper) ListIngresses(w http.ResponseWriter, r *http.Request) { @@ -12734,19 +12933,8 @@ func (siw *ServerInterfaceWrapper) DeleteInstance(w http.ResponseWriter, r *http r = r.WithContext(ctx) - // Parameter object where we will unmarshal all parameters from the context - var params DeleteInstanceParams - - // ------------- Optional query parameter "graceful_shutdown" ------------- - - err = runtime.BindQueryParameter("form", true, false, "graceful_shutdown", r.URL.Query(), ¶ms.GracefulShutdown) - if err != nil { - siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "graceful_shutdown", Err: err}) - return - } - handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - siw.Handler.DeleteInstance(w, r, id, params) + siw.Handler.DeleteInstance(w, r, id) })) for _, middleware := range siw.HandlerMiddlewares { @@ -14105,6 +14293,9 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/images/{name}", wrapper.GetImage) }) + r.Group(func(r chi.Router) { + r.Post(options.BaseURL+"/images/{name}/tag", wrapper.TagImage) + }) r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/ingresses", wrapper.ListIngresses) }) @@ -15115,6 +15306,60 @@ func (response GetImage500JSONResponse) VisitGetImageResponse(w http.ResponseWri return json.NewEncoder(w).Encode(response) } +type TagImageRequestObject struct { + Name string `json:"name"` + Body *TagImageJSONRequestBody +} + +type TagImageResponseObject interface { + VisitTagImageResponse(w http.ResponseWriter) error +} + +type TagImage200JSONResponse Image + +func (response TagImage200JSONResponse) VisitTagImageResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type TagImage400JSONResponse Error + +func (response TagImage400JSONResponse) VisitTagImageResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + +type TagImage404JSONResponse Error + +func (response TagImage404JSONResponse) VisitTagImageResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + +type TagImage409JSONResponse Error + +func (response TagImage409JSONResponse) VisitTagImageResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(409) + + return json.NewEncoder(w).Encode(response) +} + +type TagImage500JSONResponse Error + +func (response TagImage500JSONResponse) VisitTagImageResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + type ListIngressesRequestObject struct { Params ListIngressesParams } @@ -15397,8 +15642,7 @@ func (response CreateInstance500JSONResponse) VisitCreateInstanceResponse(w http } type DeleteInstanceRequestObject struct { - Id string `json:"id"` - Params DeleteInstanceParams + Id string `json:"id"` } type DeleteInstanceResponseObject interface { @@ -17014,6 +17258,9 @@ type StrictServerInterface interface { // Get image details // (GET /images/{name}) GetImage(ctx context.Context, request GetImageRequestObject) (GetImageResponseObject, error) + // Create or update a local image tag + // (POST /images/{name}/tag) + TagImage(ctx context.Context, request TagImageRequestObject) (TagImageResponseObject, error) // List ingresses // (GET /ingresses) ListIngresses(ctx context.Context, request ListIngressesRequestObject) (ListIngressesResponseObject, error) @@ -17729,6 +17976,39 @@ func (sh *strictHandler) GetImage(w http.ResponseWriter, r *http.Request, name s } } +// TagImage operation middleware +func (sh *strictHandler) TagImage(w http.ResponseWriter, r *http.Request, name string) { + var request TagImageRequestObject + + request.Name = name + + var body TagImageJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.TagImage(ctx, request.(TagImageRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "TagImage") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(TagImageResponseObject); ok { + if err := validResponse.VisitTagImageResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // ListIngresses operation middleware func (sh *strictHandler) ListIngresses(w http.ResponseWriter, r *http.Request, params ListIngressesParams) { var request ListIngressesRequestObject @@ -17896,11 +18176,10 @@ func (sh *strictHandler) CreateInstance(w http.ResponseWriter, r *http.Request) } // DeleteInstance operation middleware -func (sh *strictHandler) DeleteInstance(w http.ResponseWriter, r *http.Request, id string, params DeleteInstanceParams) { +func (sh *strictHandler) DeleteInstance(w http.ResponseWriter, r *http.Request, id string) { var request DeleteInstanceRequestObject request.Id = id - request.Params = params handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { return sh.ssi.DeleteInstance(ctx, request.(DeleteInstanceRequestObject)) @@ -18891,386 +19170,387 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9+3LbOLY3+ioonf1V23skWb7l4q6u77jtdNq7444nTjLf3q0cGSIhCWMSYAOgbKUr", - "VeeveYCp84TzJKewAJAgBUq0E9vxdGqqph2RxHVhYV1/649OxNOMM8KU7Bz80ZHRjKQY/jxUCkez9zzJ", - "U/KG/J4TqfTPmeAZEYoSeCnlOVOjDKuZ/ldMZCRopihnnYPOGVYzdDUjgqA5tILkjOdJjMYEwXck7nQ7", - "5BqnWUI6B52tlKmtGCvc6XbUItM/SSUom3Y+dTuC4JizZGG6meA8UZ2DCU4k6da6PdVNIyyR/qQH3xTt", - "jTlPCGadT9Di7zkVJO4c/OZP40PxMh//nURKd36YK36uMIvHizOe0GixPNlXlOXX0BvCueIpVjRC0nyD", - "MvgIjbEkMeIM4UjROUGUjXnOYvT26AxFnDES6cbkkPGxJGJOYjQRPEVqRtCMSwXvKIGjS6TwOCH9Iet0", - "a/tBmH4Sr1+lv82ImhERGCyVyLaCJlwgNaMSUaafRqTvb5gSOVle2W6HxgkZKZoSnqvlhfqZX6GEsylM", - "y7WL0lwqNMNzgj4SwdHvOU7oZEHZtHmRxmTCBUE/LzKSYoayBEdEIqoQZYq72Zg1KmlsPw0RF50yLsgo", - "JlJRhnX7o4wLcyKqo38Nf+AEee/C0OB9pGZYOSpnXKFLQrLqRPEVvqwu4287O93ng8HgQ7dDFUnNscLX", - "NM3TzsGT/f3d/W4npcz8e7sYPWWKTInQw7e/YCHwwpuO5LmIyCiisVg1kyihhCl0dHL85pYT6GwP+vC/", - "rWedbmf7+U5/+8kz+Pf2k44/raWFr4780+qjd66wyuUyDzKnaWQJZeQRyfKsf83TMRGIT1CUC0GYShYI", - "jhSJWxBdZdqD0FZEnE3oNBfuCIaOXGU5Z1gizAzT6NX4RdlYq3MXaSYW8ys2EiTFlOk1XhrEG/cI6ROK", - "7CHSQ4o4U4IniWYKSpE0U9Kdoq5m4wzhLEtoBKyncqj20oHsdDssTxL9sDbCcrdJQqcUXmi1NFR6m+S+", - "RYojwhQRxQlvszQVttjUcbncwd0o+WKrLmc8iUc5UzRZ7vWd/tmuqd8TlWhGkhjxyaSL6ARhpFvRPxsa", - "ry77zmDnSW+w1xs8ebv99GDw/GCw/z+dbmfCRYpV56ATY0V6epfb7A0wbUlZFN4dVieRVF9IgkRmd4oL", - "q7KBYxLxlCDd9MqR733+yBuvm6NyU/WLyL5Y3qyBre7Xrox21J1gqQomBPtF1WKEA2N6S1MiFU4zzYf0", - "GLzFbOJCrsH6PriVX7nA25+1wIxcq5FdoeB8QvRBrjMS6RuRO25SCBi6PUveBcu6FxoXBEs9YC0m6cv0", - "t07OZJ7pq5vEoyzBSrerZSogg1FKpdSfFj/EVBo+0u04Ih8xrkYiZ8y8yIi64uLSf9O2MqJZp9uZYTma", - "T7O80111bVWJGrogCc4ktGd3XIyIEFx0jGi8GE24cJuk79xyCVc0tbRCsrhiAyvU6XYqC1CwczcXN+5i", - "V4ODg16AloTRKowaAJNZHrjf1vJwi6GtZuzmFjFCtNtmZD+WVQ4QUzxlXCoayVZsHoQHvb0pjwOs87ho", - "DtGYMEUnlAgrVxMk9P2QEuQaQboRRBnKZe0cFKL/iMy1rjaa741UlC0vSk2x8TfPk03KG9G7lYvtL07K", - "GiKtzj2oOM0xhTN5TObUXC1V2c1uzSgWdE5EgH0XAoBhheY9tKHPumYhjDOyWVkpNqcxxW3YQQxjGtEA", - "9ZwdnSDzGJ0co40Zua52svN0/KzT3CTDaYAWfs5TzHr6QOhhufbhXb/tV3tBFYWnaT6aCp5nyy2fvD49", - "fYfgIWIg4fotPtsJSapZREc4jgWRMjx/99Af22AwGBzgnYPBoD8IjXJOWMxF45Kax+El3R7EZEWTrZbU", - "tr+0pL++Pzk+OURHXGRcgM629uD4y+PPyyeb6q6E6P/HnCbxMtWP9c+kYZ2IkFQqza9+NK8hQYwqh65m", - "XBIU4WhG0NgoLaCeQ3ttSL7o2N1eoREcU5lx4PPo/WkpzYFqSK5JlKtKv9+jmGpdOLJClTe5FiOKuN4j", - "ff+GJAtYAWTf0aq2cqLTrYWASBC8pjv9RqvOls9/bshrlMqm1t0rms2nNEmoJBFnsfT7oEw92WuejHeK", - "zbW51NUL/TNKiZR4StAGmKVA3TAcXktbE0wTEm+2k7CbJvN3PvbutcqZAzLo4XG0vbMbZGgpnpJRTKfW", - "rlinQf27lo91OwrB2+GJgITRbh7QpSCT5f5+gvsEOhFkQgTRFP+Z3WWCzwnDVqX6D+i3839tlQbXLWtt", - "3YLFPCtf/9Tt/J6TnIwyLqkZ4RKbsE80GcFSI/giPGZ4tGqvPYqSCovV5wPe+AInsRQ2166NNf1oeQtP", - "137yVr9TZ+jAjgoBx+MCjXz7hZa0AiILZ8o+qJmA+RQllBk1SPNvsxcg7C0y8kPCp5udL7YOxfIvH349", - "7lswL/NDQ2v6WbfQChI+9VdzRrBQY1JZzIZ71TZUjq5x+c8qx6d2gWJJRqs5yBlljMRgc7cH27ypZeug", - "7gOn6JKq0VzfwKEzB8P6hSpk32hsKuHR5YQmZDTDcmaNlHFMjcH1rDKTgAhZcWZgMBK4BkG0AaX6/OfD", - "nf0nyHYQWENr/dUvLM/E+1o3b8ULhcUYJ0mQNprJ7eZ39DKFhCmgNPg23T0FBTrCNJyuY3fTKu+5nJm/", - "gHfrUcHdp9mAJq9E/71EuN3OdU+32ptjAauum/eG9VfXk/fbj2Wn3q9nRf/ej2/sULyffnKj8n47Kgfo", - "VsWoSUETOEw6LFVyylSPMtgCLYBby5O5Okozq7VkIL2NhYQHN/tnCVZHSyIV2njz09Hu7u7zzfbCFZWX", - "I0k/ktF0vFJutoO2grL+DOnP9FU5pVM8Xigi++gIM8YVGhMUzTCbkhjhidJf2bFWFPD9wTonTKOQREST", - "iKQ+7uHnz66vsXr+hF7J5x/TsZj+fTfo/wTrouY3zRY4bHdTv4oEZoFNvN1FneJrrfkbKgk5P0+NuwoV", - "L9k9kPUh9FGp0E/otV5zhbYrKx1c3LD2V7iwGGe9nNHfc9jtLMGLZR1QEZz2gCJCy2v4hjlAcv3xkfr8", - "mG/cTK8wVfrkFD5TO+Uu4kms750JFVJ12nvDbiQYEPGlRKTKKQtLTMsUUV/Axku9GGfjWUnohESLKCHL", - "zN2x70zk1twaE33nwJ+FAbEtG3dDqXBi9+NZ0UXl5+Oyv8rvL0znn7qdI5zhMU2o48xVPm198iNr+Fu3", - "S35jx+bTN/bLT93OhGCVCxJYy3OID0DGTtxLyJwkyL6tifcAHZ6dIJmLCbjMgVrNu4hcZ1wSOWSCTLGI", - "EyKB1K9mNJoV5soxji4lwqwMC0CHyRVeSJQJIglTB0M2LCzkctjpoqFRvuzfhkrM3yY0xL3EpoJI+w9y", - "TaJhp6vbSvhUDjt99DOXqqfVZStLHaChtcfIYQdtWJNahqVUM8Hz6WzIUszwlKSEqS6CIA0wA2vOlCw2", - "EWYxGnYEl0Qp3CNpngDv140dZllChuycJjTiDKU4en1uv72iaobemI+8yxNmrC/rLsqs6VnilAwZNmZQ", - "vZMLlAk+JijBOYtmRCLCJlxEZLOPzojouSV2e4uwIHozjBEX5UwfEYLLzZC/fUCEKWEjHdozFz2Tm9Cf", - "XvpCiZY3+fLEfPGp8I7c5ONf7SeaVdkZB3UUsSit6R49mxuxMPW766jiCegOGawo7CrVb10xVNmySYKn", - "QCreITLEKIcs4mkKip7tBUkyJwInxQ7pO46DPUHfDfZGGrIo4Xncmy0yIuZUctHVNwSJIBhBdNHvJM27", - "0Kn+q5fSSPB5ijgbskST8RZO4yd7m310iBIt98TF/KkhbktiwAnAHkD1aIcd7Gzyw46ZGJVIX/x99IIp", - "QQ3NIWkIbmxu0hpxtd08j1kt3W2wPzdp7Nx8seThMD9bgu4usViPbkoCLMjY46KhO2sFA16Wwd26rnZG", - "2fEVuxVhZneqQp0HiCqEs4xgIRFlQ1ZSE4uBSu1HmSCwHJIqu3UpUWgjxQrYi5phNWTAIb6TqLL7m30E", - "cqOJDys5Ehh6BUkWZkBkyNyYr2iSgOEQQWCD4XDmoIFlzIuggMOEWcH7ktLbZZ1f1i9amUDliJi+//ri", - "9B2SC6lIisaUYbHYRC4iTfNaQ5rLjrmwyGh3DzR3ENWqs445RD3pbxFnVQGyfl7XuhCsCFrSxToK+5mH", - "QiyxiGahMDqp0NHZO6QfU0UiTcSV8WKRPgm6krhsaE73ikGINetdaQ6Yzto5c32kYMTrJntS3CNhEalw", - "xy97u8CaU/pxSUIizatMPI3R3wiyzmXEU2Dptc20LLRpiVzjcl3vEm1wuaUnvOndLPpMi5z10WuGQIxA", - "FSliyGxQZZTkMZHI4+eoNN86+aISBlWQ/ZD96//9/4wsEaM5xYW0gd5ToXKc0I9Gi50InBLN9KpXWjQj", - "0eWQ1cUQpFuVHGF3qRTLTKV/oQg6nSnE+FV/yE7MmIoBb0g+UVdYkDzTaibq9eyoe1bO2jRBXNAQVbUQ", - "zd9qe+Mtzg1jCSvuvGJDu8sUto5Ufy3FliqtTrEiVzigEL/UpNebUwkha0AT9l10ctZH8BhcCdHMPNVc", - "VEuxaMNQBWXTIYMnViTW5AUSrSEz65rso9cpVQXxM15cL/amG7IZlmhMCAMvYjI3AcglqS5IfQM624P+", - "9mDQH/S3Q0djqsc+Unw0zYN2V3fXxVqYUej9aU/x3vtTpASeTGik6Sgjwo7aXDD1QQf5ecpjkjSstPvQ", - "hVAkfa1kCBpPtZCjl8uK/uY3czllROhxvT080xccrP73aNhhWJlvSk7fywSf05jE6NfDt/pqjV6fb9pV", - "s6qpaVhLGEEjeLcj8zEjqmn85inE4Ia3AkJp1zFfs0RLW7SOvD9PotEXv2nguxUyScnBtHSihdiCBPvo", - "VUV+tfKnC1zRIuiyRF+04A7x96gY7Q8g0KCUYCYhhFLL+r6QQeWQOemjInBoSRvZFdWfipwZLmXuw57H", - "fUEsMfIISGRFkP5c/39vLnl0aUM8jJZZcFqGUzjd/tItSVZUtRJy1poAPIUlsKSFROYLnlpHkAxncsYV", - "aNXuH1uCSMUF2dTKuY2X0s8znEuC/oJSknKxGDL3ftecswQrE8NQfDvRjKmDNqKEM+LbEUwAAR4yqXiW", - "kdg6Jr7XkhocblwEEXJR2KrNS1pLhi7klpEAJCpmARtQemCSRbnJ3kw2wdQAsymmpeecp2bYM66yJJ/2", - "7Dw7aCOhc2KnrSdIPxLTxhgnCeesZ+Mh9aul7QQeIYXFlChrepbQPtDMsDNkG3BukaEhrV7mjEZwm8N7", - "0yzv+SaODtp4efZuq4xZGjLvMXwSU3nZo7yX0JQqsJRQeYlOtl4joW9o+JmyqRk9vGymU0wS3rdT7KP3", - "OMntOQVq1+urFWEj/cNIe0rkajbJkwM0/2iFsOp+FDtZSEXbu11raUrAwtQoybw/RRLPiSNI+Eqr1FLp", - "Y4TjueZh0lpjL9GGJaee9XMB3UnECIkR40Pmt6UPMpCpHmJd5EfGECNRbZGGrBTbapHk7mpzLsPCFOEE", - "n/YWm9UaTYOP4fO1lhuoyOeFUl+7SDLa7FU9PDsxgcM4KpfJ018Jep0Rpt+KeZSnhGmZaMLdq9UoskF/", - "pykyraH7c/9aKbqXMy4UmlKtCGsxjjPLzYadXkyF0sdf5pMJvQYtMgcrkJFpjHGzqznUsJOzS8av2LBT", - "HebTJ3hnEo931+6GG3a3sobBfQCmZ63Sjfl6q31oRysdZlrU1JK75vae5m8p/Au4y8qUJH3piZ4+KwmF", - "yH1H210TrTslTKuq+tEENHYnoTW4fHpBUfbOvUo38sQ07KiJnG3c0DVzmCZ8DBeem0gZdNpHJ9W16yJs", - "LB9U1le5yJbwAkPRBulP+/p2ySLaGwwGgx7e6Q0GvUGd2pO9ngl5z7BSROgB/j+/4d7Hw97/DHrPP5R/", - "jvq9D3/5j6BC3jJa1fm17Tw33FnqIjdYP4S1PtB14a239qz5w28+vGBaaNzpSBA4BjhZO4azXM6OvNeb", - "af3oZDnqzaxUzKNLIvqUbyV0LLBYbLEpZdcHWpyTqrpuq99dZWAJJMcYwaiwO2CJrIXlt605FhQz9cGM", - "EQ1944CW3xDcGvreP4YRoV7PtWPZl4u8rzgA+uhXrhC2Hmmt0h3AS8v9TgVOUwzWzjlOaAyHw7rZJI2J", - "Mc4yeKbf0FKSICoXDO0NBu7JyBuUlhu4dFMZdqzhVEQzZA1CAoEpxGTIRCSD6Fse41qakm8nWTpp+pj9", - "Zav4a8P92f/wl83//R9flNSB0FbQuDFn3JCf1eK8gUttJPyKiAhLghKiZyu7KKZTqqQR3mIIjQKD2vco", - "MsEeJnRRa3zMmajhvSo5p4sezmjPWl6Mu/sVYVM16xw82Q0u7ob9o/fhP91PDQsr8iSkr73hOVhe4bGf", - "eO3G0Mr54lY3T8DpklJ2Yj7bXhYnP2+H3URW7bRR6hq3Wt8xRVbQmoEsZ+B/6naiNCREzIkQ+igavnZ0", - "eqxVmEtiGZrWGtEwHwx2I3gB/iT2Fy3EaVUGftusSjw2hjAg7PzWIdGMQxhkkvCbGCm7dabeFCW40kcW", - "Wu2S+y/HFIIT3bjFIdjdvojGgl8SAS4ZvcSUSHRJFsZcMK0YNAmbozkWzjX4FtISjNJqH4E3iIB6rJVY", - "SM6fcQjwMGzR6mwmDYrgpOSWKabM2D57MuKZlvGYfc1Y9C8Im1+gFGdGrRfEGM1TrIigWl0sDYspianm", - "0UNG4GCgDOszH0VcxJADzY0zvVyF7yS6MGriBTR/QZkm6wtzMGum0j86r9+9/fH1u1+PR6/PXvx6eDL6", - "5cV/65/NR52D3/7oGDCNIvDqR4IFEeg//oD5fjLBsxDT1znM1YwLq+06Fz3Eq+CM9nlGGKb9iKedbuc/", - "/X9++PTBhXuazL25PgaBgX0KipnW9hmIwXcJUBLZpBqXzqmXTLMoZ3bwzA01U76R/G50JEBFoXw0zmQw", - "N6Vutyjl0O3B4PTHLRs9su/+sdlH1nULw9c8iAsrHsuZJp8ClwP8aUnCI5uhMWlKwXddhRg8+FozTkMh", - "4jXmVL66zKN6vfLpDVjR1piyLam3oRfdbN2Bbm4dqPyCzangDNRzEJfGCZHVs/Lr6+MXoxe/vu8c6Isg", - "ziObiHX2+s3bzkFndzAYdEIESq4zKogMBj4ejiVPckUQvFTGmPbRaS4huNMyj0kOdlCQzyRKc5WDWkSu", - "oySXmk+BNKBUEso/ftbb2X67/exgMDgYDP6ndcCqJv01zPvl2TuToQ7nneBEzUbgmlv34c/w7hG8CqwC", - "bJOg2gdWqdhTZ63Up9d+gzZmVenKTA4BQQ47uy9/NAdq+yWcJUeINhW4aMU0XFu9lz+G1sWzRi1LeqWx", - "DTLcWOyofhmMxtjrc2nNRL4nvMcwQOyAeIFFjMYci/j7aiwNfPnXF6fvvpMIrBU4sX4iI3rDN0aW5ESC", - "ruxifEpjq+zWlgAsP6kxJWKGCHhK51SznF6aUu6+q7Ila0yp2FD7noMpYMfzwoU63Y6emf2Pm2Cn25l/", - "rOZUrzcH2tCYVrriGiUQJxllZIUW+JUI+1dcXCYcx2Al+pKyvhfvVoOhMQ+q210op47Al9CdxpjFVzRW", - "s1HMr5gecuCGtE9Q8XJxTV7rmeDkX//45/vT0iCy/XKc2Ttze2f/M+/M2i2pmw7m2hQTybPwNN5l4Um8", - "P/3XP/7pZvKwkzAi5a3Ec7v/L0wLdYAai9tlovEb/K6FHFbm5/IGX/oaDIPQlcvnRCR44V0ndkyd7QHw", - "9NqoBDXuRvudZ0Jefbno1py49rJujdsZhK+Pb7ajr9x2JAiw3lFWYOWtOiBvzNulWh8gugDN/Yhl4Xtt", - "Q2kFoW3vnNo/d5ZJLkxx8pJmJphihKdF7ucqeL3zS5pZpRm+cK58w+jjHNTsMeeqP2QGPkefTaAcck0i", - "5yBV6PDsRJroAH2lwaWxLAhpFdzDXYLXjS9U5KyLxrnSejVXBFkLh83j12OBl8cE5WWkZk3LtRNcjj6A", - "ZbkkgpFkZLRY2XJlzEfIftS4ODDVCZYW7kuoPKuu1/Evp+do43jBcEoj9Itp9ZTHeULQuRHVNqur1x2y", - "TACGiu5EkyO1/fIJ4rnq8UlPCULcEFNorHB2WCSJ+cuzdxaLRG72h+wN0QtLWGyjSp1EUYSVsu+U8TdX", - "mvX7ry16E96M86G3PFzn9vXydLU3+3U7SiUhBLnEyNYJnRCTrCB4WlKfy5/ratXrJS8BD8yB7KN3Uh/I", - "gRx2NIPimrWOuZr5KtyEkiSWvppR4lSWr2klL6zHlSpjTSHZmwWdw1GWV0l3p9sIHDg3UQlaZqgouEGU", - "FZvdsryKBlfVN6jY+7sEWlRVJIq2NmDTMqCgrg2LtKZcI/G3NuV6xsUlo66zfP3RMltgdfsnzA1kpSm7", - "NH59Rl/nppGl/ALzc9fN7BardFKsSc0A/mWW51B6xsJWgKVGgjHXvEQbFzijfUvH/YinF1108Z+VHzSP", - "c4q/lpOvkFkN4JtM/+S3L5dCetcYMG8d1ovl7ffjUDbCTaH5NlICM2kCzGc4I330M1xWSJFUi2CAz+fH", - "1DF+9T3iRjh3nw6ZHpqsBhk6M7akU0bZdBOixccE4Tg2tm5rqnKBJzKE8+sMUEsSsBmdResBVF0bZo4u", - "nM35osoTly3SywYba6Je0tTNkoCGDjYMtZXmSnevJ1wI1DxXBr3LTr2KrFaze6+LiLFjKWxwt9j/84Jd", - "1MGT5wFVHXJBJqWjwvOYNDkmrEAWdppckoVJELAOErzkIvF9I2EPho3ltuKF710ZAzw0t/GdhWPFuEis", - "VwSibmtB3yF/wbqt0OvVevmrKm8wAaXnJltSjNViHTCn40J6cqY/sLZJAosPKvSByWe66Bqdn4CGhZgm", - "lsQFpdebh3hciN67sL/0bWsXRXbclwDbLtMe/K1F3s468daEMFvxSJG4W5UNLgnJ5PpJmbjMWYO/T5Ar", - "QR0jc8iOLcVQmy6SWmXo8wwgL7zGVkRCtW1iOT3UrK83Zofpi22AGey92Q9w/FSgfuuI93FNOzWQKdUu", - "L3CSXKAN+9ImEuTvAIdq94pxVhL726MzRwJFnNX7066mSM0FLmZKZSP9f3KkT/FFvTH7rTvhJRr5swHo", - "kXt7u9VkBTPgWrNVO3EQBqZ5a85yObujwKUGa7SDfTcWaXcDZbmcoQ2Fp/roWLwffU9sfq6lmjJJolyQ", - "9Wf9EMQjizkDY0owZb2f3749Q4JMqYQc22A0v4kJbwAnswp8Gaflpqt41Vy5s7u3/+Tps+f9+FL0SST6", - "uewRLFVvu49T/JEzfCW12LWVLnCWHcy31/J5swXF+JpZvtM2m0mBp5olu0jcNprrUflJ6Sm7pCxu28Av", - "+t310Z5Osb5rf0MmSC/PpgID3OWX9DbcOqAHVrN5V9eUAFkdyZtLxVMf+3ZjdQBvdbHmPOnFWOGbxO+a", - "4S6H6qYL05RRyYNmvqa46PN6MHQFsCZYCOBzo6vcWELb0gSmawwJJB4pHshbdbLDyTFgb9t3W6GGAvav", - "4qP5hPJ12e0QM1LJWTVSiTVv6CZ6WUSt9cwlfICwahYBdIf3p35QSX/IeiCFHaDjooOi2aJJDCoGjo17", - "e4OLSuIsOJHHi02E0fvTPnpbjPY7iay32KILFymTOXjSQCjqGZHMH0AOGSZU1T+3pkIDRLwJsTPcPusX", - "5UrAKFkYtcCGNaa1+ZiEZtgoe5f4mCstjbSrYLnemPtoFTRXJRRivzfY7m3vv90e3DQU4g6wlkNtHVZ5", - "iw1697nP0buT4x1rity8NRrXF0djDrOy4zJaH23kkniZqFwEY/S9UPiGGPxbh9bfUaR8iTu56l2zEm/1", - "m3cBMx0CwrJIlTcHgq4zzLVoo97klg06iwzMD+Up8QR5m1UR0WDK8TGVlz8Kgi8BPn/53gYAhpG5z8KB", - "doCSMV4AKhQYuQTnaiJRAZ5SGr+3957uPdt9svdsMAgAGS8TPI/oCPJmWg3g9dEJSvCiwPTbAP99jMYJ", - "H1cJfX/3ybOng+fbO23HYTzW7dah0L/dV2jDrshfXIkz96QyqJ2dp092d3cHT57s7LUalXUbtBqUczFU", - "RJKnu0/3tp/t7LVahZBe98IBS9cF+FCVg0NTGAjyPmVGIjqhEQJANqQ/gJT5GWWkcM5Wz+QYx64WRPju", - "UJgmcmVYm+nMvmnsrWmeKJolxDyDDWnlkoCZH0NLweJajBW1N27WkoXjXhvw5OZSvALiTEzG+XRqse6K", - "pTu1aeyl8ERJEh+gQmVbyedgN8uBfWiiAzuHltTwSqtOFnXOIwJzdRmIBkFQQSdm0yqzcrEOlGV5kCQa", - "l/KnXBj0CWgU4THPlfGq22IpZScGyIWBlzlncTtzx09cXK5NbNA3cVETpqXBwGW4mwz4Wm67L/QV3m8T", - "I2CfS/TGfGEMheXPWV4tiNeFnqxBkbm0/LiwG9tm2kqXYbkFbOYuXMb0V/LOewrl601M9NOX1bDFlEAt", - "JLVWYtGUYuKRzuH11nlS+sO1hpQW687I1X0sOmT69TTZ9iTD2d2s+KqI4cLW4OXpcxtn30dwuiDIzyXo", - "107auYOasEveHzKbaOThFYAjDbAETJj4jFCBuKBTWu24ame9x5jdm1CmI65bU6f/YVMMnizcgCEeYJCW", - "JyYE1hQA8YFe7Z50up3zomiUZUzVpXlTFN5aWpEysn4Zc+fs3U0jbzPBJzQEhAORQPapVdRcTOqrvcF5", - "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe9TRrsUKK7G1Wh91VahcadB3zNF", - "J6Uo/jwk2gKKxzifTIgYpQFb20/6OTIvFCgEpz9WxVstRrdVos8qmwNa9ARHlE03W69+wD5Xm8Y6mL+X", - "Z+/e2Co+TQDIequKSj8G+7iPfi3qXKGXZ+9kCXLUDxju6kB6DXldZ7OFpBFOTIumSAVlvr0NiLO1wHxW", - "fmgtkwGxOVwezR0EtDGfZjkcw/M3vZPX77fSmMy7lTFBXN2MJ0SPe9PjFnMHLlAmoVWYxLzJ8GEIQ7Y9", - "QN5aFSe49SJ55zWwOoornIxkwkMhPG/1QwQP0cb7n0x2sB5BF2WVrdS/+8g/Pn0/CZ4YAJRv6PYcOqxb", - "UCsHPKhK1hHJOtXpVToNHRWT2bQs8izXJuSX1Y3ml+vr4ZlGmvs9cslXNRu3lSuRydEy8IkuKgGZT40R", - "21pKJMmwwIoki1p4ZBXfnCw7mMk1iW6Q/fVCv/7J1LTIBRmpmSByxpNqWMJud7kuqoQQ4DmxpaDMnDw7", - "vOIoxeISLkYnV6OcmRWoRpDvroN7mSmV3WBSP799e2aUbUXEHCf1HAS55HA/JgleoDFRV4QwNxUsEfZD", - "UOs5nLKhHI9Qo4wIyqtr2NkN9HtuwpLRVOCIIPOVq6pst0TCrdl2KW0vAQDBKCJSNuzv9qr9tZ9O8qTd", - "HoeGtb22CHl0kw1+e3TmysoUZXvdMu8sr/IZET1z5Fz93tVbuyNXFzhyXTGDC7skMIwJVDyyiR9+XqYL", - "N4KKTvrzSi6kxxykn71n+4FTYJaqa875h1bCXv24h/zqKWZxqPyxifc3mfJTgM+CoGORgyuJxiYWySTv", - "G7HcT1sQBMeUESlracZRLpJOt9Ob2FkdbG0lPMIJIAju7W4/21od1bkynNdGL41iukrdczFOJgrGJbUa", - "+DWYdJUktnCWtTCImXVccz8Ae1oOH4SSx/pu8yQ8F08+GCxlTF/jSLkibWAhq3hAsX9sAee3Mh9oMDV1", - "V7So/dw/n4OgSzrDalYl/60l2ocwFQgx1DRirA+1dTRE/jEoUXERqsTFhbIJeGPiYhCL+9BF+DlE24qv", - "bPDMn+WT/f3d/XV8CJhN7ZjbcxeYqnm7mmJMvOW259c2ANFSVZnDHenVxUT1uqyhKc0Rl0hq9YLyjLAb", - "ref+3u7Ozdaz7UROXJRWjS+FEFqOTo+NTBRxpjBlRKCUKBxjhatMBkxLmstAqRdMUsjSmXy/mrU0hDP4", - "kCu3rVP1pZzhDSXr3jiE5hQzOtEM2b7p9yxneGf/yYEprBmTyd7+k36/f1Mgihcl8kSrrdgyMXMeJkVf", - "zj5vH+4Ab6LNXP7onB2+/VkzslwKc2ltyTFlB96/i3+WD+AP888xZWGcija1WOlkqQZrNTwst0DAJD5A", - "ZbltJ/e0qtcftg1DsDCA4wRR3ypBk3cH71bQOK2i+N8gVXdF6qoWV16zZNG4Nrcuq1oWHFdeOVU/+atF", - "aVX6cbW725m74B3bp8GDLqrOLju6b1U3WK4srbhUeSsjrCimmCTmr4gzwNkNVVasXJHuWYvCXHCN2Apc", - "RZf+j0Xv3o9H/kC8311hL+8nW2Lxww0jVFYKpH9blkPXcyEnjq45zGHbY3ErtC1na2HhgqHZD3wX3iaK", - "q9r76+l//f5/5NnTv2///ur9+/+ev/yv41/pf79Pzl5/FuTIakTAB4X1+2JIfqZEuw/n15aUTrGKAjY6", - "rf41rLB9YiwOKppBAU40JgdD1kOvqCLClHOr5SIOO2iDgKYEX2lxF2rVmDSwTf3xmXEw6o//cGLwp3ob", - "sc0RF3ZDCugPmY9jnmLKNodsyGxbyE1Egl6g/4pRhDNTk40ypPXfBRoLKKBnPT5l5130B86yT5tDZovm", - "G3DrDEMNskmRhMWcv9aOykSl2tdJgQJhnC5DVtzWBSSecfv1S/x8SpJ6Ck/DoqzW36zm9GwQAg+E9BK9", - "kVAjBlSQgrI1GRV5L+jZYHNZn1ujYxQ0tIL8rF/c5B8e5iFzcVPO4jGJaQR8xaXtzWxiZ5ExaSjNGvEy", - "wa8XsDdvTC5ZjHCuZpoXRTbPPeL8kpIubGkXTQQUeOiaL417fcaz3njRm/GswDzAwgSfYOOgrirZ/6dn", - "J9p7TwSd2J6CqeuaRAJCJxwZOzOTAVhYF5Ym9taU12Ba9JkT+7qp0yJNaQ4TWa1ywVz1CAI1JAFco6A+", - "EpLJv0dRQsHqJGc8T2I0A/w7pZsJQdh1BkUGCx5HMZnU/12NMNjZfwIarPv37k7rBFKzdKuoLE8COm3q", + "H4sIAAAAAAAC/+y9+3LjNrY3+ioonf1V7D2SLN/64lTqO47d6XgnTnva7p5v7yhHhkhIwpgEGACUrU51", + "1flrHmDqPOE8ySksACRIgRLtbtvtSddUTdwiievCwrr+1h+diKcZZ4Qp2Tn4oyOjGUkx/HmoFI5m73mS", + "p+Qt+T0nUumfM8EzIhQl8FLKc6ZGGVYz/a+YyEjQTFHOOgedM6xm6HpGBEFzaAXJGc+TGI0Jgu9I3Ol2", + "yA1Os4R0DjpbKVNbMVa40+2oRaZ/kkpQNu187HYEwTFnycJ0M8F5ojoHE5xI0q11e6qbRlgi/UkPvina", + "G3OeEMw6H6HF33MqSNw5+NWfxm/Fy3z8dxIp3flhrvi5wiweL854QqPF8mR/piy/gd4QzhVPsaIRkuYb", + "lMFHaIwliRFnCEeKzgmibMxzFqOLozMUccZIpBuTQ8bHkog5idFE8BSpGUEzLhW8owSOrpDC44T0h6zT", + "re0HYfpJvH6V/jYjakZEYLBUItsKmnCB1IxKRJl+GpG+v2FK5GR5ZbsdGidkpGhKeK6WF+pHfo0SzqYw", + "LdcuSnOp0AzPCfpABEe/5zihkwVl0+ZFGpMJFwT9uMhIihnKEhwRiahClCnuZmPWqKSx/TREXHTKuCCj", + "mEhFGdbtjzIuzImojv4N/IET5L0LQ4P3kZph5aiccYWuCMmqE8XX+Kq6jL/u7HRfDgaD37odqkhqjhW+", + "oWmedg6e7e/v7nc7KWXm39vF6ClTZEqEHr79BQuBF950JM9FREYRjcWqmUQJJUyho5Pjt3ecQGd70If/", + "bb3odDvbL3f6289ewL+3n3X8aS0tfHXkH1cfvXOFVS6XeZA5TSNLKCOPSJZn/UuejolAfIKiXAjCVLJA", + "cKRI3ILoKtMehLYi4mxCp7lwRzB05CrLOcMSYWaYRq/GL8rGWp27SDOxmF+zkSAppkyv8dIg3rpHSJ9Q", + "ZA+RHlLEmRI8STRTUIqkmZLuFHU1G2cIZ1lCI2A9lUO1lw5kp9theZLoh7URlrtNEjql8EKrpaHS2yT3", + "LVIcEaaIKE54m6WpsMWmjsvlDu5GyRdbdTnjSTzKmaLJcq/v9M92Tf2eqEQzksSITyZdRCcII92K/tnQ", + "eHXZdwY7z3qDvd7g2cX284PBy4PB/v90up0JFylWnYNOjBXp6V1uszfAtCVlUXh3WJ1EUn0hCRKZ3Sku", + "rMoGjknEU4J00ytHvvfpI2+8bo7KTdUvIvtiebMGtrpfuzLaUXeCpSqYEOwXVYsRDozpgqZEKpxmmg/p", + "MXiL2cSFXIP1fXArv3KBtz9pgRm5USO7QsH5hOiD3GQk0jcid9ykEDB0e5a8C5b1IDQuCJZ6wFpM0pfp", + "r52cyTzTVzeJR1mClW5Xy1RABqOUSqk/LX6IqTR8pNtxRD5iXI1Ezph5kRF1zcWV/6ZtZUSzTrczw3I0", + "n2Z5p7vq2qoSNXRBEpxJaM/uuBgRIbjoGNF4MZpw4TZJ37nlEq5oammFZHHFBlao0+1UFqBg524ubtzF", + "rgYHB70ALQmjVRg1ACazPHC/reXhFkNbzdjNLWKEaLfNyH4sqxwgpnjKuFQ0kq3YPAgPentTHgdY53HR", + "HKIxYYpOKBFWriZI6PshJcg1gnQjiDKUy9o5KET/EZlrXW003xupKFtelJpi42+eJ5uUN6J3KxfbX5yU", + "NURanXtQcZpjCmfymMypuVqqspvdmlEs6JyIAPsuBADDCs17aEOfdc1CGGdks7JSbE5jituwgxjGNKIB", + "6jk7OkHmMTo5RhszclPtZOf5+EWnuUmG0wAt/JinmPX0gdDDcu3Du37bP+8FVRSepvloKnieLbd88ub0", + "9B2Ch4iBhOu3+GInJKlmER3hOBZEyvD83UN/bIPBYHCAdw4Gg/4gNMo5YTEXjUtqHoeXdHsQkxVNtlpS", + "2/7Skv7y/uT45BAdcZFxATrb2oPjL48/L59sqrsSov/vc5rEy1Q/1j+ThnUiQlKpNL/63ryGBDGqHLqe", + "cUlQhKMZQWOjtIB6Du21IfmiY3d7hUZwTGXGgc+j96elNAeqIbkhUa4q/X6LYqp14cgKVd7kWowo4nqP", + "9P0bkixgBZB9R6vayolOdxYCIkHwmu70G606Wz7/uSGvUSqbWnevaDaf0iShkkScxdLvgzL1bK95Mt4p", + "NtfmUlev9M8oJVLiKUEbYJYCdcNweC1tTTBNSLzZTsJumszf+di71ypnDsigh8fR9s5ukKGleEpGMZ1a", + "u2KdBvXvWj7W7SgEb4cnAhJGu3lAl4JMlvv7Ae4T6ESQCRFEU/wndpcJPicMW5XqP6Dfzv+1VRpct6y1", + "dQsW86x8/WO383tOcjLKuKRmhEtswj7RZARLjeCL8Jjh0aq99ihKKixWnw944zOcxFLYXLs21vSj5S08", + "XfvJhX6nztCBHRUCjscFGvn2Ky1pBUQWzpR9UDMB8ylKKDNqkObfZi9A2Ftk5LuETzc7n20diuVfPvx6", + "3HdgXuaHhtb0s26hFSR86q/mjGChxqSymA33qm2oHF3j8p9Vjk/tAsWSjFZzkDPKGInB5m4PtnlTy9ZB", + "3QdO0RVVo7m+gUNnDob1E1XIvtHYVMKjqwlNyGiG5cwaKeOYGoPrWWUmARGy4szAYCRwDYJoA0r1+Y+H", + "O/vPkO0gsIbW+qtfWJ6J97Vu3ooXCosxTpIgbTST2+3v6GUKCVNAafBtunsKCnSEaThdx+6mVd5zOTN/", + "Ae/Wo4K7T7MBTV6J/nuJcLudm55utTfHAlZdN+8N66+uJ++378tOvV/Piv69H9/aoXg//eBG5f12VA7Q", + "rYpRk4ImcJh0WKrklKkeZbAFWgC3lidzdZRmVmvJQHobCwkPbvZPEqyOlkQqtPH2h6Pd3d2Xm+2FKyqv", + "RpJ+IKPpeKXcbAdtBWX9GdKf6atySqd4vFBE9tERZowrNCYommE2JTHCE6W/smOtKOD7g3VOmEYhiYgm", + "EUl92MMvX9zcYPXyGb2WLz+kYzH9+27Q/wnWRc1vmi1w2O6mfhUJzAKbeLeLOsU3WvM3VBJyfp4adxUq", + "XrJ7IOtD6KNSoZ/QG73mCm1XVjq4uGHtr3BhMc56OaO/57DbWYIXyzqgIjjtAUWEltfwDXOA5PrjI/X5", + "Md+4mV5jqvTJKXymdspdxJNY3zsTKqTqtPeG3UowIOJziUiVUxaWmJYpor6AjZd6Mc7Gs5LQCYkWUUKW", + "mbtj35nIrbk1JvrOgT8LA2JbNu6GUuHE7sezoovKz8dlf5XfX5nOP3Y7RzjDY5pQx5mrfNr65EfW8Ldu", + "l/zGjs2nb+2XH7udCcEqFySwlucQH4CMnbiXkDlJkH1bE+8BOjw7QTIXE3CZA7WadxG5ybgkcsgEmWIR", + "J0QCqV/PaDQrzJVjHF1JhFkZFoAOk2u8kCgTRBKmDoZsWFjI5bDTRUOjfNm/DZWYv01oiHuJTQWR9h/k", + "hkTDTle3lfCpHHb66EcuVU+ry1aWOkBDa4+Rww7asCa1DEupZoLn09mQpZjhKUkJU10EQRpgBtacKVls", + "IsxiNOwILolSuEfSPAHerxs7zLKEDNk5TWjEGUpx9ObcfntN1Qy9NR95lyfMWF/WXZRZ07PEKRkybMyg", + "eicXKBN8TFCCcxbNiESETbiIyGYfnRHRc0vs9hZhQfRmGCMuypk+IgSXmyF//Q0RpoSNdGjPXPRMbkN/", + "eukLJVre5ssT88XHwjtym49/sZ9oVmVnHNRRxKK0pnv0bG7EwtTvrqOKJ6A7ZLCisKtUv3XNUGXLJgme", + "Aql4h8gQoxyyiKcpKHq2FyTJnAicFDuk7zgO9gR9N9gbaciihOdxb7bIiJhTyUVX3xAkgmAE0UW/kzTv", + "Qqf6r15KI8HnKeJsyBJNxls4jZ/tbfbRIUq03BMX86eGuC2JAScAewDVox12sLPJDztmYlQiffH30Sum", + "BDU0h6QhuLG5SWvE1XbzPGa1dLfB/tymsXPzxZKHw/xsCbq7xGI9uikJsCBjj4uG7qwVDHhZBnfrutoZ", + "ZcdX7FaEmd2pCnUeIKoQzjKChUSUDVlJTSwGKrUfZYLAckiq7NalRKGNFCtgL2qG1ZABh/hGosrub/YR", + "yI0mPqzkSGDoFSRZmAGRIXNjvqZJAoZDBIENhsOZgwaWMS+CAg4TZgXvS0pvl3V+Wb9oZQKVI2L6/uur", + "03dILqQiKRpThsViE7mINM1rDWkuO+bCIqPdPdDcQVSrzjrmEPWkv0WcVQXI+nld60KwImhJF+so7Ece", + "CrHEIpqFwuikQkdn75B+TBWJNBFXxotF+izoSuKyoTndKwYh1qx3pTlgOmvnzPWRghGvm+xJcY+ERaTC", + "Hb/s7QJrTunHJQmJNK8y8TRGfyPIOpcRT4Gl1zbTstCmJXKNy3W9S7TB5Zae8KZ3s+gzLXLWR28YAjEC", + "VaSIIbNBlVGSx0Qij5+j0nzr5ItKGFRB9kP2r//3/zOyRIzmFBfSBnpPhcpxQj8YLXYicEo006teadGM", + "RFdDVhdDkG5VcoTdpVIsM5X+hSLodKYQ49f9ITsxYyoGvCH5RF1jQfJMq5mo17Oj7lk5a9MEcUFDVNVC", + "NH+t7Y23OLeMJay484oN7S5T2DpS/aUUW6q0OsWKXOOAQvxak15vTiWErAFN2HfRyVkfwWNwJUQz81Rz", + "US3Fog1DFZRNhwyeWJFYkxdItIbMrGuyj96kVBXEz3hxvdibbshmWKIxIQy8iMncBCCXpLog9Q3obA/6", + "24NBf9DfDh2NqR77SPHRNA/aXd1dF2thRqH3pz3Fe+9PkRJ4MqGRpqOMCDtqc8HUBx3k5ymPSdKw0u5D", + "F0KR9LWSIWg81UKOXi4r+pvfzOWUEaHHdXF4pi84WP1v0bDDsDLflJy+lwk+pzGJ0S+HF/pqjd6cb9pV", + "s6qpaVhLGEEjeLcj8zEjqmn85inE4Ia3AkJp1zFfs0RLW7SOvD9NotEXv2ngmxUyScnBtHSihdiCBPvo", + "54r8auVPF7iiRdBlib5owR3ib1Ex2u9AoEEpwUxCCKWW9X0hg8ohc9JHReDQkjayK6o/FTkzXMrchz2P", + "+4JYYuQRkMiKIP25/v/eXPLoyoZ4GC2z4LQMp3C6/aVbkqyoaiXkrDUBeApLYEkLicwXPLWOIBnO5Iwr", + "0KrdP7YEkYoLsqmVcxsvpZ9nOJcE/QWlJOViMWTu/a45ZwlWJoah+HaiGVMHbUQJZ8S3I5gAAjxkUvEs", + "I7F1THyrJTU43LgIIuSisFWbl7SWDF3ILSMBSFTMAjag9MAki3KTvZlsgqkBZlNMS885T82wZ1xlST7t", + "2Xl20EZC58ROW0+QfiCmjTFOEs5Zz8ZD6ldL2wk8QgqLKVHW9CyhfaCZYWfINuDcIkNDWr3MGY3gNof3", + "plne800cHbTx+uzdVhmzNGTeY/gkpvKqR3kvoSlVYCmh8gqdbL1BQt/Q8DNlUzN6eNlMp5gkvG+n2Efv", + "cZLbcwrUrtdXK8JG+oeR9pTI1WySJwdo/sEKYdX9KHaykIq2d7vW0pSAhalRknl/iiSeE0eQ8JVWqaXS", + "xwjHc83DpLXGXqENS0496+cCupOIERIjxofMb0sfZCBTPcS6yI+MIUai2iINWSm21SLJ3dXmXIaFKcIJ", + "Pu0tNqs1mgYfw6drLbdQkc8Lpb52kWS02at6eHZiAodxVC6Tp78S9CYjTL8V8yhPCdMy0YS7V6tRZIP+", + "TlNkWkP35/61UnQvZ1woNKVaEdZiHGeWmw07vZgKpY+/zCcTegNaZA5WICPTGONmV3OoYSdnV4xfs2Gn", + "Osznz/DOJB7vrt0NN+xuZQ2D+wBMz1qlG/P1VvvQjlY6zLSoqSV3ze09zd9S+Gdwl5UpSfrSEz19VhIK", + "kfuOtrsmWndKmFZV9aMJaOxOQmtw+fSCouy9e5Vu5Ylp2FETOdu4oWvmME34GC48N5Ey6LSPTqpr10XY", + "WD6orK9ykS3hBYaiDdKf9vXtkkW0NxgMBj280xsMeoM6tSd7PRPynmGliNAD/H9+xb0Ph73/GfRe/lb+", + "Oer3fvvLfwQV8pbRqs6vbee54c5SF7nB+iGs9YGuC2+9s2fNH37z4QXTQuNOR4LAMcDJ2jGc5XJ25L3e", + "TOtHJ8tRb2alYh5dEdGnfCuhY4HFYotNKbs50OKcVNV1W/3uKgNLIDnGCEaF3QFLZC0sv27NsaCYqd/M", + "GNHQNw5o+Q3BraHv/WMYEer1XDuWfbnI+4oDoI9+4Qph65HWKt0BvLTc71TgNMVg7ZzjhMZwOKybTdKY", + "GOMsg2f6DS0lCaJywdDeYOCejLxBabmBSzeVYccaTkU0Q9YgJBCYQkyGTEQyiL7lMa6lKfl2kqWTpo/Z", + "X7aKvzbcn/3f/rL5v//js5I6ENoKGjfmjFvys1qcN3CpjYRfExFhSVBC9GxlF8V0SpU0wlsMoVFgUPsW", + "RSbYw4Quao2PORM1vFcl53TRwxntWcuLcXf/TNhUzToHz3aDi7th/+j99p/up4aFFXkS0tfe8hwsr/DY", + "T7x2Y2jlfHGrmyfgdEkpOzGfbS+Lk5+2w24iq3baKHWNW63vmCIraM1AljPwP3Y7URoSIuZECH0UDV87", + "Oj3WKswVsQxNa41omA8GuxG8AH8S+4sW4rQqA79tViUeG0MYEHZ+7ZBoxiEMMkn4bYyU3TpTb4oSXOkj", + "C612yf2XYwrBiW7c4hDsbl9EY8GviACXjF5iSiS6IgtjLphWDJqEzdEcC+cavIC0BKO02kfgDSKgHmsl", + "FpLzZxwCPAxbtDqbSYMiOCm5ZYopM7bPnox4pmU8Zl8zFv1LwuaXKMWZUesFMUbzFCsiqFYXS8NiSmKq", + "efSQETgYKMP6zEcRFzHkQHPjTC9X4RuJLo2aeAnNX1KmyfrSHMyaqfSPzpt3F9+/effL8ejN2atfDk9G", + "P736b/2z+ahz8OsfHQOmUQRefU+wIAL9xx8w348meBZi+jqHuZpxYbVd56KHeBWc0T7PCMO0H/G00+38", + "p//P3z7+5sI9TebeXB+DwMA+BsVMa/sMxOC7BCiJbFKNS+fUS6ZZlDM7eOaGminfSH63OhKgolA+Gmcy", + "mJtSt1uUcuj2YHD6/ZaNHtl3/9jsI+u6heFrHsSFFY/lTJNPgcsB/rQk4ZHN0Jg0peC7rkIMHnytGaeh", + "EPEacypfXeZRvV759BasaGtM2ZbU29CLbrfuQDd3DlR+xeZUcAbqOYhL44TI6ln55c3xq9GrX953DvRF", + "EOeRTcQ6e/P2onPQ2R0MBp0QgZKbjAoig4GPh2PJk1wRBC+VMaZ9dJpLCO60zGOSgx0U5DOJ0lzloBaR", + "myjJpeZTIA0olYTyj1/0drYvtl8cDAYHg8H/tA5Y1aS/hnm/PntnMtThvBOcqNkIXHPrPvwR3j2CV4FV", + "gG0SVPvAKhV76qyV+vTab9DGrCpdmckhIMhhZ/f19+ZAbb+Gs+QI0aYCF62Yhmur9/r70Lp41qhlSa80", + "tkGGG4sd1S+D0Rh7fS6tmcj3hPcYBogdEC+wiNGYYxF/W42lgS//+ur03TcSgbUCJ9ZPZERv+MbIkpxI", + "0JVdjE9pbJXd2hKA5Sc1pkTMEAFP6ZxqltNLU8rdd1W2ZI0pFRtq33MwBex4XrhQp9vRM7P/cRPsdDvz", + "D9Wc6vXmQBsa00pXXKME4iSjjKzQAr8QYf+ai6uE4xisRJ9T1vfi3WowNOZBdbsL5dQR+BK60xiz+JrG", + "ajaK+TXTQw7ckPYJKl4urskbPROc/Osf/3x/WhpEtl+PM3tnbu/sf+KdWbslddPBXJtiInkWnsa7LDyJ", + "96f/+sc/3UwedxJGpLyTeG73/5VpoQ5QY3G7TDR+g9+1kMPK/Fze4Etfg2EQunL5nIgEL7zrxI6psz0A", + "nl4blaDG3Wi/80zIqy8X3ZoT117XrXE7g/D18dV29IXbjgQB1jvKCqy8VQfkrXm7VOsDRBegue+xLHyv", + "bSitILTtnVP7584yyYUpTl7RzARTjPC0yP1cBa93fkUzqzTDF86Vbxh9nIOaPeZc9YfMwOfoswmUQ25I", + "5BykCh2enUgTHaCvNLg0lgUhrYJ7uEvwuvGFipx10ThXWq/miiBr4bB5/Hos8PKYoLyM1KxpuXaCy9EH", + "sCxXRDCSjIwWK1uujPkI2Y8aFwemOsHSwn0JlWfV9Tr+6fQcbRwvGE5phH4yrZ7yOE8IOjei2mZ19bpD", + "lgnAUNGdaHKktl8+QTxXPT7pKUGIG2IKjRXODoskMX999s5ikcjN/pC9JXphCYttVKmTKIqwUvaNMv7m", + "SrN+/7VFb8KbcT70lofr3L5enq72Zr9uR6kkhCCXGNk6oRNikhUET0vqc/lzXa16veYl4IE5kH30TuoD", + "OZDDjmZQXLPWMVczX4WbUJLE0lczSpzK8jWt5IX1uFJlrCkke7OgczjK8irp7nQbgQPnJipBywwVBTeI", + "smKzW5ZX0eCq+gYVe3+XQIuqikTR1gZsWgYU1LVhkdaUayT+1qZcz7i4ZNR1lq8/WmYLrG7/hLmBrDRl", + "l8avT+jr3DSylF9gfu66md1hlU6KNakZwD/P8hxKz1jYCrDUSDDmmpdo4xJntG/puB/x9LKLLv+z8oPm", + "cU7x13LyNTKrAXyT6Z/89uVSSO8aA+adw3qxvPt+HMpGuCk030ZKYCZNgPkMZ6SPfoTLCimSahEM8Pn8", + "mDrGr79F3Ajn7tMh00OT1SBDZ8aWdMoom25CtPiYIBzHxtZtTVUu8ESGcH6dAWpJAjajs2g9gKprw8zR", + "pbM5X1Z54rJFetlgY03US5q6WRLQ0MGGobbSXOnu9YQLgZrnyqB32alXkdVqdu91ETF2LIUN7g77f16w", + "izp48jygqkMuyKR0VHgekybHhBXIwk6TK7IwCQLWQYKXXCS+byTswbCx3Fa88L0rY4CH5ja+s3CsGBeJ", + "9YpA1G0t6DvkL1i3FXq9Wi9/VeUNJqD03GRLirFarAPmdFxIT870B9Y2SWDxQYU+MPlMl12j8xPQsBDT", + "xJK4oPR68xCPC9F7l/aXvm3tssiO+xxg22Xag7+1yNtZJ96aEGYrHikSd6uywRUhmVw/KROXOWvw9wly", + "LahjZA7ZsaUYatNFUqsMfZoB5JXX2IpIqLZNLKeHmvX1xuwwfbENMIO9N/sBjp8K1G8d8T6uaacGMqXa", + "5SVOkku0YV/aRIL8HeBQ7V4xzkpivzg6cyRQxFm9P+1qitRc4HKmVDbS/ydH+hRf1huz37oTXqKRvxiA", + "Hrm3t1tNVjADrjVbtRMHYWCat+Ysl7N7ClxqsEY72HdjkXY3UJbLGdpQeKqPjsX70ffE5qdaqimTJMoF", + "WX/WD0E8spgzMKYEU9b78eLiDAkypRJybIPR/CYmvAGczCrwZZyWm67iVXPlzu7e/rPnL1724yvRJ5Ho", + "57JHsFS97T5O8QfO8LXUYtdWusBZdjDfXsvnzRYU42tm+U7bbCYFnmqW7CJx22iuR+UnpafsirK4bQM/", + "6XfXR3s6xfq+/Q2ZIL08mwoMcJef09tw54AeWM3mXV1TAmR1JG8uFU997NuN1QG81cWa86QXY4VvE79r", + "hrscqpsuTFNGJQ+a+Zrios/rwdAVwJpgIYBPja5yYwltSxOYrjEkkHikeCBv1ckOJ8eAvW3fbYUaCti/", + "io/mE8rXZbdDzEglZ9VIJda8oZvoZRG11jOX8AHCqlkE0B3en/pBJf0h64EUdoCOiw6KZosmMagYODbu", + "7Q0uKomz4EQeLzYRRu9P++iiGO03EllvsUUXLlImc/CkgVDUMyKZP4AcMkyoqn9uTYUGiHgTYme4fdYv", + "ypWAUbIwaoENa0xr8zEJzbBR9i7xMVdaGmlXwXK9NffRKmiuSijEfm+w3dvev9ge3DYU4h6wlkNtHVZ5", + "iw1697nP0buT4x1rity8MxrXZ0djDrOy4zJaH23kkniZqFwEY/S9UPiGGPw7h9bfU6R8iTu56l2zEhf6", + "zfuAmQ4BYVmkytsDQdcZ5lq0UW9yywadRQbmh/KUeIK8zaqIaDDl+JjKq+8FwVcAn798bwMAw8jcZ+FA", + "O0DJGC8AFQqMXIJzNZGoAE8pjd/be8/3Xuw+23sxGASAjJcJnkd0BHkzrQbw5ugEJXhRYPptgP8+RuOE", + "j6uEvr/77MXzwcvtnbbjMB7rdutQ6N/uK7RhV+QvrsSZe1IZ1M7O82e7u7uDZ8929lqNyroNWg3KuRgq", + "Isnz3ed72y929lqtQkive+WApesCfKjKwaEpDAR5nzIjEZ3QCAEgG9IfQMr8jDJSOGerZ3KMY1cLInx3", + "KEwTuTKszXRm3zT21jRPFM0SYp7BhrRyScDMj6GlYHEtxoraG7drycJxrw14cnMpXgFxJibjfDq1WHfF", + "0p3aNPZSeKIkiQ9QobKt5HOwm+XAfmuiAzuHltTws1adLOqcRwTm6jIQDYKggk7MplVm5WIdKMvyIEk0", + "LuUPuTDoE9AowmOeK+NVt8VSyk4MkAsDL3PO4nbmjh+4uFqb2KBv4qImTEuDgctwNxnwtdx2X+grvN8m", + "RsA+l+it+cIYCsufs7xaEK8LPVmDInNp+XFhN7bNtJUuw3IL2MxduIzpr+SdDxTK15uY6KfPq2GLKYFa", + "SGqtxKIpxcQjncPrrfOk9IdrDSkt1p2R64dYdMj062my7UmGs/tZ8VURw4WtwcvT5zbOvo/gdEGQn0vQ", + "r520cwc1YZe8P2Q20cjDKwBHGmAJmDDxGaECcUGntNpx1c76gDG7t6FMR1x3pk7/w6YYPFm4AUM8wCAt", + "T0wIrCkA4gO92j3pdDvnRdEoy5iqS/O2KLy1tCJlZP0y5s7Zu9tG3maCT2gICAcigexTq6i5mNSf9wbn", + "ve2/mqh5TX4gsVFmoodSHtdqPNn3211Er8/enTWNqai6hPzRLc1pBbxPGe1SoLgaV6P1VVuFxp0Gfc8U", + "nZSi+MuQaAsoHuN8MiFilAZsbT/o58i8UKAQnH5fFW+1GN1WiT6rbA5o0RMcUTbdbL36AftcbRrrYP5e", + "n717a6v4NAEg660qKv0Y7OM++qWoc4Ven72TJchRP2C4qwPpNeR1nc0WkkY4MS2aIhWU+fY2IM7WAvNZ", + "+aG1TAbE5nB5NHcQ0MZ8muVwDM/f9k7evN9KYzLvVsYEcXUznhA97k2PW8wduECZhFZhEvMmw4chDNn2", + "AHlrVZzg1ovkndfA6iiucDKSCQ+F8Fzohwgeoo33P5jsYD2CLsoqW6l/95F/fPp+FjwxACjf0O05dFi3", + "oFYOeFCVrCOSdarTq3QaOioms2lZ5FmuTcivqhvNr9bXwzONNPd75JKvajZuK1cik6Nl4BNdVAIynxoj", + "trWUSJJhgRVJFrXwyCq+OVl2MJMbEt0i++uVfv2jqWmRCzJSM0HkjCfVsITd7nJdVAkhwHNiS0GZOXl2", + "eMVRisUVXIxOrkY5MytQjSDfXQf3MlMqu8Wkfry4ODPKtiJijpN6DoJccrgfkwQv0Jioa0KYmwqWCPsh", + "qPUcTtlQjkeoUUYE5dU17OwG+j03YcloKnBEkPnKVVW2WyLh1my7lLaXAIBgFBEpG/Z3e9X+2k8nedJu", + "j0PD2l5bhDy6zQZfHJ25sjJF2V63zDvLq3xGRM8cOVe/d/XW7sjVBY5cV8zgwi4JDGMCFY9s4oefl+nC", + "jaCik/68kgvpMQfpZ+/ZfuAUmKXqmnP+Wythr37cQ371FLM4VP7YxPubTPkpwGdB0LHIwZVEYxOLZJL3", + "jVjupy0IgmPKiJS1NOMoF0mn2+lN7KwOtrYSHuEEEAT3drdfbK2O6lwZzmujl0YxXaXuuRgnEwXjkloN", + "/BpMukoSWzjLWhjEzDquuR+APS2HD0LJY323eRKeiycfDJYypm9wpFyRNrCQVTyg2D+2gPNbmQ80mJq6", + "K1rUfumfz0HQJZ1hNauS/9YS7UOYCoQYahox1ofaOhoi/xCUqLgIVeLiQtkEvDFxMYjFfegi/ByibcVX", + "Nnjhz/LZ/v7u/jo+BMymdsztuQtM1bxdTTEm3nLb82sbgGipqszhjvTqYqJ6XdbQlOaISyS1ekF5Rtit", + "1nN/b3fnduvZdiInLkqrxpdCCC1Hp8dGJoo4U5gyIlBKFI6xwlUmA6YlzWWg1AsmKWTpTL5dzVoawhl8", + "yJW71qn6XM7whpJ1bx1Cc4oZnWiGbN/0e5YzvLP/7MAU1ozJZG//Wb/fvy0QxasSeaLVVmyZmDkPk6Iv", + "Z5+2D/eAN9FmLn90zg4vftSMLJfCXFpbckzZgffv4p/lA/jD/HNMWRinok0tVjpZqsFaDQ/LLRAwiQ9Q", + "WW7byT2t6vWHbcMQLAzgOEHUt0rQ5P3BuxU0Tqso/rdI1V2RuqrFlTcsWTSuzZ3LqpYFx5VXTtVP/mpR", + "WpV+WO3uduYueMf2afCgi6qzy47uO9UNlitLKy5V3soIK4opJon5K+IMcHZDlRUrV6R71qIwF1wjtgJX", + "0aX/Y9G79+ORPxDvd1fYy/vJllj87ZYRKisF0r8ty6HruZATR9cc5rDtsbgV2paztbBwwdDsR74L7xLF", + "Ve39zfS/fv8/8uz537d///n9+/+ev/6v41/of79Pzt58EuTIakTAR4X1+2xIfqZEuw/n15aUTrGKAjY6", + "rf41rLB9YiwOKppBAU40JgdD1kM/U0WEKedWy0UcdtAGAU0JvtLiLtSqMWlgm/rjM+Ng1B//4cTgj/U2", + "YpsjLuyGFNAfMh/HPMWUbQ7ZkNm2kJuIBL1A/xWjCGemJhtlSOu/CzQWUEDPenzKzrvoD5xlHzeHzBbN", + "N+DWGYYaZJMiCYs5f60dlYlKta+TAgXCOF2GrLitC0g84/brl/j5lCT1FJ6GRVmtv1nN6cUgBB4I6SV6", + "I6FGDKggBWVrMiryXtCLweayPrdGxyhoaAX5Wb+4yT88zEPm4qacxWMS0wj4ikvbm9nEziJj0lCaNeJl", + "gt8sYG/emlyyGOFczTQvimyee8T5FSVd2NIumggo8NA1Xxr3+oxnvfGiN+NZgXmAhQk+wcZBXVWy/0/P", + "TrT3ngg6sT0FU9c1iQSETjgydmYmA7CwLixN7MKU12Ba9JkT+7qp0yJNaQ4TWa1ywVz1CAI1JAFco6A+", + "EpLJv0VRQsHqJGc8T2I0A/w7pZsJQdh1BkUGCx5HMZnU/12NMNjZfwYarPv37k7rBFKzdKuoLE8COm3q", "WF8Ljm3YJAzAiAcjZwhfExOkb0BbyArsFIrDf8+Ra6g8cQUjMd4pk9ImbTGCRHrJbJvBrCN7DCyCxwjb", - "09TmPlo6hZUkphYtmGAB+CxpgR3ywiRPvn11jhQRqUtn34j07sApMUARPSplbutiHR6dvtjsd4K4RxVX", - "FWzVyiSn6qAD0Ac2eKApJqK00eCUdNHJMSSv2mul1MUg2+AnLlBibsXyMjoA7IyquQebinonx1YATRZl", - "BIIRW4adTddiVr/eDtCbQgXExVCKNMSStlyT5WUCzdp4NJMKsdR6LWsV/GNW/bP3MSQ+QLlBw4sBu7Hx", - "/mpvc3RIUPqiqlnIbnwh+UEhjfYvb++/NMrxl5fRd28mo1sf8CibYRmi7pnv1YSXlvbddyNX2b1oDuGp", - "9DuSNHi2/uaKrnjXkCL6nqt8HoL13O9tb7/d3ru5+e6mALVVZCoPta7AqG0PLnsXIK0ByFWqRo2x3kg/", - "tpHdzi7y/hTNsGTfKXhYs45s7z5tY5SAXttGSfvx0XxihlRwKQdzVUT3GsCvS5okRoCRdMpwgp6jjfOT", - "l7+cvHq1iXro9evT+las+iK4P7fAqoVbANbRJHwFkI4qGfqoSOV7+/YVHK6EQDaEkcMvb49gu9a02ALR", - "1g3u5dk7cPxjOXJxlM2pg7hMvyXXVCq5DHLWKhz5cxB0zaftau67SZo2ytL7q2F4f67gxAZR6zbvAD/X", - "xZIvLecDQMs+ZM7e1wdruxKI9nPRZK2d4Y7AZBuvtBAQaw2gYL/pdrs9LOydDKeC7xJiW76E4xKqb43D", - "2u3QQDLpodQXD4nRyVlZeKl0Rrjma3N6vtPffvIMaoduD9ow9hRHK/o+PTxq3/lgx9wyB3h8EMUHoLDf", - "1mdlCduoIDi5wguovWeWdtgxF6an3XrH1iqSreJrluFub4duWxfjGvBrQZx1gUtylK4s/dEiW7COYZbm", + "09TmPlo6hZUkphYtmGAB+CxpgR3yyiRPXvx8jhQRqUtn34j07sApMUARPSplbutiHR6dvtrsd4K4RxVX", + "FWzVyiSn6qAD0Ac2eKApJqK00eCUdNHJMSSv2mul1MUg2+AHLlBibsXyMjoA7IyquQebinonx1YATRZl", + "BIIRW4adTddiVr/eDtDbQgXExVCKNMSStlyT5WUCzdp4NJMKsdR6LWsV/GNW/bP3MSQ+QLlBw4sBu7Hx", + "/mpvc3RIUPqiqlnIbn0h+UEhjfYvb+8/N8rx55fRd28no1sf8CibYRmi7pnv1YSXlvbddyNX2b1oDuGp", + "9DuSNHi2/uaKrnjXkCL6nqt8HoL13O9tb19s793efHdbgNoqMpWHWldg1LYHl70PkNYA5CpVo8ZYb6Qf", + "28huZxd5f4pmWLJvFDysWUe2d5+3MUpAr22jpP34aD4xQyq4lIO5KqJ7DeDXFU0SI8BIOmU4QS/RxvnJ", + "659Ofv55E/XQmzen9a1Y9UVwf+6AVQu3AKyjSfgKIB1VMvRRkcp3cfEzHK6EQDaEkcOv7o5gu9a02ALR", + "1g3u9dk7cPxjOXJxlM2pg7hMvyU3VCq5DHLWKhz5UxB0zaftau67SZo2ytL7q2F4f6zgxAZR6zbvAT/X", + "xZIvLecjQMs+Zs7elwdruxKI9lPRZK2d4Z7AZBuvtBAQaw2gYL/pdrs7LOy9DKeC7xJiW76E4xKq74zD", + "2u3QQDLpodQXD4nRyVlZeKl0Rrjma3N6udPffvYCaoduD9ow9hRHK/o+PTxq3/lgx9wyB3h8EMUHoLDf", + "1WdlCduoIDi5xguovWeWdtgxF6an3XrH1iqSreJrluFu74ZuWxfjGvBrQZx1gUtylK4s/dEiW7COYZbm", "FqExpUlCJYk4i2VVRp5hiWRmgElNCYxCgh8yGGAXFZWIQUpBOIpEXpoerXRt5f08s3QPZTgzzrQOADj8", - "v5CFRCkFJ2jRPYQ+SlQkpcRDtiFcAlORqQQVOGP9A6QDdF3YuR4aVVDqQ38wZHKWK83ENvvoiDOZp0RY", - "qywaU/AYbSKZG5UWxgursdAMU9KYiCHTrwWgT/8o1JODJ4PBYNDtFJrcrv73IERNd+r87FtoX5OCC+B7", - "zIL8AqqfyBnKWUxEUY6bGHKoh8jd0HH6mZi+7vN24pX9vJSrwgdzHQRwO2zfzwVUhaE26OcQHXoL5Xz/", - "9iJ6qxQgJ7/a5B/71egmEQwERTxPYq3xjfVtZwxyJLZmSEmU4c7mXSrRO1MKszp1G3qsOPo9J2KB3p+e", - "VsIeBJloHtBu4sAlGvaBZzfahp01NpK1o7mJe9mDn70PyNm6pOJJiF8cYNb3MLqcZEOhFcNWRXFeZV/T", - "WmUwi4Qys0+aaFZMsGbKj8l8lOchrUQ/crAk796dHFcoBeMn288Gz573no23n/T24sF2D2/vPunt7OPB", - "ZDd6uru9s7si46RFFtntE8OCqmkgYLgIDx+5MPVQ9HBTkkBNCLCBz1eUxfyqcs8EI1H93m2U67rul2PY", - "Ww8hmPmSYKmMnaCBZZzCbUoi3baJ/LaZikWJobBF8cnbwfbnmllgcA3M+K3ImfFfmtz6wlafegP2N6s6", - "ztvxVhiQyzBZt1p+5+0XbXCw//xg/3MXzWVJrBtjnZzucXObQq8c9nAtDcOlAnoGG2cJ7Fjhw5jPbdZG", - "p9spEkvgb7h1a0HLxeNW2VJNB7YbZiOr+HdD0vBJRTGAkBCDPRcfaJGgyDcd5woVqela1jhKeB4jz+hl", - "oLjA4XXiKQm6GfA/WVuYgdY0WQ9amQAsZ6iYQJlmxODo043YDOMD9BLehUc4NfqTHYSp2+H7uHC8MIEp", - "+ny5ro02s3rI51aRgW+0VoP0v2DaehmsbXR1E0YMOkC/cvimUKsYrxtZzeugzyy/XjfIbtjUX4cYAZ1Z", - "me4A/VTIcYUkaCW/DUnsnyPLsEqgls1Kurzd8Y6mlnLnvNTvbsesaKfbcQsFKeLLyeLvSqpfOn8+KYYi", - "tghO4CyXybi5oomFp4aZUKloJG2Wht7cJvnClhQi8choKU3BnybD02oyxUdOfHl/ijYAgfAvyFqQ9b82", - "i0DRyl2383zv+ZOnO8+ftMIZKge4XgY9gvzj5cGtFUijLB9ZI0TT1I/O3hkjQ2TU9yLI5P2pD+uQCa5Z", - "j565a9Dv/Hn/uQ+vFPN8nHgePIvFZtBcYcOCCGIFL2oIOPydJnM6mbDfP0aXO38XNN2+fiJ3xtsNsLGm", - "o7B968T34i8Zg8m4Z8oDhRFwgKCEbASJekMkzACdE4WAfnoIR6BHFGnDluQclJRd8SBh7e3u7j57ur/T", - "iq7s6LyDMwJrV+BStiPwjhi8iTbenJ+jLY/gTJsOSwGQvZnVMcPnDNnavoOqQNrfHuyGqKTh4i6pxrY9", - "TxuX/L3V0+yk7KJD9nOhwy2d8uBq7+4Onu7tP9tvd4ytHXYkrldzGJcbZJbHAtD7O78B0uTbwzMEmbcT", - "HFWNKC4U60ajUjcaFRRPMKDnNxjYs6dP9vd2d7bboZ2Fojssjl/lwFZ5V+DQBYgisBuBpVhmvd2m2yIk", - "ThkCe0OiBNP0MHK5DLXbx4Cbj4R5rdyENheDNf0vXVwtvm1lRSpsQyYTxogGXKCcFSU1+ut9n1/EhdnM", - "tc31sJ6rh/JfmF49C8tjSofdYikzQeaU5/ILNMSVSU6dJJyLG33bpLC8ITJPlPEzUonen34HPEXTGpKK", - "ZFUdylLjCvCiW07uRue5QiJhIm9arFa70WbrV02423Bqu6uQKyrcoBEyLNacK2froyyPcBLlUEQGF/up", - "ZwVoP5B7n2XJwgTRJwnnDEUzzMAbIWzJKjZFGM14EveDIaf6yWgSDF/gVyjhBuz4kpDM1lcxg9CfaRGG", - "zgna8CuLGVKq1fvcTw2TsRU0qtS4n4YLF2IZygorcs71emLFPRxe80nF5JjwqQSlUEF6QL8O/55hYaL+", - "MTP1guap0SUDkc2BIdaYeehGNTcpn1gF14ockNFtVhJHgkuJSEKnUJvm/WktUXhFclmRLrw+crI62Bak", - "azyHgasMrjjZuqxY6H4MJM58zg0JNAzJeStiEp1xMsUsh4orHiFbi3e/ddzhjEs1KgCgbjhYqUZQRiEX", - "pESJK9LbC3uQeyd4LzrWdpvlsgG+t/p6iarCTTUNsJmnBlc0vFrdggZDZLwMgbUSdauE8apjNt0EpK3E", - "3acSWqUePhjagOQSjy152PGbbaJRwiqr7mdJW7VVMV/tDc7b4qethks7w2p2wiY8ALJxAxels0TbsNCM", - "iJRCIREUE0ZJ7HTJwldpTV2QmZ1IguKc2JUz8qnAdsGxOd4AlMGcjYyyaY3X1ztsYx42Y1hdZQH6tS+2", - "iSuS4czVtyKHtTKBgRLhMoe1VbQllaOwO2u5YUGmeYIFqmMErhiyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJ", - "wq9G+pH8Aeay2Wp2+oNRU0meczM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j", - "1x6hV2HI93YGTWnpDY1WEtKXIRhvyrktyQZPfC4DSXwrpRxXTIjEFhveiD1ZLk1dkxa3kgMfdS7A23l0", - "qhkan4cBcmT4dQ0BBI0JJNi4qS1zjRZssc1UgiUVcjlDf+fjqkG0bXxtoFDXBiuxKASZBAPpYUdXGqTN", - "G0tr4u3uTcAegK3qicJHN8RQWFfSrAxkauInb5aqe82IXTLq5mgqfbWoXOECLQqcANtre8CAeh22QGAw", - "wMFItYCKplBGZuEVF5RozIUA5GUt4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahc", - "aCVrSuI++pun4gFWdZqphQVBB+P5dxLxK1aMccj8QerGc6nbOWTGsihyKJJfvqSbBa1PEwqkB4MTTAnq", - "8G8ngsiZP/dQrUgt411xETcW4Vkg9wrUdgEfK1L8kjCflRXNBFVD09DIfLUcLmcKvcJTq3+iSu1VVK+t", - "urq/XBIRFhKLKRWvtApd8Y6Kp5wYtBWAHoGaevYvw+ILuJEW4CJl8391TZY/nRWNV3+rveYBiDg830Nj", - "tg2aYCOTL1ML9ql60taGqkC+2SrYmGVfAtpwscquMElVEvAKhLS6J9ulvNWj8t1otiSJqr3vPdt/+qRl", - "hZbPctYZmKwv7Zqbpytccg07ddrG7/Ns/9nz57t7+893buRhcQkcDfvTlMTh7w/aINdKH9bkX//45/vT", - "mtdnH4KdBzcalEnhCA+pIY2jOqD3p//6xz/dqG49oBCjWYbibvDbN0bpJP5OukCBqguvnZNshX5/WDES", - "4ILNoA0ymRAwg47MuvXKwdTwNtpJwTjDEVWLACPHVyasvHilBindxh1UHWxI5DVtW/hRzblkPi6zOzdc", - "5+g/jW+4RgvPWhd6kvm4yQ/9ut6r8UKXXgs/xqFFiIEsaowvG7iL+VxhWYmc1n9HkODgUrmW01rMG6vh", - "bes5BxDFYuuZeaGAIVj0mjxpP/K3v7adnt+yYtapr/iHFeew+QjeyOobuJEDRt9ofQ5rjT/YC/B2X43G", - "fgm2lTXuKvXaylv35v22SNNdLkhQ3GA378/LTLzJh3XwXaBHOwa75GXb3QpJNFCTl3QSMKDxhPSKQD2b", - "kYJkbjyC+sxbPPdAqmR0ySeTKqjsfjMIOeDrQFaV6wUrpTWTLiLXzmZRR7A2YDrDzr4cdrQKMOxsp8NO", - "zW0VzFNM8fXIdlAFURmsQgUv88xrg5RuBuOER5emuhgUre6jAUoJZhLlDA5/zau2PVjtHep2Mm9vCgxu", - "YkKcltgWjGlMZnhOofSD9alMK4GY5JoqCQGj0M4BirmBVaqUVrUz1K+ZLMKDctJw6WC2sA3rBvV7nLmI", - "1vJdMPBNoKAr+0gE71pUAM2xX78+7ZoABgg9NAOrxDe6iZoRaAZZdFGrY1D+Ho4fHidkBOOu4+Kny+vo", - "J3+DZ1UQSZS0QNklOdSIAEU8Z6oOmJ+2U+Sq+VvLV1LOINjPhn8AAJrt3dblj0kEJ1Iun8Uqod+CuGt5", - "A3alQ4kDuyEShkMBvqSwr/iNdQjXB2CMDV5VZNOOH9dtvIQjqbgto1Wc6hG5jgiJ68ia4VfaxsrbL4Ox", - "8q+wBeMpChbbtyHeeXl2/bvLpIKxNq22H9PPOOsBDIjbUgvZYTD4LChMldAqGN8edsQohGMaeqFNajO5", - "Xr3Wv5JrBUDkcZ4YdLkw6VpWZS+jdSt+6xTCpgPNxfr6/ndQrs3Em9+qYJsNVX+Imm32rTup07a0O+dE", - "uXfPLRk1V+uvVFSpuLRcwL97pRpjY0ipi+wFj7bTzRoJ7s3CVhGLftsyGZLhlIwyQSb0egXxmBeMYlzF", - "DykPUpHBYIA8N1J8jfaeomiGhayNndHpTCWLagDOXgC06LOKGQqiCHOGwjY7X+6m+3A52s1up996SDg+", - "9zB4lmqHWJF0tAqg+qj0tlnrfIYXYMVpdBI+3d0bDHZ3BrdCqHbDusFyHZWf2NJ/1XaaUuq876yjvxKl", - "6rdQZDMv15O9EhSSootlkkoQnB5A4k2GI4ISMgE0uqKO93rPYr3r1YO3ApVFfino322U3Tfng6/Wpim6", - "suDebhod51ysgv34z9c4RBvYTLSEXRfIudvtDZ683d492H9ysL19F6jSxSI1ZXs8/bh99TTZwZO95Nni", - "6e/bs6fTnXQ3qIddUlOCpw2t/qLfbYyyKS/JKmhQhaWhDTuHjIh6oeB6gW1JEspITxYZUuvTFFfwAuN/", - "X3v+b2bnNzNYKTucVyfpixBYlYtToayHAbqyk1npu6jP5uR49SxulYFUH0iY3upDAfJqNxgoBbHd+UwI", - "hJy1vIbeeS+2vohWZsWtu4pCHnY46cFdbljxEHnXEBC8Wa+6wJcvuYDtdMoFVbN09W1RvFbgdUPc9Eep", - "4iqwUh+dTBlUCfd/LsLkfCVKf9zpdpKPe9UzY39vD7FloX4LArRb7UsFLcLIoAj96lWAV0rFQ5hIdq2r", - "6zH/sN3bfg5xCMnHvR8GvefViIOuWS1/+bbd25VfB23W0K+152o0bT+/UcS1W89VFPQLDVWKK+9lCwJs", - "abysyeyuDpdwW9ng8vHSHtcgcxoF0M+V9OzlNvKFppgkeBECgfcMtbKmPfpEhsZkSplsY7fdHRSG2/10", - "2OmjQ4vEDbpsWYG/0jzUXvfohKYpiamWMY3q35zBsNPSFlfXJW5WBMR9FZDW+mFx7fl6iIR1CVfrrsn+", - "Z+Tjfpb2207jXYXeAXY1p6ICWBe82EV0gjCrVQKlbI4TGttEekiMhHi1A4eIVpKs5QGylAOdnaSLplyh", - "MoW+pb0tZ812wWL85BrsrSswMwxB7HwRQJQCqYuuYl8nxygTPM6jMn80gUGXiB8ir2GhrRDy14fk3qV9", - "AxKzJ1yg9faNJoNGO/tk037XbJOaYJu3enuwfqvvxCjS7eRZvJ6HmZfacbAbQaSvSUEMmGiqy16TBL3J", - "fGjB0d/4K7is8xpbcqRFojxzDhZNU8uUFHC3gIshFNd7TBKir6nlRhBP4jJLgsqSi65nqdtPns2aXJzg", - "kVoeyC+EZFpXAfwj6C/FbBEcmKvvWdwlGwMHqy2Nw6tn6gLZ1aoO7ulaSaxxq3wTblOtAsPlazZvg5dy", - "6Zm/CzBtXzRbRkBxDL8ipL1pxtq3X7qwt0b78V2Y5R5SSHttXQ81fFSH3lvAkLv+y1hgLdZViXcv5J4P", - "kcVbqxk3Yb7Ws0B9q/Nh73+MlRmN+gdbP/zl/+59+M+gtbmmN0siejGZQKDRJVn0TJUfraP3q4inUGJA", - "C9NTSyoEp2BDAjRxexj98e4PCqax+BWnS1OACC2vRM/22gn95T+a45u8ZXwHfHItyX52BY67qFSquLuO", - "NlIipi6W3CWSbfaHDHLTLslCIq/wlxVpHKF+J4tPvAh0dGHEwD5h8ws0plBJUQ6Z1mpxFJFMaxO2lgw1", - "5cA5cB9BcOK3YwuQucRv65A08QQEvT9dgst9/e7tj6/f/Xo8en324tfDk9EvL/4bQjyueqaHuKdpb2//", - "iS0C7q/kdrAQxc3rKfTRqQ3Tt67+SQ4KLeB0SZTmKoegEHIdJbmkc+cgVMntKycsJ+vevhLBZ0LtKpWE", - "ohIsJHRCJwT8+nCd2KAaKh0xUgnV061xgzK0fGMbwhl2gJN6xe9DdSv0VoRXu9zY6qI/mbVjoQaENHDY", - "IeMVytwHtBcqAa/CxX54L6MNyBxxJV5d4uzmzUBRD4sGg5GHX7iSz+D5l6i2+W5lec05T3pavWkoSRC0", - "Jpu1CEbOQ1MmI6HT5HSYjgMyvDXtTukUB/wMIX/CF6mK6Qa0NmNqaf8by4OF8xiO6/UazLE0S1WrL1Az", - "EkjVa05zSLVUOypL/1eDZ3Jmc1epF1tXTVRNmdqy1WtDeBkxB9TwVdnK5Slz6Ig9+Gh9Eu5KvcqbmTeS", - "5r05depDTcFZsUBnemmuZkQQbyPggxIH/4ZLZvNyWqCwmOp/GRFlzKpL6tFSKbibJdooLD9uCYps42Vz", - "+Oo6B6f4uugBXClYLvkfYR5lnaXtlz8CJv0bV1uSTlwTMIyachdGYK9S0ao1cVS1vBk+VS3P27wfPHiW", - "V63gfk1nq0acZR8V0gzR498wVT9xAepgM+bJnQO5w+UfEwEYcHWY9lYY5zQl8YjnavX5t6Xr7ZVf1B8t", - "69c61RcDEUeVdN4mXuBQOcoxLK+0Xg4S5YKqxbleLxvMDWmQrmgsLCR0BD+XHUOhzk+fwGg8CSSMvCSM", - "CBpBGVR9HlPMQGNC70+9animMOISXiuIQK+PTqy5wUH+gvpIFZCei7s8PDvpdDtzIozK3Rn0d/sDOMwZ", - "YTijnYPObn+7P+iAVjWDKW5B6XqbP23zjQvF9SS2ktCP7iX9pcApUfDFbwEkAIg7tK+DCoKnnhKZYSqs", - "FpklgFBgCIbqrwHX312oB+ZW7pplb20zhTRjyH4h2Wu7uR9AUIazA9PcGQwssLmy1y/k7piEga2/2+jR", - "st9WUp1dogDM/ZKa52TLYuk/dTt7g+0bjWnVUODshjp+x7BN4iWgne/fcCFu1ekJM2l5NsnahkP5Jw4I", - "yT9rv33QeybzNMVi4RbMX62MyybBmEiE3btGj1MSRZpVQDGePnrNiHmOsELYRC6LnEENY/ehptDqKTBt", - "u00uQIp+5PHiiy1hpQ9no/hUZWf6uHxaoucvRzsFGS9vpH3kELYN1d4DAf2IiwLcD3ZS9gbP777TI84m", - "CY0U6hUEbOORqYSQnwTwwh32EBfo95wrjIpw/kd0pK3MOi7IrVteRVt/0PiTOd4JCZnBz4hIMTPJEead", - "NYd+6Tgbl0R5nFfeao7wobQH3FQOhMdcVCDIVY+of23VhcHl62gvgMBg+zTTix+Q8Pfu4YTbyRY1WB/y", - "yEHlS5RL8piOk3WxjUshJCjLvSTqa6H5wX1eWbaIwJ/wFD0WAn5JCgmv3K2lS2ErEzkzCnBQAnxTJiza", - "776rCn9vyydelAz4NXTTUM5CGb8qjhd95NbUKP1qARBLgsA84+Vr5UwP72s5YTv3ccJgxoWn6Ns19e2a", - "WnXKDbW4KcDB9E55CxvEjSwQfz77w42tD99sD+1tD60sD4xcWevC3/m4j2xEasRjguSM50mMxgQZvCMX", - "e6Kw6E8/IiyiGZ0TALWDIm15omiGBUSWpCjGChsfeqNhYqVZomhuSzfXc3GI5QLXcSwkGQEO36gJf7KM", - "QKSMkRjpTyx0XwknuFS325z9oIG9aLC8GtHVjEtS4Pkx5d3mkN4sjXYMzfaH7K0FetULCMHUjtdIkgBc", - "7Qr7D2cID5n94HvHQlwgmMRpybmwAMxAapApzbYsp7bpkY5kxENYO28Jw0z1ZEYiOqGRndYlWdh4zmCD", - "reou6QG7cb4/LRI20M5mGK8N4BnD4LzHxTNkKanqv2EQBB0leVw6uRyEEBZjnCTBwhzThI9xMjLrc0kC", - "PsGX8IZdFL++v/MmMR4TU6s9W6gZZ+bvfJwzlZu/x4JfSSKGnc3+kEEihl1rEndLARFdQSG3NOP6nAme", - "mj63zBC3/rgki0/9ITuMU8ocRcAnOJEckWv4DupbAWaG4V4N9GBOU9gPfpRLxVMf+dTRnRkmz1WWK5tR", - "IonqhlA/h0xx9IfDdvy09UfZ4ydwFhMcazrxXjFTAtm6adRyhPXsR/BqwN1OYAGGHX2RmjCPqcBMGdjO", - "ApwSTf0t3SiqI+hDullf4QgzlPHMVJYAopphTXKVNgCrAScJUnCU3LdacIedbJiPhd5Lx424ewYorXaM", - "KEOnP3qHabD3LHyeJIkECUWU/Nf5618R3Mp6D8xrZbiWSelgWmBAcQ6uU8fTXuBohoyjCooJDjs0HnYK", - "d268CWPNpQ2X6fXAp/iDHtoPppsujX/o93VTxl15gH77w7RyoM9Slhoc0GHnUxd5D6ZUzfJx8exDeEGb", - "4MvOK4wAbZhrbhM4CaaANOPd+OaKxCxG3N4CyQJhVHIgP3BlTBkWi1WJhIGltyvIJyaS0VuMP4YQuTjs", - "HAxd7OKw0x12CJvDbzbAcdj5FF4B67VsrlwH91nh3CyI6MlgsLkeCduub8Bn2cIx8IV1wEatqCi7qXfQ", - "wrD+ufwD/9b6Z+H6wUx3XkITGcXfGd8foQPCk9h9TTTggqiJ3ZhFJHFi93pDz/07D/RmRSRJ7ptAH4o8", - "C/dYgdT/qMgRNqs8RivN9w9McYP7ulQqZvuHod9HZz8PWM+t7ZzMXahzuE4JYNBYVRqZlxGW6BzG1DvX", - "yvcL+LVv/+t0P8BUvEj49OLAqO4o4VOUUGbzAbxAZS0e2LWEjwwMTfGdRaVxReI2jCTxr3/8EwZF2fRf", - "//inxXb/1z/+Ccd9y8CrQY3pixnBQo0JVhcH6BdCsh5O6Jy4yUAVWDInYoF2B9bmD4+QV+reSmlyyIbs", - "DVG5YF7ehKnXJm2D1lWg50NZTqSF8dEv0oktJmNiGwN2G3eWzVLe64nuBuAQYQbeBPSt6GgAsOSoKbRt", - "NdFO2GRq5lwxmtbDNJeC9dbzF0WulaHenhngDRkMLHHo3MEDO2m0cX7+YrOPQNsyVAEFg0B3KJuxakT/", - "G09az5MMR6kyFFhlw5sinOExTagzOTZUOzFHMMXRjDJSxhcXWOOuiQM3Us1jDs9OkA2E7MKrQ/b6fAtM", - "rIpEKhekazmBsAijZTk0bvNcoAfgX1RBdFjPvjtkE4IhT+jk2DABD4S7yAcsGmYA5AExrlRVKq91h8wg", - "yVrkYn3wUh6TBD6C/qdYkSu86KKi1q2rjpJgpRVi2dUvD5nBerVr0AOoEuQNsw/8zAyp5yJ5bc6WIJNE", - "q8YQgW/KfkPfGxMukI1w9qr8u+5MkqUZll60FEevz/X8pqAJcmMPhJZen7vd2OwiyVGUUKCGCLMhm0Ig", - "kAPv5ayyq0VC2QyLuBdxfQn4YE6XjF8lJJ428dgjn8juUJKp9BM4Tj/XyfWxCRez5QnoQ2wA6lZ77o7t", - "O+1cd7bFP5PvzhaCvIHzzlhwieE3ZnW/OfJaOPLC6+aceiHP2rFDYLy7iF/TxQMF/DraW15z88Rbsoew", - "6KENB20DXhEu0NnRCcJxLIiUm//e9j49U0Olpfyn70fNih8i9MSOhQsL+mftLVUCeSzs4I0dNcJuXvX6", - "uv79tlUpvtN40xV1eMor7+5vj1qnN7lGSqG3pLVvN8naYFsqIw5lBktq6YFolJBCfCnOqU9F66zKJoy3", - "uHJWikuWPZ8cuwN5f/Zl23XO6nfDPTDF4xpDfEBGWE219qtmPyZqflfsokObXmF+/rpIc3B/UtB9m6JD", - "ZP6Y1MW4tmyaCxqgk8YL9CVRBt7kLvV020Ng4udEuFNtBrowsy6mZT5FBqcFJgSWmNW674l5pZ3qa9r7", - "M2m+sDw3kVjskn8TUVoou+VarVJwT2wJ6LvTb6GHG6m3Xy5sxRJYYJHBijp2biewrG5guWDR5rfIlS9O", - "0SausVRihZs3iQtLtkFTKvSs+5LrDplfb1zLdFavpQxNEjqdWSdATCcQq6f8+t0wyp17GGVRJ1tgRWyI", - "4mPM+z3Ti2y9wHMiFHp9dGLW379St/6AoNX1qpJjXitv13dvXvUIi3hcOE+aZVL75AsrTIb+K7m893/q", - "HmE+K3XiQZPA+Bn7b4LJkYl/71P+v3Z+SuhYYLH4Xzs/4SSjjPyv3cMEKyLV5p0Ry+C+brr7VmAeMfFp", - "/YVWFw1YE5sCZOwagb94q6XM797/U4n9ZtI3EvyLdf0m+7eR/f3lWin+2624UwXA9PFAHq6C2EKrDY++", - "Qdrcg9HUUqQHaVPxIpWgNjMuFTx6fPnNNqicFhTnXxstrf/lgVx5fTjSPTnuwkJCRWmoaGHTB+/JF+DG", - "ce/Cre33/h0Bh+mYTnOeSz8zMcUqmhFps3YTUmXAj03sLq/nRsH7K6bSwX1eHfcuV3+j+zuS+Osbapi3", - "ceitk/ndW21lfvu+lvkNoqnNbLZlN7quJNNmQ6C1wzRtS8YV6NflAPDQuEK6CHqnFZVSXUCgQRwM2f/W", - "+sdviuD0ww8uhTIfDHaewO+EzT/84LIo2akjFcKUoLaC3uGvx+BFnUKgLBTZKxO26+MwNbuB9FxZgX87", - "Bal0JLfXkBwVftOQWmlI3nKt1pDsXtytilQtTXLvOpKjt9CCW0zxP6eW9Cd3j1Q0OJlPJjSihEGBF0hM", - "l0vxgEaT++YZuWVCMrP+SC+YqCKJtFYjC661RkIva0p/yWidbiPOO0dYKZJmCk0FjsgkT0xlBCRnuYr5", - "FXOw7zBBV0GIlvMJXe+uqZFrJJyEFq7+21bTLSp+3beq62ptP84sMJ7Z4rVWuSxFm2bt8mGJ9251yhZX", - "7f1rlY+ZxIz6trx0mdYQAmWMTAGrNDcpc8WXJQJaH719+8qlx2n1RLiiWIq7SliuSOiQ+ZWw+uhFWWLM", - "vOBa0OoDiW06LSQN2tpSMcFxQhmBeGIiQ5ls1fp1D3osvrwEHC7O10oCvudjacutPpwE/GCs4F5kzZNK", - "FWteGiT8un3FaXHyJpyaR8WvLAMKMJ6QrLeFc8V7NuF2a8YNClsYiPIswRHgUOrXDESaxTgwmIh+UwBc", - "IHiSEGGg77JcOXFryIrBUeYVpLeS2YVufpQzRZOLrgnnAfwSiTBbWPynIat0ZmU+yEOGHHsYoSCZGXGt", - "UqUeNOW5hLcgZdjvEuHkCi/kkNnMZfM5VPUVJDIokUnSRz9zAI1AeIop8xivKZf4nRyyCxonZGQxHy4Q", - "lUjOuFCEkRilfE5ktV+CRUKJgEkcYb1yEqV4AeBrBofSrA/PiAE4qyBLcP1vzGIKhfd0z8WUD4YMo53B", - "AKUEM2nzxCWewIVj20AwiMqAvkcY7Q2e269q+wYAwW75N/RpEoLMeYTHyQIRTcWAVKE2YQNTWwjTFBTW", - "2zehQpr9KuybtsJZZWOpdHUd4y7KWZkJD7b+nBWJ63q7VC4YzNN6AQkVxTVowT/GJMJ6PRmv9gOwizyK", - "chG6IPVWexVZ/x0FR29657BU4TzzBEwGEYlhzxlXMzjTHI7S5vcNVFUS1Z/jogkeEi4QRh5dlxYNEuXA", - "GjcApvCiLC/IXLngi83v3dnRx9cyAnf8DVDgY7mfgIj4ZFI5gOuvJnOAV+V3LJPwn/WcHrm6sj6Liyme", - "Mi4VjRwzrJeh/6YQtlYIV69skJonXFz6slWVfn/i4rKtBmbBT+njUsT8GX6Fjgg9PACafnh/BFjDjbKi", - "iebelbQ6fRWnFIQuqqQLdOYo4WyqT1Fplb93t4Gv1W0Y0Dh9mQrj7C4gfrQSMrI/mtK0ejK28Ce4GCLb", - "6kPzIt37PTijfuUK0TRLSEqgdG3PEJve7BIOCsr8U+mBIt2MV+pT5ecuG11QmviDrhOHgK7chm2A9L68", - "XUGmmvDpetDBonOHsBdAHRyyd9LAgV8Y19MFKniwFmgNxD+6mtFoBgiEoLfq9g1AIc6yiwJ8efMAvYSD", - "7GNQQ+cbBthf05rkCTHAgvM0vThYLs76/vQUPjLgg6YM68UBcgVZi/tD6rd8REE9iwRLhX61OIkbhTIO", - "O3qhsNY3i/ltWqzBEhx7yEK4g4xc2QbpBF14EIQXDfhYjt++4lP51biKypIGZi6KI6s6Am0SFneagjxo", - "Enb8bA8GIaTtlkiIZhh3DIS4NJhXfFqUU6iQMs6ytuRrhwlUPE/TFTSMNjxYNalinqu/SBUTIeBjS91N", - "xI02cGRLaeFLTagWRM8d7E0gv2Aok8E3Dy6VZqqdboewPO0c/Gb/NU/TTrdjx+Phot9AuF+DKFlvcDnk", - "Ru+MBxv5TSy/CSBkldl7iJC1m8Oq080S+Rvzwp/eW+hsdg9IhiAf1Iy4X5MI6o23avBhvEC2hJE9v4+R", - "AfwlihIuScXB83jAs6yhqyYzNhuK3Br39PDi3FUbahPBcm4/PXdffgW697pYETdm5KZ770EjyyN4zInA", - "cmk2Ey7qiEvrokm+ekL6cluyNNU2FPKNNm9uZWxFmFpPWGYR9oPYVJ/DueIpVjSCykfRjHPpkX0Bj2xq", - "lFnjcUGZYFoxWq7NILjQpHphzdAXVo04sCYzhP1Hto8+fG7zDsJfuEflFz95VoGC43ed6A/VAaA0u6Bk", - "gjKcS6KlujwlKFpEmiuaUlcERzMU4UzlgkAVP4JSymiapz7utd6xOQaMjovt9KKLxrlCCRZT0MrMQxds", - "E/E0JSwmYJ8bshnBc6pVSoESrAiLFj1JoPrvnKArLi4TjmMwMWQxBk8PVA8URFMggIinROEYKwyCzoU+", - "8SOTxHRRFAQ2aj0j1yU1xEMmcva9qWigm71wA71ABCC7qZwVhSMjHBMWBaGsz79uNvblbdHnRNUn+kCR", - "QbfipQ8ZKuTbXN1wvo4ookcWi82F3cY2bH6F0CubVdhq9ocjo3/PI23m6ub4QA6mYolXneKvw7NUEN1X", - "4116ePcRFyjOTXfeqQQy/7P6hAqG4gdbQWap2cbbOoaKCnnFMt+I52394f48uYUt7yvhhN1Gxb6pFlM5", - "6a+B5dpVvRXPfSAjprUl+Ta5h2PBLqLrwcQnLjwu91iMrZZhm6NZ8G2fOymBQfvi7BvbrrNtG/BwW7bt", - "bLNLLn2PkVPWgxjRMAe3ZtxGVm1NB/+m2Si12Xks88FZZOm5uDe2eFIwQsMaM7xIOI7/DEHCK/xHERfC", - "wF8AoMZjgl/1rIZ+egDY5soib12Xrfn+9HSziUsItZJHCPWIOYSXkqM/S+NlA+7rORGCxhalFB2dHttw", - "XSqRyFkfvU6pQoqjS0KyMqMFsgr7en4OCGS5oHwF8aPbIUyJRcYpU2tHUb56N4P5dKsy9PfMJy2e9zd3", - "eGt3OFj2Hx87Ay4DORtmAqs1U4XV2jqjlE24SI1chsc8161rHqSXSe+nQSqY0ITIhVQkNVGJkzyB4wa1", - "IWz9X/ud2eUuxOTqk2PS5TIiUiol5UwOmc0VyYjQfevPdftegFXQIaBwwV/PDJP8OoL39GBMvBpWTasG", - "kE1QV7Rz0NnCWbYVY4UbAsTs8D5jSD9BNB6Si3TMExqhhLJLiTYSemnUEzSXKNF/bK4M5xvBd1+6uvHt", - "T5Ze6RM24cHacYZmC2L+U2V1WbbmHJOPjq29JP5hcfwHNjrM1tbXTxYEJz2oR+yAe1CuaEI/GlanG6FS", - "0cikHOFi7d6fFky1P2SnRAn9DobUtiQxiAagXW5lgkdbw3ww2I0yCuhvuwQGBwyv+XEKPR6dvTNpqCTl", - "YtEdMv0PaPjt4Znx7k6wtSZ4A7WFk9HJ1us1Ac7nsEz/xhGCZoIr0QuCG/7NJXhzjJHGMyQbjijPVqlK", - "PPvTh7BaCe6bXeFx2hUA5KmYzUYB7OXQuMI2hDlP8lT/w/xxsg7XTOFo9h5e/WqkXTOctd24CT6KQ2nn", - "FBNT2/JBnB5mwR5rzKpeODcFEGIq0YDBW+BQ/Rmp+8ub7/11/ArdnXZFXd3Yr+Zs3ffNZ8fgEDb89Xgs", - "x9xQmpuJ4qutT1eYNluffkx4dCktFItvNtR6G+Cr6x9LPGzrIgQxATJDkYUwMkBZRHaHrGaANIg/EmGk", - "iEgpw8kWzNk0AsjezoqF55xCgnYEeSo9SWPATEoAvhvg7/RswFDlGvA8utJW1vLf8Z2RiqMxiXhKHNr5", - "Zkh1+xum6icuqtDlXwtffOutP0ACYgr29jVo7c09fhZ6+ym+hlDpOLcOZTeijZe8/NGYgroI9mbY2R3I", - "YaeLhp2ddNjRO3CEwYSKFdpHKWW5IrKPjo19C1JwnwyQJBFnsXSg686CtzuQTQm5hiwbsjufwHf3KfZY", - "qoKlfGM7CbEH/R7S30PSDtrwD5w9k3EXDl2MeK6Mud+eK/tWTBSYRzbv3VfrnZFvun0bTv43e3wrPAp2", - "WbNLb+sNZ89yOSPNJrdXppBRrsYA5u2Ki8oZ+jsfyy5i5MpYw4VU/SW+p78+Mx3cR6EB3dVNigzYuX+r", - "MNCiwkC5VmGwRhNgqa9kRx0GsZFcZ1woQHG0ufaGhkCTAOQIHuEEvT46GbJIsyIDLShIyoE7WTx0cwsf", - "/u0cvTh600XHUOgS/ZyPN/voNUsWrty48dEMmZHEDPOKMENjQ7UkDl3PZuxAPXcZLK47eKDK0eZkBDwr", - "bq9ckHi3MyM4Bonkj84rbjoLoA6/eaUPEAD/mi+Lbe+sFD46b4gSi97hRBGx3OypzZNiBWaGvaQdBJ0V", - "3Azwpe5QOuS1sk8jGxhojN2dTgAp49O3og93XyD1frxkJk7ElNsb54A0yiDJAMeLxxXLJGeoYI4hFuhf", - "10XZhKYsYcvLVioY0GVT5PdXZHJfybsq2PL/rqcLZvpoHU1ZZZ80ERflVtZ6el1y8MzAIVtHVYQzHFG1", - "6CKcJPaOsjdBEZHSK8TfsSD4MuZXrD9kb4pCLzahFx2dves6Ry2Kqbw0LVhfbB+9nhMh83ExOAQHzXiN", - "Yc1JPGSKowgnUZ5ocYNMJiSCXFyo3yIbfLnFUDp3eHbKToLFZryo9vzR1bgL0wTsXkkWdYrbMlu9JUiU", - "YJo2g49bQQ0CDiHUYKwb5QxRNklsSFUkuJTINtUjCZ3ScWIDhGQfvZ0RJHFKhixLMGNEoFyaqHg99F4m", - "iJS5SfDWDQBIr6GoLiqBBTPBlQ1NSDgX0kQTaAp/f4qkItkKMntjWj6FOd+RbGsatz09kJG6NoZmU4h9", - "BekNMZRiFlzTUZ64AMZ7DUU3A3poKfGxHPy3gk6nROhTgQ2TNeF45li75TSHvpKx3Fjv8rx4q129y6JV", - "LyvRy9hbCQw3KrG2487Nov4CnV/SRuxA++hmWcS/6I9a9l3NVg0Pwj76zFmGSnf+O1bJPPeSBNsasEoK", - "f2zmJG/klaNaSbRdD6vVOrP2LjNdW+NnPRhs1mNGy8KV9NkmhffrI4TB/aI83HeRtcdNWxW0q4pu2pDy", - "vx5N/6ugwLuB0X9glJNbwOh/VXn3gHP+cPgnwYP6UHn0Fd+zK7b7p0fCv6v0eQOHD3BsTenzhuvZ4NWV", - "itJ7+047Ncm2+GeS4G284w3kd7fs37T+FiqDt1jrXNCa4EmaqYULaLO+yjLoTNKPpN/gCC7iVu/OFXyL", - "kM4vRx6OThsDOv+ctfEfJGbUlg6kEp0cB4rOPzKMQf/MVS6WLX3r9LCIZnROmo3u1RNslygTpJfxDJwr", - "sVkwux7uLlNY9KcfkW3eYq7af0HtSYDqJzGKqSCRShamDqjmCKaP7yQSXGsC8JyLRXOUiDkiPwmeHtrZ", - "rLkP7ZmyxrAyzjBd9GKscG/uuM0KE9pnRHe6eErN8BBl6OWPaINcK2EqXKCJ1nwQnRRLSq4jQmIJNLnp", - "D3h70GDZpB/JaDpuM8oVtUpe21owKMql4qnb+5NjtAG1z6aE6b3Qov4EJNlM8DmNSVwZY2fOE7Oq2w0L", - "elO7qxYqisJ1Trkwg3sQGabNhTT9SLMqWyhCYsaUYRjc2qog1TNlkvh1f5gyF4Bj98iN4tsVZjW/Dafs", - "aEqEOpx2ERXnBuJ589s195ivOT8Zyt1pldvOheesNl63y49qmbZ0F4Ufity5+zVbv/96UnqofJTZPNZ0", - "Pi8U0iaz+ddFgoP7ux/u21z+/hGngL4kTvn2TOXQgG4xRDCvIKY7JnOS8CyFeujwbqfbyUXSOejMlMoO", - "trYg9nvGpTrYe/50t/Ppw6f/PwAA//91ZvGSme8BAA==", + "P5GFRCkFJ2jRPYQ+SlQkpcRDtiFcAlORqQQVOGP9A6QDdF3YuR4aVVDqQ38wZHKWK83ENvvoiDOZp0RY", + "qywaU/AYbSKZG5UWxgursdAMU9KYiCHTrwWgT/8o1JODZ4PBYNDtFJrcrv73IERN9+r87FtoX5OCC+B7", + "zIL8AqqfyBnKWUxEUY6bGHKoh8jd0nH6iZi+7vN24pX9vJSrwgdzHQRwO2zfTwVUhaE26OcQHXoH5Xz/", + "7iJ6qxQgJ7/a5B/71eg2EQwERTxPYq3xjfVtZwxyJLZmSEmU4c7mXSrRO1MKszp1G3qsOPo9J2KB3p+e", + "VsIeBJloHtBu4sAlGvaBZ7fahp01NpK1o7mNe9mDn30IyNm6pOJJiJ8dYNb3MLqcZEOhFcNWRXFeZV/T", + "WmUwi4Qys0+aaFZMsGbKj8l8lOchrUQ/crAk796dHFcoBeNn2y8GL172Xoy3n/X24sF2D2/vPuvt7OPB", + "ZDd6vru9s7si46RFFtndE8OCqmkgYLgIDx+5MPVQ9HBTkkBNCLCBz9eUxfy6cs8EI1H93m2U67rul2PY", + "Ww8hmPmSYKmMnaCBZZzCbUoi3baJ/LaZikWJobBF8dnFYPtTzSwwuAZmfCFyZvyXJre+sNWn3oD9zaqO", + "8268FQbkMkzWrZbfeftFGxzsvzzY/9RFc1kS68ZYJ6cH3Nym0CuHPVxLw3CpgJ7BxlkCO1b4MOZzm7XR", + "6XaKxBL4G27dWtBy8bhVtlTTge2G2cgq/t2QNHxSUQwgJMRgz8UHWiQo8k3HuUJFarqWNY4SnsfIM3oZ", + "KC5weJ14SoJuBvxP1hZmoDVN1oNWJgDLGSomUKYZMTj6dCM2w/gAvYZ34RFOjf5kB2Hqdvg+LhwvTGCK", + "Pl+ua6PNrB7yuVVk4But1SD9L5i2XgZrG13dhBGDDtAvHL4p1CrG60ZW8zroM8uv1w2yGzb11yFGQGdW", + "pjtAPxRyXCEJWslvQxL758gyrBKoZbOSLm93vKOppdw5L/W72zEr2ul23EJBivhysvi7kuqXzp9PiqGI", + "LYITOMtlMm6uaGLhqWEmVCoaSZuloTe3Sb6wJYVIPDJaSlPwp8nwtJpM8ZETX96fog1AIPwLshZk/a/N", + "IlC0ctftvNx7+ez5zstnrXCGygGul0GPIP94eXBrBdIoy0fWCNE09aOzd8bIEBn1vQgyeX/qwzpkgmvW", + "o2fuGvQ7f9l/6cMrxTwfJ54Hz2KxGTRX2LAggljBixoCDn+nyZxOJuz3D9HVzt8FTbdvnsmd8XYDbKzp", + "KGzfOvG9+EvGYDLumfJAYQQcICghG0Gi3hIJM0DnRCGgnx7CEegRRdqwJTkHJWVXPEhYe7u7uy+e7++0", + "ois7Ou/gjMDaFbiU7Qi8IwZvoo235+doyyM406bDUgBkb2Z1zPA5Q7a276AqkPa3B7shKmm4uEuqsW3P", + "08Ylf2/1NDspu+iQ/VzocEunPLjau7uD53v7L/bbHWNrhx2Jm9UcxuUGmeWxAPT+zm+ANHlxeIYg83aC", + "o6oRxYVi3WpU6lajguIJBvT8FgN78fzZ/t7uznY7tLNQdIfF8asc2CrvChy6AFEEdiOwFMust9t0W4TE", + "KUNgb0mUYJoeRi6XoXb7GHDzkTCvlZvQ5mKwpv+li6vFt62sSIVtyGTCGNGAC5SzoqRGf73v87O4MJu5", + "trke1nP1UP4L06tnYXlM6bA7LGUmyJzyXH6GhrgyyamThHNxq2+bFJa3ROaJMn5GKtH702+Ap2haQ1KR", + "rKpDWWpcAV50x8nd6jxXSCRM5E2L1Wo32mz9qgl3G05tdxVyRYUbNEKGxZpz5Wx9lOURTqIcisjgYj/1", + "rADtB3LvsyxZmCD6JOGcoWiGGXgjhC1ZxaYIoxlP4n4w5FQ/GU2C4Qv8GiXcgB1fEZLZ+ipmEPozLcLQ", + "OUEbfmUxQ0q1ep/7qWEytoJGlRr303DhQixDWWFFzrleT6y4h8NrPqmYHBM+laAUKkgP6Nfh3zMsTNQ/", + "ZqZe0Dw1umQgsjkwxBozD92o5iblE6vgWpEDMrrNSuJIcCkRSegUatO8P60lCq9ILivShddHTlYH24J0", + "jecwcJXBFSdblxUL3Y+BxJlPuSGBhiE5b0VMojNOppjlUHHFI2Rr8e63jjuccalGBQDULQcr1QjKKOSC", + "lChxRXp7YQ9y7wTvRcfa7rJcNsD3Tl8vUVW4qaYBNvPU4IqGV6tb0GCIjJchsFaibpUwXnXMptuAtJW4", + "+1RCq9TDB0MbkFzisSUPO36zTTRKWGXV/Sxpq7Yq5s97g/O2+Gmr4dLOsJqdsAkPgGzcwkXpLNE2LDQj", + "IqVQSATFhFESO12y8FVaUxdkZieSoDgnduWMfCqwXXBsjjcAZTBnI6NsWuP19Q7bmIfNGFZXWYB+7Ytt", + "4opkOHP1QuSwViYwUCJc5rC2irakchR2Zy03LMg0T7BAdYzAFUOWizSh7KpN63KRjnlCI6Q/qDugJzxJ", + "+PVIP5LfwVw2W81OfzBqKslzbgZnE/DMhtT6LafwnZ7lZi39FywxW+b7LYBmaROmFQzJ/oEmxMLovWP0", + "xiP0Kgz53s6gKS29odFKQvoyBONtObcl2eCJz2UgiW+llOOKCZHYYsMbsSfLpalr0uJWcuCjzgV4N49O", + "NUPj0zBAjgy/riGAoDGBBBs3tWWu0YIttplKsKRCLmfo73xcNYi2ja8NFOraYCUWhSCTYCA97OhKg7R5", + "Y2lNvN29DdgDsFU9UfjolhgK60qalYFMTfzk7VJ1rxmxS0bdHE2lrxaVK1ygRYETYHttDxhQr8MWCAwG", + "OBipFlDRFMrILLzighKNuRCAvKwlHM7cbADfRMs8eq0dwBS6mJEFEiTFlA0ZZYWRFFDLCGJkToSXjsqF", + "VrKmJO6jv3kqHmBVp5laWBB0MJ5/IxG/ZsUYh8wfpG48l7qdQ2YsiyKHIvnlS7pZ0Po0oUB6MDjBlKAO", + "/3YiiJz5cw/VitQy3jUXcWMRngVyr0BtF/CxIsWvCPNZWdFMUDU0DY3MV8vhcqbQKzy1+ieq1F5F9dqq", + "q/vLJRFhIbGYUvFKq9AV76h4yolBWwHoEaipZ/8yLL6AG2kBLlI2/1fXZPnTWdF49bfaax6AiMPzPTRm", + "26AJNjL5MrVgn6onbW2oCuSbrYKNWfYloA0Xq+wKk1QlAa9ASKt7sl3KWz0q341mS5Ko2vvei/3nz1pW", + "aPkkZ52Byfrcrrl5usIl17BTp238Pi/2X7x8ubu3/3LnVh4Wl8DRsD9NSRz+/qANcqP0YU3+9Y9/vj+t", + "eX32Idh5cKtBmRSO8JAa0jiqA3p/+q9//NON6s4DCjGaZSjuBr99Y5RO4u+kCxSouvDaOclW6PeHFSMB", + "LtgM2iCTCQEz6MisW68cTA1vo50UjDMcUbUIMHJ8bcLKi1dqkNJt3EHVwYZEXtO2hR/VnEvm4zK7c8N1", + "jv7T+IZrtPCidaEnmY+b/NBv6r0aL3TptfBjHFqEGMiixviygbuYzzWWlchp/XcECQ4ulWs5rcW8sRre", + "tp5zAFEstp6ZFwoYgkWvyZP2I3/7a9vp+S0rZp36iv+24hw2H8FbWX0DN3LA6Butz2Gt8Qd7Ad7tq9HY", + "L8G2ssZdpV5beevevt8WabrLBQmKG+z2/XmZibf5sA6+C/Rox2CXvGy7WyGJBmrykk4CBjSekF4RqGcz", + "UpDMjUdQn3mL5x5IlYyu+GRSBZXdbwYhB3wdyKpyvWCltGbSReTG2SzqCNYGTGfY2ZfDjlYBhp3tdNip", + "ua2CeYopvhnZDqogKoNVqOBlnnltkNLNYJzw6MpUF4Oi1X00QCnBTKKcweGvedW2B6u9Q91O5u1NgcFN", + "TIjTEtuCMY3JDM8plH6wPpVpJRCT3FAlIWAU2jlAMTewSpXSqnaG+jWTRXhQThouHcwWtmHdoH6PMxfR", + "Wr4LBr4JFHRlH4jgXYsKoDn2mzenXRPAAKGHZmCV+EY3UTMCzSCLLmp1DMrfw/HD44SMYNx1XPx0eR39", + "5G/wrAoiiZIWKLskhxoRoIjnTNUB89N2ilw1f2v5SsoZBPvZ8A8AQLO927r8MYngRMrls1gl9DsQdy1v", + "wK50KHFgN0TCcCjAlxT2Fb+1DuH6AIyxwauKbNrx47qNl3AkFbdltIpTPSI3ESFxHVkz/ErbWHn7ZTBW", + "/mdswXiKgsX2bYh3Xp5d//4yqWCsTavtx/QzznoAA+K21EJ2GAw+CwpTJbQKxreHHTEK4ZiGXmiT2kxu", + "Vq/1L+RGARB5nCcGXS5MupZV2cto3YrfOYWw6UBzsb6+/z2UazPx5ncq2GZD1R+jZpt9617qtC3tzjlR", + "7t1zS0bN1forFVUqLi0X8O9eqcbYGFLqInvBo+10s0aCe7OwVcSi37ZMhmQ4JaNMkAm9WUE85gWjGFfx", + "Q8qDVGQwGCDPjRTfoL3nKJphIWtjZ3Q6U8miGoCzFwAt+qRihoIowpyhsM3Ol7vpPlyOdrPb6bceEo7P", + "PQyepdohViQdrQKoPiq9bdY6n+EFWHEanYTPd/cGg92dwZ0Qqt2wbrFcR+UntvRftZ2mlDrvO+vor0Sp", + "+i0U2czL9WSvBYWk6GKZpBIEpweQeJPhiKCETACNrqjjvd6zWO969eCtQGWRXwr6dxtl98354Ku1aYqu", + "LLi3m0bHORerYD/+8zUO0QY2Ey1h1wVy7nZ7g2cX27sH+88OtrfvA1W6WKSmbI/nH7avnyc7eLKXvFg8", + "/3179ny6k+4G9bArakrwtKHVn/S7jVE25SVZBQ2qsDS0YeeQEVEvFFwvsC1JQhnpySJDan2a4gpeYPzv", + "a8//7ez8ZgYrZYfz6iR9EQKrcnEqlPU4QFd2Mit9F/XZnByvnsWdMpDqAwnTW30oQF7tBgOlILY7nwiB", + "kLOW19A778XWF9HKrLh1V1HIww4nPbjLDSseIu8aAoI361UX+PIlF7CdTrmgapauvi2K1wq8boib/iBV", + "XAVW6qOTKYMq4f7PRZicr0TpjzvdTvJhr3pm7O/tIbYs1G9BgHarfamgRRgZFKFfvQrwSql4CBPJrnV1", + "PebvtnvbLyEOIfmw992g97IacdA1q+Uv37Z7u/LroM0a+rX2XI2m7Ze3irh267mKgn6ioUpx5b1sQYAt", + "jZc1md3V4RJuKxtcPl7a4xpkTqMA+qmSnr3cRr7QFJMEL0Ig8J6hVta0R5/I0JhMKZNt7La7g8Jwu58O", + "O310aJG4QZctK/BXmofa6x6d0DQlMdUyplH9mzMYdlra4uq6xO2KgLivAtJaPyyuvVwPkbAu4WrdNdn/", + "hHzcT9J+22m8q9A7wK7mVFQA64IXu4hOEGa1SqCUzXFCY5tID4mREK924BDRSpK1PECWcqCzk3TRlCtU", + "ptC3tLflrNkuWIyf3IC9dQVmhiGInc8CiFIgddFV7OvkGGWCx3lU5o8mMOgS8UPkNSy0FUL++pDc+7Rv", + "QGL2hAu03r7RZNBoZ59s2u+abVITbPNWbw/Wb/W9GEW6nTyL1/Mw81I7DnYriPQ1KYgBE0112WuSoDeZ", + "31pw9Lf+Ci7rvMaWHGmRKM+cg0XT1DIlBdwt4GIIxfUek4Toa2q5EcSTuMySoLLkoutZ6vazF7MmFyd4", + "pJYH8hMhmdZVAP8I+ksxWwQH5up7FnfJxsDBakvj8OqZukB2taqDe75WEmvcKt+E21SrwHD5ms3b4KVc", + "eebvAkzbF82WEVAcw68IaW+bsfbtly7srdF+fB9muccU0t5Y10MNH9Wh9xYw5K7/MhZYi3VV4t0LuedD", + "ZHGBp1CUbo0fpZFUQgUcwcmBkcK2ihRUstT/RFCuUzrAFicgwOcuo9dW9UNUITzF9RINTfUfjSd6fQxR", + "c0WVC2siaAK/rafD+ub3w97/GHM7GvUPtr77y//d++0/g2b3mgFBEtGLyQQirq7IomfKHSk8rV6hf5ha", + "C1qrmNozQ3AKxjSAVbdcyR/v/qDgnotfcLo0BQhV82oVba+d0F/+oznQy1vGd3BhrD27n1yK5D5Ktiru", + "7uWNlIipC6p3GXWb/SGDJL0rspDIq4BmZTt3Yr+RxSdeKD66NOTeJ2x+icYUSkrKIdPqPY4ikmm1yhbV", + "oaYuOgc2LAhO/HZsJTZ3Xqxn1gRWEPT+dAk3+M27i+/fvPvlePTm7NUvhyejn179N8S6XPdMD3FP097e", + "/jNbDd1fye1gRY7bF5boo1Obr2BjHiY5aPYAWCZRmqscomPITZTkks6dp1Qldy8hsZy1fPeSDJ+IOaxU", + "EgrPsNjYCZ0QCHCAe9VGF1HpiJFKKCNvrTyUoWXRxRDOsANXiuJOkggW8NBbEV7tcmOri/5s1u4uMWis", + "gcMOqb9Q7z+gxlEJwB0uCMZ7GW1ACo2rdesyiDdvhw57WDQYDMH8zCWNBi8/R9nRdyvrjM550tN6XkNt", + "hqBZ3axFMIUAmjKpGZ0m78t0HFBmrI17Sqc44HAJOVY+S3lQN6C1qWNL+99YJy2c0HFcL1xhjqVZqlqh", + "hZq1RKpec75HqsX7ESTaBqKIcmaTeKkXZFjN2E2Z2rJlfEPAITEH+PRVadvlKXMwkT34aH028koF05uZ", + "N5LmvTl1elRN01uxQGd6aa5nRBBvI+CDsiDALZfMJii1gKMxZRAzIsrgXZfdpMVz8LtLtFGYwNwSFGnX", + "y36B1QUfTvFN0QP4lLBccsTCPMqCU9uvvwdw/reuyCaduCZgGDUtNwxFX6WiVWviqGp5M3yqWp63eT94", + "8CyvWsH9ms5WjTjLPiqkGaLHv2GqfuAC9OJm8Jd7R7SHyz8mAsDw6nj1rcDeaUriEc/V6vNva/jbK78o", + "xFoW8nU2AAxEHFXympt4gYMnKcewvNJ6OUiUC6oW53q9bFQ75IO66rmwkNAR/Fx2DBVLP34E6/kkkDnz", + "mjAiaAT1YPV5TDEDjQm9P/XKApoKkUvAtSACvTk6sXYXh30MejRVQHouAPXw7KTT7cyJMLaHzqC/2x/A", + "Yc4IwxntHHR2+9v9QQe0qhlMcQtq+NtEcqtRFxr8SWwloe/dS/pLgVOi4ItfA5AIEIBpXwcVBE89JTLD", + "VFgtMksAqsEQDNVfQ4EDd6EemFu5a5a9tfEY8q0hDYhkb+zm/gaCMpwdmObOYGAR3pW9fiGJyWRObP3d", + "htGW/baS6uwSBfD+l9Q8J1sWS/+x29kbbN9qTKuGAmc31PE7hm02MwHtfP+WC3GnTk+YyU+02eY2Lsw/", + "cUBI/ln79Te9ZzJPUywWbsH81cq4bBKMiUTYvWv0OCVRpFkFVCXqozeMmOcIK4RNCLfIGRRzdh9qCq2e", + "AtO22+QCrel7Hi8+2xJW+nA2io9VdqaPy8clev58tFOQ8fJG2kcOatxQ7QMQ0Pe4qET+aCdlb/Dy/js9", + "4myS0EihXkHANjCbSoh9SgA43YEwcYF+z7nCqMhreEJH2sqs44LcuuVVtPUHjT+a452QkD/gjIgUM5Ml", + "Yt5Zc+iXjrPxzZTHeeWt5ggfapzATeXQiMxFBYJc9Yj611ZdGFy+jvYCUBS2TzO9+BEJf+8BTridbFGM", + "9jGPHJQARbkkT+k4WV/juBRCgrLca6K+FJofPOSVZasp/AlP0VMh4NekkPDK3Vq6FLYykTOjAAclwLdl", + "5qb97puq8HdRPvHChcCvoZuGuh7KOJhxvOgjt6ZG6VcLwJoSBOYZL18rZ3p4X8oJ23mIEwYzLjxFX6+p", + "r9fUqlNuqMVNAQ6md8pb2CBuZYH489kfbm19+Gp7aG97aGV5YOTaWhf+zsd9ZENzIx4TJGc8T2I0JsgA", + "P7kgHIVFf/oBYRHN6JwAuh9Uq8sTRTMsIMQmRTFW2PjQGw0TK80SRXNburmeC8gsF7gO6CHJCEJgRk1A", + "nGUoJmWMxEh/YqNmSlzFpQLm5uwHDexFg+XViK5nXJIC2JAp7zaHPG9ptGNotj9kFxbxVi8gRJU7XiNJ", + "Ari9K+w/nCE8ZPaDbx0LcRFxEqcl58ICwBOpgeg027Kc46dHOpIRD4EOXRCGmerJjER0QiM7rSuysIGt", + "wQZbFaDSA3bjfH9aZK6gnc0wcB2ELoVRio+LZ8hSUtV/wyAaPEryuHRyOSwlLMY4SYIVSqYJH+NkZNbn", + "igR8gq/hDbsopcOl9CYxHhNTtD5bqBln5u98nDOVm7/Hgl9LIoadzf6QQUaKXWsSd0sBEV1DRbs04/qc", + "CZ6aPrfMELf+uCKLj/0hO4xTyhxFwCc4kRyRG/gO4sYAPMRwrwZ6MKcp7Ac/yqXiqQ8B6+jODJPnKsuV", + "Ta2RRHVD8KdDpjj6w4Fcftz6o+zxIziLCY41nXivmCmBbN00ajnCevYjeDXgbiewAMOOvkhNmMdUYKYM", + "fmmB0omm/pZuFGUi9CHdrK9whBnKeGZKbABRzbAmuUobAFqBkwQpOEruWy24w042zMdiEKbjRgBCgxhX", + "O0aUodPvvcM02HsRPk+SRIKEIkr+6/zNLwhuZb0H5rUyXMvktjAtMKA4B9ep42mvcDRDxlEFVRWHHRoP", + "O4U7N96EsebShsv0euBT/E4P7TvTTZfG3/X7uinjrjxAv/5hWjnQZylLDSDqsPOxi7wHU6pm+bh49lt4", + "QZtw3M4rjABtmGtuEzgJpgC549345orELEbc3gLJAmFUciA/cGVMGRaLVRmVgaW3K8gnJpLRW4w/hhC5", + "OOwcDF3s4rDTHXYIm8NvNsBx2PkYXgHrtWwu4Qf3WeHcLIjo2WCwuR4S3K5vwGfZwjHwmXXARq2oqD+q", + "d9Di0f65/AP/1vpn4frBTHdeYjQZxd8Z35+gA8KT2H1NNOCCqIndmEUkcWL3ekPPwzsP9GZFJEkemkAf", + "izwL91hRsuBJkSNsVnmMVprvH5niBg91qVTM9o9Dv0/Ofh6wnlvbOZm7UOdwwRYA47GqNDIvIyzROYyp", + "d66V71fwa9/+1+l+AC55mfDp5YFR3VHCpyihzOYDeIHKWjywawkfGTye4jsLz+Oq5W0YSeJf//gnDIqy", + "6b/+8U8Lcv+vf/wTjvuWwZmDYtuXM4KFGhOsLg/QT4RkPZzQOXGTgXK4ZE7EAu0OrM0fHiGv5r+V0uSQ", + "DdlbonLBvLwJU7hO2gatq0DPh7KcSItnBGlCE1tVx8Q2Buw27iybpXzQE90N4ELCDLwJ6FvR0QCA6lFT", + "cdxqop2wydTMuWI0rYdpLgXrrecvitwoQ709M8BbMhhY4tC5gwd20mjj/PzVZh+BtmWoAionge5QNmPV", + "iP5XnrSeJxmOUmUosMqGN0U4w2OaUGdybCj7Yo5giqMZZaSMLy5A110TB26kmsccnp0gGwjZhVeH7M35", + "FphYFYlULkjXcgJhoVbLunDc5rlAD8C/qILosJ59d8gmBEOe0MmxYQIeGnmRGFk0zADRBGJcqaqUoOsO", + "mYHUtRDO+uClPCYJfAT9T7Ei13jRRUXRX1cmJsFKK8Syq18eMpNqaNegB5gtyBtmH/iZGVLPRfLanC1B", + "JolWjSEC39Q/h743JlwgG+HcLfNKXXcm29QMSy9aiqM353p+U9AEubEHQktvzt1ubHaR5ChKKFBDhNmQ", + "TSEQyKEYc1bZ1SKhbIZF3Iu4vgR8VKsrxq8TEk+beOyRT2T3KMlU+gkcpx/r5PrUhIvZ8gT0ITZIfas9", + "d8f2nXauO9vin8l3Zyti3sJ5Zyy4xPAbs7pfHXktHHnhdXNOvZBn7dhBUd5fxK/p4pECfh3tLa+5eeIt", + "2WNY9NCGw/gBrwgX6OzoBOE4FkTKzX9ve5+eqaHSUv7T96NmxY8RemLHwoVFP7T2liqBPBV28NaOGmE3", + "r3qhYf9+26pUIWq86YqCROWVd/+3R63T21wjpdBb0trXm2RtsC2VEYd6iyW19EA0SkghvhTn1KeidVZl", + "E8ZbXDkrxSXLnk+O3YF8OPuy7Tpn9bvhAZjicY0hPiIjrKZa++XDnxI1vyt20cFurzA/f1mkOXg4Keih", + "TdEhMn9K6mJcWzbNBQ3QSeMF+pooA29yn3q67SEw8XMi3Kk2A12YWRfTMp8ig9MCEwJLzGrd98S80k71", + "Ne39mTRfWJ7bSCx2yb+KKC2U3XKtVim4J7YW9v3ptxUktwcOW7EEFlhksKKOndsJLKsbWC5YtPk1cuWz", + "U7SJayyVWOHmTeLCkm3QlAo966HkukPmF17XMp3VaylDk4ROZ9YJENMJxOopv5A5jHLnAUZZFAwXWBEb", + "ovgU837P9CJbL/CcCA+p0b9St/6AoNX1qpJjXitv13dvf+4RFvG4cJ40y6T2yWdWmAz9V3J5H/7UPcF8", + "VurEgyaB8RP23wSTowK683/t/GDBO//Xzg84ySgj/2v3MMGKSLV5b8QyeKib7qEVmCdMfFp/odVFW2JN", + "WwpP/TzUKmU6+NrbUGYFdtYZu4tEms9GfZ9fyKtj9baS8B6M7rUm9dCIJSeuLIHZUi4celUNkvjBDuN5", + "hbYe3HRW6Z5KGAHIWE8wYJgLB4CKLYI0dXRm2QSbAsT2GrtA8VZL04B7/09lHTCTvpV9oFjXryaCNiYC", + "f7lWWgnsVtyrncD08UiO8ILYQqsNj74iXz2Ab8VSpId8VXE2l9hXMy4VPHp6MAj2KqEFxfnXRksnYXkg", + "V14fjnRPjruwkFCBHyoA2SzjB3IZunE8uA5s+314oecwHdNpznPpJzCnWEUzIm1yf0KqDPipaefl9dyo", + "n3/BVDp4yKvjwdXvr3R/T4aB+oYa5m38/utkfvdWW5nfvq9lfgN8bAEQbJmiritht9mQj+Ggj9uScQUh", + "ejlPJDSukC6C3mlFpVQXEGgQB0P2v7X+8asiOP3tO5dpnQ8GO8/gd8Lmv33nkq3ZqSMVwpSgtuLo4S/H", + "EGwxhXh6KEpa4jrUx4HSXCpDeq76yL+dglTGm7TXkBwVftWQWmlI3nKt1pDsXtyvilStYPTgOpKjt9CC", + "29IDf04t6U/uRa1ocDKfTGhECYM6UGD8k0thw0aT++pAvSNuAbNhC17MYUUSaa1GFlxrjYRe1uB/8HjT", + "k7Lo4ENrj67c/9PMv+SZrZ9t9bVSWmhW2L40ehg87O318IraUyYxoxEtL12mhe5AATHjOUlzk6xafFli", + "D/bRxcXPLjFVS/zClaNT3NWgc3WKh8yvQddHr8rifuYF14KWyElsE9khXddWdYsJjhPKCETyExnKIa1W", + "jnzUY/H5hcpwWcyHduC2OZa24vPjCZWPxgoeRHw7qRTS56WO71fMLE6LE+Hg1DwpfmUZUIDxhMSnLZwr", + "3rOp7lszbvAPwxCwZwmOAAFWv2bACS26iEEj9ZsCyBDBk4QIAzqZ5cqVSh2yYnCUKV6UVbalsC5186Oc", + "KZpcdk0gHSAHSYTZwiKvDVmlM6wUSTMFCACAbgEjFCQzI67ViNWDpjyX8BYk6/tdIpxc44UcMosZYD6H", + "wuKCRAafNUn66EcOcC2mVrLHeE2h0m/kkF3SOCEji7ZyiahEcsaFIozEKOVzIqv9EiwSSgRM4gjrlZMo", + "xQuAPTQIsGZ9eEYMtGAF04Xrf2MWUyh5qXsupnwwZBjtDAYoJZhJi9Ag8QQuHNsGgkFUBvQtwmhv8NJ+", + "Vds3gOZ2y7+hT5MQZM4jPE4WiGgqNqWkN2EDU1uC1tQ019s3oUKa/SpMhra2YGVjqXQVVeMuylmJQQHm", + "85wVkBF6u1QuGMzTOtYIFcU1aGF3xiTCej0Zr/YDgKc8inIRuiD1Vnu1kP8dBUdveuewVGGEhwS08IjE", + "sOeMqxmcaQ5HafPbBqoqierPcdEEDwkXCCOPrksjAYlyYI0bABB6WRb2ZK5Q9+Xmt+7s6ONrGYE7/gai", + "86ncT0BEfDKpHMD1V5M5wKsyq5ZJ+M96To9cRWefxcUUTxmXikaOGTr8ZV9p/qoQtlIIV69skJonXFw1", + "h9P+wMVVWw3MxRo+LUXMn+EXaNvXwwOI98c38YOB2SgrmmgeXEmr01dxSkHookoWUbYo4WyqT1Fp6H5w", + "S7yv1W0YuEZ9mQrjPy7AtbQSMrI/mqLQXtAyWO0j2+pj8yLd+wP4d37hCtE0S0hKoGh0zxCb3uwSiG28", + "sEBxBRzZ7XilPlU+aoDRBaVx6XedOAR05TZsA6T35e0KMtWET9fDfRadO2zLAN7nkL2TBoj/0nhzLlHB", + "g7VAa4proOsZjWaA/Ql6q27fQIPiLLssYM83D9BrOMg++jt0vmFKamhakzwhBtJznqaXB8tlkd+fnsJH", + "BvbTFEC+PECuFHJxf0j9lo/lqWeRYKnQLxahdKNQxmFHLxXW+mYxv02L8lnC0g9ZCPGTkWvbIJ2gSw/8", + "87IBmc7x25/59NGEsW5zMREzF8WRVR2BNgmLO01xEzQJ435uDwYhjPuWGKRmGPcMQbo0mJ/5tChkUiFl", + "nGVtydcOE6h4nqYraBhteICGUsU8V3+RKiZCwMeWupuIG23gyBaxw1eaUC18pTvYm0B+weggU1kguFSa", + "qXa6HcLytHPwq/3XPE073Y4dj1eR4BbC/Ros13qDy1Esemc8wNavYvltoFirzN7DYq3dHFadbpbI35oX", + "/vTeQmeze0QyBPmgZsT9kkRQb7xVgw/jBaYsjOzlQ4wMgGdRlHBJKg6epwNbZw1dNZmx2VDk1rinhxfn", + "rs5Xm6CQc/vpufvyC9C918WKuDEjN90HDxpZHsFTTsGXS7OZcFHHOlsXTfLFE9Ln25KlqbahkK+0eXsr", + "YyvC1HrCMouwH8Sm7iPOFU+xohHUHItmnEuP7AtgclMd0BqPC8oE04rRcm1Q/qUm1Utrhr60asSBNZkh", + "7D+yffThcxvKH/7CPSq/+MGzChQcv+tEf6jLIRFGY0HJBGU4l0RLdXlKULSINFc0ReYIjmYowpnKBYH6", + "mQSllNE0T33Eeb1jcwzoOJfb6WUXjXOFEiymoJWZhy7YJuJpSlhMwD43ZDOC51SrlAIlWBEWLXqSQN3t", + "OUHXXFwlHMdgYshiDJ4eqNspiKZAgO9PicIxVhgEnUt94kcmL+iyKMVt1HpGbkpqiIdM5OxbU0tEN3vp", + "BnqJCIDlUzkrSrZGOCYsCoLIn3/ZbOzz26LPiapP9JEig+7ESx8zVMi3ubrhfBlRRE8WZaEFm18h9Mpm", + "FbaaUOHI6N/zSJu5ujk+koOpWOJVp/jL8CwVRPfFeJce333EBYpz0513KoHM/6w+oYKh+MFWkKxptvGu", + "jqGiNmWxzLfieVt/uD9P7mDL+0I4YbdRsW+qglZO+ktguXZV78RzH8mIaW1Jvk3u8Viwi+h6NPGJC4/L", + "PRVjawV3rODbPndSAoP2xdlXtl1n2zbg4a5s29lml1z6HiOnrAcxomEObs24jazamg7+TbNRarPzWOaj", + "s8jSc/HgIIKONWZ4kXAc/xmChFf4jyIuhEGUAIyKpwR87FkN/fQAsM2V5RW7Llvz/enpZhOXEGoljxDq", + "CXMILyVHf5bGywbcN3MiBI0douTR6bEN16USiZz10ZuUKqQ4uiIkKzNaIKuwr+fnsDVqw66DaHQ7hCmx", + "yDhlau0oylfvZzDlD9xCjHyBoqRF0v/qDm/tDgfL/tNjZ8BlIGfDTGC1ZqqwWlvhl7IJF6mRy/CY57p1", + "zYP0Mun9NJVyJzQhciEVSU1U4iRP4LhBVRZbedt+Z3a5CzG5+uSYdLmMiJRKSTmTQ2ZzRTIidN/6c92+", + "F2AVdAgoXPDXM8Mkv4zgPT0YE6+GVdOqAQoSVPTtHHS2cJZtxVjhhgAxO7xPGNIPEI2H5CId84RGKKHs", + "SqKNhF4Z9QTNJUr0H5srw/lG8N3nrit+95OlV/qETXiwaqOh2YKY/1RZXZatOcfkk2Nrr4l/WBz/gY0O", + "s7X1lcsFwUkPKoE7LByUK5rQD4bV6UaoVDQyKUe4WLv3pwVT7Q/ZKVFCv4MhtS1JDKIBaJdbmeDR1jAf", + "DHajjAKg2i6BwQHDa36cQo9HZ+9MGipJuVh0h0z/Axq+ODwz3t0JttYEb6C2ZDk62XqzJsD5HJbp3zhC", + "0ExwJXpBcMO/ugRvjzHSeIZkwxHl2SpViWd/+hBWK8F9tSs8TbsCgDwVs9mYChyBUCxnuYr5NQvbEOY8", + "yVP9D/PHyTqoMIWj2Xt49YuRds1w1nbjJvgkDqWdU0xMVdlHcXqYBXuqMat64dwUQIipRAMGb4FD9Wek", + "7s9vvvfX8Qt0d9oVdRWbv5iz9dA3nx2DQ9jw1+OpHHNDaW4miq+2Pl1j2mx9+j7h0ZW0UCy+2VDrbQBZ", + "rn8sIaatixDEBMgMRRbCyABlEdkdspoB0iD+SISRIiKlDCdbMGfTCIBlOysWnnMKCdoR5Kn0JI0BMykB", + "RGyAv9OzAUOVa8Dz6Epb085/x3dGKo7GJOIpcQDimyHV7W+Yqh+4qKKBfyl88cJbf4AExBTs7WsA0Jt7", + "/CRA9FN8A6HScW4dym5EG695+aMxBXUR7M2wszuQw04XDTs76bCjd+AIgwkVK7SPUspyRWQfHRv7FqTg", + "PhsgSSLOYulwzJ0Fb3cgmxJyDVk2ZHc+g+8eUuyxVAVL+dZ2EmIP+j2kv4ekHbThHzh7JuMuHLoY8VwZ", + "c789V/atmCgwj2w+uK/WOyNfdfs2nPxv9vhWeBTssmaX3tYbzp7lckaaTW4/m9pAuRoDPrYr6ytn6O98", + "LLuIkWtjDRdS9Zf4nv76zHTwENj9uqvb4PbbuX8F7W8B2l+uVRis0QRY6ivZUYdBbCQ3GRcKUBxtrr2h", + "IdAkADkCKv69OToZskizIgMtKEjKgTtZiHFzCx/+7Ry9OnrbRcdQYhb9mI83++gNSxau0L/x0QyZkcQM", + "84owQ2NDtSQOXc9m7EA99xksrjt4pJrt5mQEPCtur1yQeLczIzgGieSPzs/cdBZAHX77sz5AAPxrviy2", + "vbNS+Oi8JUoseocTRcRys6c2T4oVmBn2knYQdFZwM8CXukPpkNfKPo1sYKAxdnc6AaSMj1/rKNx/aeKH", + "8ZKZOBFTwW6cqydbChUOYsEcQyzQv66LSgRNWcKWl61UMKDLpsjvL8jkvpJ3VbDl/11PF8z0yTqasso+", + "aSIuKpis9fS65OCZgUO2jqoIZziiatFFOEnsHWVvgiIipVeIv2NB8FXMr1l/yN4WtVNsQi86OnvXdY5a", + "FFN5ZVqwvtg+ejMnQubjYnAIDprxGsOak3jIFEcRTqI80eIGmUxIBLm4UBJFNvhyi6F07vHslJ0E67d4", + "Ue35kysbF6YJ2L2SLOoUt2W2ekuQKME0bQYft4IaBBxCqMFYN8oZomyS2JCqSHApkW2qRxI6pePEBgjJ", + "PrqYESRxSoYsSzBjRKBcmqh4PfReJoiUuUnw1g0ASK+hqC4qgQUzwZUNTUg4F9JEE2gKf3+KpCLZCjJ7", + "a1o+hTnfk2xrGrc9PZKRujaGZlOIfQXpDTGUYhZc01GeuADGBw1FNwN6bCnxqRz8C0GnUyL0qcCGyZpw", + "PHOs3XKaQ1/JWG4sIXlevNWuhGTRqpeV6GXsrQSGG5VY23HndlF/gc6vaCN2oH10uyzin/RHLfuuZquG", + "B2EffeIs/yyV+c+9JMG2BqySwp+aOckbeeWoVhJt18Nqtc6svc9M19b4WY8Gm/WU0bJwJX22SeH98ghh", + "8LAoDw9dZO1p01YF7aqimzak/K9H0/8iKPB+YPQfGeXkDjD6X1TePeCcPx7+SfCgPlYefcX37OrX/umR", + "8O8rfd7A4QMcW1P6vOF6Nnh1paL03r7TTk2yLf6ZJHgb73gL+d0t+1etv4XK4C3WOhe0JniSZmrhAtqs", + "r7IMOpP0A+k3OIKLuNX7cwXfIaTz85GHo9PGgM4/Z7n5R4kZtaUDqUQnx4E67k8MY9A/c5WLZUvfOj0s", + "ohmdk2aje/UE2yXKBOllPAPnSmwWzK6Hu8sUFv3pB2Sbt5ir9l9QexKg+kmMYipIpJKFqQOqOYLp4xuJ", + "BNeaADznYtEcJWKOyA+Cp4d2NmvuQ3umrDGsjDNMF70YK9ybO26zwoT2CdGdLp5SMzxEGXr9PdogN0qY", + "ChdoojUfRCfFkpoC/hJoctMf8PagwbJJP5DRdNxmlCtqlbyxtWBQlEvFU7f3J8doA2qfTQnTe6FF/QlI", + "spngcxqTuDLGzpwnZlW3Gxb0tnZXLVQUheuccmEG9ygyTJsLafqBZlW2UITEjCnDMLi1VUGqZ8ok8ev+", + "MGUuAMfukRvF1yvMan4bTtnRlAh1OO0iKs4NxPPm12vuKV9zfjKUu9Mqt50Lz1ltvG6XH9Uybek+Cj8U", + "uXMPa7Z+/+Wk9FD5JLN5rOl8XiikTWbzL4sEBw93Pzy0ufz9E04BfU2c8u2ZyqEB3WKIYH6GmO6YzEnC", + "sxTqocO7nW4nF0nnoDNTKjvY2oLY7xmX6mDv5fPdzsffPv7/AQAA///BBhRZHPQBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/scopes/scopes.go b/lib/scopes/scopes.go index 4773437a0..9d078c1fb 100644 --- a/lib/scopes/scopes.go +++ b/lib/scopes/scopes.go @@ -229,10 +229,11 @@ var RouteScopes = map[string]Scope{ "POST /resources/memory/reclaim": ResourceWrite, // Images - "GET /images": ImageRead, - "POST /images": ImageWrite, - "DELETE /images/{name}": ImageDelete, - "GET /images/{name}": ImageRead, + "GET /images": ImageRead, + "POST /images": ImageWrite, + "POST /images/{name}/tag": ImageWrite, + "DELETE /images/{name}": ImageDelete, + "GET /images/{name}": ImageRead, // Pushes (outbound image pushes to remote registries) "POST /pushes": ImageWrite, diff --git a/openapi.yaml b/openapi.yaml index a0b640ef4..4d5830c2e 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1290,6 +1290,15 @@ components: tags: $ref: "#/components/schemas/Tags" + TagImageRequest: + type: object + required: [target] + properties: + target: + type: string + description: Target OCI image reference with a tag. The local tag points to the source image without pulling it again. + example: docker.io/library/nginx:stable + Image: type: object required: [name, digest, status, created_at] @@ -2640,6 +2649,63 @@ paths: schema: $ref: "#/components/schemas/Error" + /images/{name}/tag: + post: + summary: Create or update a local image tag + operationId: tagImage + security: + - bearerAuth: [] + parameters: + - name: name + in: path + required: true + schema: + type: string + description: URL-encoded source image name or digest + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/TagImageRequest" + responses: + 200: + description: Image tag created + content: + application/json: + schema: + $ref: "#/components/schemas/Image" + 400: + description: Invalid source or target image reference + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 404: + description: Source image not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 409: + description: Source image is not ready + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 401: + description: Unauthorized + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /images/{name}: get: summary: Get image details diff --git a/stainless.yaml b/stainless.yaml index 0d95928b2..dbd40fbc9 100644 --- a/stainless.yaml +++ b/stainless.yaml @@ -97,11 +97,13 @@ resources: # [reference]: https://www.stainless.com/docs/reference/config#model models: image: "#/components/schemas/Image" + tag_image_request: "#/components/schemas/TagImageRequest" methods: list: get /images create: post /images get: get /images/{name} delete: delete /images/{name} + tag: post /images/{name}/tag instances: models: From e177d18a4cc5883652c536331fa72ad41760faa0 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:29:39 +0000 Subject: [PATCH 02/13] Fix image tagging review findings --- cmd/api/api/images.go | 2 +- lib/images/manager.go | 19 ++- lib/images/tag.go | 4 + lib/images/tag_test.go | 81 ++++++++++- lib/images/testseed.go | 111 --------------- lib/images/testutil/testutil.go | 94 ++++++++++++- lib/middleware/resolve.go | 8 +- lib/middleware/resolve_test.go | 31 ++++- lib/oapi/oapi.go | 233 ++++++++++++++++++++------------ 9 files changed, 362 insertions(+), 221 deletions(-) delete mode 100644 lib/images/testseed.go diff --git a/cmd/api/api/images.go b/cmd/api/api/images.go index 812134ab0..7ae20ebdc 100644 --- a/cmd/api/api/images.go +++ b/cmd/api/api/images.go @@ -95,7 +95,7 @@ func (s *ApiService) CreateImage(ctx context.Context, request oapi.CreateImageRe } // TagImage handles POST /images/{name}/tag. -// Note: ResolveResource skips POSTs under /images/{name}, so the source is +// Note: ResolveResource skips POST /images/{name}/tag, so the source is // resolved by ImageManager.TagImage and a missing source gets the specific // 404 body below. func (s *ApiService) TagImage(ctx context.Context, request oapi.TagImageRequestObject) (oapi.TagImageResponseObject, error) { diff --git a/lib/images/manager.go b/lib/images/manager.go index 06e6cbbe0..aa66abc4f 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -638,27 +638,34 @@ func (m *manager) claimImageTags(ref *ResolvedRef, meta *imageMetadata) { if requestedTag == "" { requestedTag = ref.Tag() } - m.claimTag(ref.Repository(), ref.DigestHex(), requestedTag, meta.PreviousTagDigest, meta.TagGeneration, meta.RequestedTag == "") + claimed := m.claimTag(ref.Repository(), ref.DigestHex(), requestedTag, meta.PreviousTagDigest, meta.TagGeneration, meta.RequestedTag == "") for _, claim := range meta.TagClaims { - m.claimTag(claim.Repository, ref.DigestHex(), claim.Tag, claim.PreviousTagDigest, claim.TagGeneration, false) + claimed = m.claimTag(claim.Repository, ref.DigestHex(), claim.Tag, claim.PreviousTagDigest, claim.TagGeneration, false) || claimed + } + if !claimed { + if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), ref.DigestHex(), true); err != nil { + slog.Warn("failed to collect stale image", "repository", ref.Repository(), "digest", ref.DigestHex(), "error", err) + } } } -func (m *manager) claimTag(repository, digestHex, tag, previous string, generation uint64, allowMissing bool) { +func (m *manager) claimTag(repository, digestHex, tag, previous string, generation uint64, allowMissing bool) bool { if tag == "" || m.tagGenerations[tagGenerationKey(repository, tag)] != generation { - return + return false } current, err := resolveTag(m.paths, repository, tag) if err != nil { if !allowMissing || !errors.Is(err, ErrNotFound) { - return + return false } } else if current != digestHex && current != previous { - return + return false } if err := createTagSymlink(m.paths, repository, tag, digestHex); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to create tag symlink: %v\n", err) + return false } + return true } func phaseStatus(err error) string { diff --git a/lib/images/tag.go b/lib/images/tag.go index 181176952..9b092f595 100644 --- a/lib/images/tag.go +++ b/lib/images/tag.go @@ -49,6 +49,10 @@ func (m *manager) TagImage(ctx context.Context, source, target string) (*Image, if err := createTagSymlink(m.paths, targetRef.Repository(), targetRef.Tag(), digestHex); err != nil { return nil, fmt.Errorf("create image tag: %w", err) } + setReferenceTags(meta, targetRef.String(), nil) + if err := writeMetadata(m.paths, targetRef.Repository(), digestHex, meta); err != nil { + return nil, fmt.Errorf("write tagged image metadata: %w", err) + } // Unlike updateExistingReference, which bumps the generation before // installing the symlink, the bump happens after the install here so a // failed tag call cannot invalidate a pending pull's claim on the target diff --git a/lib/images/tag_test.go b/lib/images/tag_test.go index 9c6462085..c91f13f5b 100644 --- a/lib/images/tag_test.go +++ b/lib/images/tag_test.go @@ -8,17 +8,45 @@ import ( "time" "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/tags" "github.com/stretchr/testify/require" ) func seedContent(t *testing.T, p *paths.Paths, repository, tag, digestHex string) { t.Helper() - require.NoError(t, SeedTestImage(p, TestSeed{Repository: repository, Tag: tag, DigestHex: digestHex, Content: true})) + seedImage(t, p, repository, tag, digestHex, true) } func seedLegacy(t *testing.T, p *paths.Paths, repository, tag, digestHex string) { t.Helper() - require.NoError(t, SeedTestImage(p, TestSeed{Repository: repository, Tag: tag, DigestHex: digestHex})) + seedImage(t, p, repository, tag, digestHex, false) +} + +func seedImage(t *testing.T, p *paths.Paths, repository, tag, digestHex string, content bool) { + t.Helper() + dir := p.ImageDigestDir(repository, digestHex) + disk := p.ImageDigestPath(repository, digestHex) + metadata := p.ImageMetadata(repository, digestHex) + linkPath := p.ImageTagSymlink(repository, tag) + target := digestHex + if content { + dir = p.ImageContentDir(digestHex) + disk = p.ImageContentPath(digestHex) + metadata = p.ImageContentMetadata(digestHex) + linkPath = p.ImageRepositoryTagSymlink(repository, tag) + rel, err := filepath.Rel(filepath.Dir(linkPath), p.ImageContentDir(digestHex)) + require.NoError(t, err) + target = rel + } + + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(disk, []byte("rootfs!"), 0o644)) + require.NoError(t, writeMetadataFile(metadata, &imageMetadata{ + Name: repository + ":" + tag, Digest: "sha256:" + digestHex, + Status: StatusReady, SizeBytes: int64(len("rootfs!")), CreatedAt: time.Now().UTC(), + })) + require.NoError(t, os.MkdirAll(filepath.Dir(linkPath), 0o755)) + require.NoError(t, os.Symlink(target, linkPath)) } func newTagTestCase(t *testing.T) (*paths.Paths, *manager, string) { @@ -61,6 +89,32 @@ func TestTagImageAliasesReadyImage(t *testing.T) { } } +func TestTagImagePersistsTargetReference(t *testing.T) { + p, m, repository := newTagTestCase(t) + digest := "f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1" + seedContent(t, p, repository, "latest", digest) + + meta, err := readContentMetadata(p, digest) + require.NoError(t, err) + meta.Tags = tags.Tags{"source": "tag"} + meta.References = map[string]tags.Tags{ + repository + ":latest": {"source": "tag"}, + } + require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digest), meta)) + + target := repository + ":stable" + tagImage(t, m, repository+":latest", target, digest) + + image, err := m.GetImage(context.Background(), target) + require.NoError(t, err) + require.Empty(t, image.Tags) + + meta, err = readContentMetadata(p, digest) + require.NoError(t, err) + _, ok := meta.References[target] + require.True(t, ok, "tagging must persist an explicit target reference") +} + func TestTagImageSameRepositoryDeletesContent(t *testing.T) { p, m, repository := newTagTestCase(t) digest := "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" @@ -109,6 +163,29 @@ func TestTagImageCrossRepository(t *testing.T) { require.True(t, os.IsNotExist(err), "content must be removed once unreferenced") } +func TestStaleTagClaimCollectsContent(t *testing.T) { + p, m, repository := newTagTestCase(t) + digest := "f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2" + seedContent(t, p, repository, "latest", digest) + linkPath := p.ImageRepositoryTagSymlink(repository, "latest") + require.NoError(t, os.Remove(linkPath)) + require.NoError(t, os.Symlink("replaced", linkPath)) + + normalized, err := ParseNormalizedRef(repository + ":latest") + require.NoError(t, err) + m.tagGenerations[repository+":latest"] = 2 + ref := NewResolvedRef(normalized, "sha256:"+digest) + meta, err := readContentMetadata(p, digest) + require.NoError(t, err) + meta.RequestedTag = "latest" + meta.TagGeneration = 1 + + m.claimImageTags(ref, meta) + + _, err = os.Stat(p.ImageContentDir(digest)) + require.ErrorIs(t, err, os.ErrNotExist) +} + func TestTagImageRejectsNotReady(t *testing.T) { p, m, repository := newTagTestCase(t) digest := "d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4" diff --git a/lib/images/testseed.go b/lib/images/testseed.go deleted file mode 100644 index 565a3a65d..000000000 --- a/lib/images/testseed.go +++ /dev/null @@ -1,111 +0,0 @@ -package images - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "time" - - "github.com/kernel/hypeman/lib/paths" -) - -const seedImageContent = "rootfs!" - -// TestSeed describes a ready image to write directly to disk, bypassing pulls, -// builds, and conversion. -type TestSeed struct { - Repository string - DigestHex string - // Tag optionally creates a tag symlink pointing at the image. - Tag string - // Name overrides the metadata "name" field; it defaults to - // repository:tag, or repository@sha256:digest when Tag is empty. - Name string - // Tags records resource tags in the metadata. - Tags map[string]string - // Content writes the image into the shared content layout instead of - // the legacy per-repository digest layout. - Content bool -} - -// seedImagePaths holds the on-disk locations for one seeded image, in either -// the shared content layout or the legacy per-repository digest layout. -type seedImagePaths struct { - dir string - disk string - metadata string -} - -func seedLayout(p *paths.Paths, s TestSeed) seedImagePaths { - if s.Content { - return seedImagePaths{ - dir: p.ImageContentDir(s.DigestHex), - disk: p.ImageContentPath(s.DigestHex), - metadata: p.ImageContentMetadata(s.DigestHex), - } - } - return seedImagePaths{ - dir: p.ImageDigestDir(s.Repository, s.DigestHex), - disk: p.ImageDigestPath(s.Repository, s.DigestHex), - metadata: p.ImageMetadata(s.Repository, s.DigestHex), - } -} - -// SeedTestImage writes a ready image to disk per s. It is exported for -// lib/images/testutil so test seeds marshal imageMetadata directly instead of -// re-declaring the schema; production code does not use it. -func SeedTestImage(p *paths.Paths, s TestSeed) error { - layout := seedLayout(p, s) - if err := os.MkdirAll(layout.dir, 0o755); err != nil { - return fmt.Errorf("create image dir: %w", err) - } - if err := os.WriteFile(layout.disk, []byte(seedImageContent), 0o644); err != nil { - return fmt.Errorf("write disk image: %w", err) - } - - name := s.Name - if name == "" && s.Tag != "" { - name = s.Repository + ":" + s.Tag - } else if name == "" { - name = s.Repository + "@sha256:" + s.DigestHex - } - data, err := json.Marshal(imageMetadata{ - Name: name, - Digest: "sha256:" + s.DigestHex, - Status: StatusReady, - SizeBytes: int64(len(seedImageContent)), - Tags: s.Tags, - CreatedAt: time.Now().UTC(), - }) - if err != nil { - return fmt.Errorf("marshal metadata: %w", err) - } - if err := os.WriteFile(layout.metadata, data, 0o644); err != nil { - return fmt.Errorf("write metadata: %w", err) - } - - return seedTagSymlink(p, s) -} - -// seedTagSymlink creates the tag symlink when s.Tag is set, pointing at the -// shared content directory or the bare digest per s.Content. -func seedTagSymlink(p *paths.Paths, s TestSeed) error { - if s.Tag == "" { - return nil - } - linkPath := p.ImageTagSymlink(s.Repository, s.Tag) - target := s.DigestHex - if s.Content { - linkPath = p.ImageRepositoryTagSymlink(s.Repository, s.Tag) - rel, err := filepath.Rel(filepath.Dir(linkPath), p.ImageContentDir(s.DigestHex)) - if err != nil { - return fmt.Errorf("rel content symlink target: %w", err) - } - target = rel - } - if err := os.MkdirAll(filepath.Dir(linkPath), 0o755); err != nil { - return fmt.Errorf("create tag dir: %w", err) - } - return os.Symlink(target, linkPath) -} diff --git a/lib/images/testutil/testutil.go b/lib/images/testutil/testutil.go index e9e11a5fb..aa28cd1e4 100644 --- a/lib/images/testutil/testutil.go +++ b/lib/images/testutil/testutil.go @@ -2,18 +2,106 @@ package testutil import ( + "encoding/json" + "fmt" + "os" + "path/filepath" "testing" + "time" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/require" ) -// Seed describes a ready image to seed; see images.TestSeed for the fields. -type Seed = images.TestSeed +const seedImageContent = "rootfs!" + +// Seed describes a ready image to seed directly to disk, bypassing pulls, +// builds, and conversion. +type Seed struct { + Repository string + DigestHex string + // Tag optionally creates a tag symlink pointing at the image. + Tag string + // Name overrides the metadata "name" field; it defaults to + // repository:tag, or repository@sha256:digest when Tag is empty. + Name string + // Tags records resource tags in the metadata. + Tags map[string]string + // Content writes the image into the shared content layout instead of + // the legacy per-repository digest layout. + Content bool +} + +type imageMetadata struct { + Name string `json:"name"` + Digest string `json:"digest"` + Status string `json:"status"` + SizeBytes int64 `json:"size_bytes"` + Tags map[string]string `json:"tags,omitempty"` + CreatedAt time.Time `json:"created_at"` +} // SeedReadyImage writes a ready image to disk per s. func SeedReadyImage(t testing.TB, p *paths.Paths, s Seed) { t.Helper() - require.NoError(t, images.SeedTestImage(p, s)) + require.NoError(t, seedImage(p, s)) +} + +func seedImage(p *paths.Paths, s Seed) error { + dir := p.ImageDigestDir(s.Repository, s.DigestHex) + disk := p.ImageDigestPath(s.Repository, s.DigestHex) + metadata := p.ImageMetadata(s.Repository, s.DigestHex) + linkPath := p.ImageTagSymlink(s.Repository, s.Tag) + target := s.DigestHex + if s.Content { + dir = p.ImageContentDir(s.DigestHex) + disk = p.ImageContentPath(s.DigestHex) + metadata = p.ImageContentMetadata(s.DigestHex) + linkPath = p.ImageRepositoryTagSymlink(s.Repository, s.Tag) + rel, err := filepath.Rel(filepath.Dir(linkPath), p.ImageContentDir(s.DigestHex)) + if err != nil { + return fmt.Errorf("rel content symlink target: %w", err) + } + target = rel + } + + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create image dir: %w", err) + } + if err := os.WriteFile(disk, []byte(seedImageContent), 0o644); err != nil { + return fmt.Errorf("write disk image: %w", err) + } + + name := s.Name + if name == "" && s.Tag != "" { + name = s.Repository + ":" + s.Tag + } else if name == "" { + name = s.Repository + "@sha256:" + s.DigestHex + } + data, err := json.Marshal(imageMetadata{ + Name: name, + Digest: "sha256:" + s.DigestHex, + Status: images.StatusReady, + SizeBytes: int64(len(seedImageContent)), + Tags: s.Tags, + CreatedAt: time.Now().UTC(), + }) + if err != nil { + return fmt.Errorf("marshal metadata: %w", err) + } + if err := os.WriteFile(metadata, data, 0o644); err != nil { + return fmt.Errorf("write metadata: %w", err) + } + + if s.Tag == "" { + return nil + } + if err := os.MkdirAll(filepath.Dir(linkPath), 0o755); err != nil { + return fmt.Errorf("create tag dir: %w", err) + } + if err := os.Symlink(target, linkPath); err != nil { + return fmt.Errorf("create tag symlink: %w", err) + } + return nil } diff --git a/lib/middleware/resolve.go b/lib/middleware/resolve.go index 80209f627..5eaa75d54 100644 --- a/lib/middleware/resolve.go +++ b/lib/middleware/resolve.go @@ -83,10 +83,10 @@ func ResolveResource(resolvers Resolvers, errResponder ErrorResponder) func(http resourceType = "ingress" paramName = "id" case strings.HasPrefix(path, "/images/"): - // POSTs under /images/{name} (currently only /tag) resolve the - // source in the handler, which returns a specific 404 body for - // a missing source; don't intercept with the generic one. - if r.Method == http.MethodPost { + // The tag route resolves the source in the handler, which returns a + // specific 404 body for a missing source; don't intercept it with + // the generic one. + if r.Method == http.MethodPost && strings.HasSuffix(path, "/tag") { next.ServeHTTP(w, r) return } diff --git a/lib/middleware/resolve_test.go b/lib/middleware/resolve_test.go index 810818465..607d8f5de 100644 --- a/lib/middleware/resolve_test.go +++ b/lib/middleware/resolve_test.go @@ -128,7 +128,7 @@ func TestResolveResource_ResolvesBuilderByID(t *testing.T) { "Builder resolver was not invoked with the path ID") } -func TestResolveResource_SkipsImagePosts(t *testing.T) { +func TestResolveResource_SkipsOnlyImageTagPosts(t *testing.T) { resolver := &mockResolver{} middleware := ResolveResource(Resolvers{Image: resolver}, func(w http.ResponseWriter, err error, lookup string) { @@ -139,12 +139,29 @@ func TestResolveResource_SkipsImagePosts(t *testing.T) { r.With(middleware).Post("/images/{name}/tag", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) + r.With(middleware).Post("/images/{name}", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) - req := httptest.NewRequest(http.MethodPost, "/images/docker.io%2Flibrary%2Falpine:latest/tag", nil) - w := httptest.NewRecorder() - r.ServeHTTP(w, req) + t.Run("tag route bypasses resolver", func(t *testing.T) { + resolver.receivedName = "" + req := httptest.NewRequest(http.MethodPost, "/images/docker.io%2Flibrary%2Falpine:latest/tag", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) - require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) - assert.Empty(t, resolver.receivedName, - "image POSTs must not be intercepted by the resolver") + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + assert.Empty(t, resolver.receivedName, + "the tag route must not be intercepted by the resolver") + }) + + t.Run("other image post resolves", func(t *testing.T) { + resolver.receivedName = "" + req := httptest.NewRequest(http.MethodPost, "/images/alpine:latest", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + assert.Equal(t, "alpine:latest", resolver.receivedName, + "only the tag route should bypass image resolution") + }) } diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index bda3c0909..aaf55b62c 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -2043,6 +2043,12 @@ type ListInstancesParams struct { Tags *Tags `json:"tags,omitempty"` } +// DeleteInstanceParams defines parameters for DeleteInstance. +type DeleteInstanceParams struct { + // GracefulShutdown Whether to attempt graceful guest shutdown before deleting the instance + GracefulShutdown *bool `form:"graceful_shutdown,omitempty" json:"graceful_shutdown,omitempty"` +} + // GetInstanceLogsParams defines parameters for GetInstanceLogs. type GetInstanceLogsParams struct { // Tail Number of lines to return from end @@ -2352,7 +2358,7 @@ type ClientInterface interface { CreateInstance(ctx context.Context, body CreateInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) // DeleteInstance request - DeleteInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + DeleteInstance(ctx context.Context, id string, params *DeleteInstanceParams, reqEditors ...RequestEditorFn) (*http.Response, error) // GetInstance request GetInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -2889,8 +2895,8 @@ func (c *Client) CreateInstance(ctx context.Context, body CreateInstanceJSONRequ return c.Client.Do(req) } -func (c *Client) DeleteInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewDeleteInstanceRequest(c.Server, id) +func (c *Client) DeleteInstance(ctx context.Context, id string, params *DeleteInstanceParams, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewDeleteInstanceRequest(c.Server, id, params) if err != nil { return nil, err } @@ -4551,7 +4557,7 @@ func NewCreateInstanceRequestWithBody(server string, contentType string, body io } // NewDeleteInstanceRequest generates requests for DeleteInstance -func NewDeleteInstanceRequest(server string, id string) (*http.Request, error) { +func NewDeleteInstanceRequest(server string, id string, params *DeleteInstanceParams) (*http.Request, error) { var err error var pathParam0 string @@ -4576,6 +4582,28 @@ func NewDeleteInstanceRequest(server string, id string) (*http.Request, error) { return nil, err } + if params != nil { + queryValues := queryURL.Query() + + if params.GracefulShutdown != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "graceful_shutdown", runtime.ParamLocationQuery, *params.GracefulShutdown); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + queryURL.RawQuery = queryValues.Encode() + } + req, err := http.NewRequest("DELETE", queryURL.String(), nil) if err != nil { return nil, err @@ -6279,7 +6307,7 @@ type ClientWithResponsesInterface interface { CreateInstanceWithResponse(ctx context.Context, body CreateInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateInstanceResponse, error) // DeleteInstanceWithResponse request - DeleteInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*DeleteInstanceResponse, error) + DeleteInstanceWithResponse(ctx context.Context, id string, params *DeleteInstanceParams, reqEditors ...RequestEditorFn) (*DeleteInstanceResponse, error) // GetInstanceWithResponse request GetInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetInstanceResponse, error) @@ -6928,6 +6956,7 @@ type TagImageResponse struct { HTTPResponse *http.Response JSON200 *Image JSON400 *Error + JSON401 *Error JSON404 *Error JSON409 *Error JSON500 *Error @@ -8269,8 +8298,8 @@ func (c *ClientWithResponses) CreateInstanceWithResponse(ctx context.Context, bo } // DeleteInstanceWithResponse request returning *DeleteInstanceResponse -func (c *ClientWithResponses) DeleteInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*DeleteInstanceResponse, error) { - rsp, err := c.DeleteInstance(ctx, id, reqEditors...) +func (c *ClientWithResponses) DeleteInstanceWithResponse(ctx context.Context, id string, params *DeleteInstanceParams, reqEditors ...RequestEditorFn) (*DeleteInstanceResponse, error) { + rsp, err := c.DeleteInstance(ctx, id, params, reqEditors...) if err != nil { return nil, err } @@ -9624,6 +9653,13 @@ func ParseTagImageResponse(rsp *http.Response) (*TagImageResponse, error) { } response.JSON400 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON401 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { @@ -11623,7 +11659,7 @@ type ServerInterface interface { CreateInstance(w http.ResponseWriter, r *http.Request) // Stop and delete instance // (DELETE /instances/{id}) - DeleteInstance(w http.ResponseWriter, r *http.Request, id string) + DeleteInstance(w http.ResponseWriter, r *http.Request, id string, params DeleteInstanceParams) // Get instance details // (GET /instances/{id}) GetInstance(w http.ResponseWriter, r *http.Request, id string) @@ -11902,7 +11938,7 @@ func (_ Unimplemented) CreateInstance(w http.ResponseWriter, r *http.Request) { // Stop and delete instance // (DELETE /instances/{id}) -func (_ Unimplemented) DeleteInstance(w http.ResponseWriter, r *http.Request, id string) { +func (_ Unimplemented) DeleteInstance(w http.ResponseWriter, r *http.Request, id string, params DeleteInstanceParams) { w.WriteHeader(http.StatusNotImplemented) } @@ -12933,8 +12969,19 @@ func (siw *ServerInterfaceWrapper) DeleteInstance(w http.ResponseWriter, r *http r = r.WithContext(ctx) + // Parameter object where we will unmarshal all parameters from the context + var params DeleteInstanceParams + + // ------------- Optional query parameter "graceful_shutdown" ------------- + + err = runtime.BindQueryParameter("form", true, false, "graceful_shutdown", r.URL.Query(), ¶ms.GracefulShutdown) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "graceful_shutdown", Err: err}) + return + } + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - siw.Handler.DeleteInstance(w, r, id) + siw.Handler.DeleteInstance(w, r, id, params) })) for _, middleware := range siw.HandlerMiddlewares { @@ -15333,6 +15380,15 @@ func (response TagImage400JSONResponse) VisitTagImageResponse(w http.ResponseWri return json.NewEncoder(w).Encode(response) } +type TagImage401JSONResponse Error + +func (response TagImage401JSONResponse) VisitTagImageResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(401) + + return json.NewEncoder(w).Encode(response) +} + type TagImage404JSONResponse Error func (response TagImage404JSONResponse) VisitTagImageResponse(w http.ResponseWriter) error { @@ -15642,7 +15698,8 @@ func (response CreateInstance500JSONResponse) VisitCreateInstanceResponse(w http } type DeleteInstanceRequestObject struct { - Id string `json:"id"` + Id string `json:"id"` + Params DeleteInstanceParams } type DeleteInstanceResponseObject interface { @@ -18176,10 +18233,11 @@ func (sh *strictHandler) CreateInstance(w http.ResponseWriter, r *http.Request) } // DeleteInstance operation middleware -func (sh *strictHandler) DeleteInstance(w http.ResponseWriter, r *http.Request, id string) { +func (sh *strictHandler) DeleteInstance(w http.ResponseWriter, r *http.Request, id string, params DeleteInstanceParams) { var request DeleteInstanceRequestObject request.Id = id + request.Params = params handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { return sh.ssi.DeleteInstance(ctx, request.(DeleteInstanceRequestObject)) @@ -19476,81 +19534,82 @@ var swaggerSpec = []string{ "U7SJayyVWOHmTeLCkm3QlAo966HkukPmF17XMp3VaylDk4ROZ9YJENMJxOopv5A5jHLnAUZZFAwXWBEb", "ovgU837P9CJbL/CcCA+p0b9St/6AoNX1qpJjXitv13dvf+4RFvG4cJ40y6T2yWdWmAz9V3J5H/7UPcF8", "VurEgyaB8RP23wSTowK683/t/GDBO//Xzg84ySgj/2v3MMGKSLV5b8QyeKib7qEVmCdMfFp/odVFW2JN", - "WwpP/TzUKmU6+NrbUGYFdtYZu4tEms9GfZ9fyKtj9baS8B6M7rUm9dCIJSeuLIHZUi4celUNkvjBDuN5", - "hbYe3HRW6Z5KGAHIWE8wYJgLB4CKLYI0dXRm2QSbAsT2GrtA8VZL04B7/09lHTCTvpV9oFjXryaCNiYC", - "f7lWWgnsVtyrncD08UiO8ILYQqsNj74iXz2Ab8VSpId8VXE2l9hXMy4VPHp6MAj2KqEFxfnXRksnYXkg", - "V14fjnRPjruwkFCBHyoA2SzjB3IZunE8uA5s+314oecwHdNpznPpJzCnWEUzIm1yf0KqDPipaefl9dyo", - "n3/BVDp4yKvjwdXvr3R/T4aB+oYa5m38/utkfvdWW5nfvq9lfgN8bAEQbJmiritht9mQj+Ggj9uScQUh", - "ejlPJDSukC6C3mlFpVQXEGgQB0P2v7X+8asiOP3tO5dpnQ8GO8/gd8Lmv33nkq3ZqSMVwpSgtuLo4S/H", - "EGwxhXh6KEpa4jrUx4HSXCpDeq76yL+dglTGm7TXkBwVftWQWmlI3nKt1pDsXtyvilStYPTgOpKjt9CC", - "29IDf04t6U/uRa1ocDKfTGhECYM6UGD8k0thw0aT++pAvSNuAbNhC17MYUUSaa1GFlxrjYRe1uB/8HjT", - "k7Lo4ENrj67c/9PMv+SZrZ9t9bVSWmhW2L40ehg87O318IraUyYxoxEtL12mhe5AATHjOUlzk6xafFli", - "D/bRxcXPLjFVS/zClaNT3NWgc3WKh8yvQddHr8rifuYF14KWyElsE9khXddWdYsJjhPKCETyExnKIa1W", - "jnzUY/H5hcpwWcyHduC2OZa24vPjCZWPxgoeRHw7qRTS56WO71fMLE6LE+Hg1DwpfmUZUIDxhMSnLZwr", - "3rOp7lszbvAPwxCwZwmOAAFWv2bACS26iEEj9ZsCyBDBk4QIAzqZ5cqVSh2yYnCUKV6UVbalsC5186Oc", - "KZpcdk0gHSAHSYTZwiKvDVmlM6wUSTMFCACAbgEjFCQzI67ViNWDpjyX8BYk6/tdIpxc44UcMosZYD6H", - "wuKCRAafNUn66EcOcC2mVrLHeE2h0m/kkF3SOCEji7ZyiahEcsaFIozEKOVzIqv9EiwSSgRM4gjrlZMo", - "xQuAPTQIsGZ9eEYMtGAF04Xrf2MWUyh5qXsupnwwZBjtDAYoJZhJi9Ag8QQuHNsGgkFUBvQtwmhv8NJ+", - "Vds3gOZ2y7+hT5MQZM4jPE4WiGgqNqWkN2EDU1uC1tQ019s3oUKa/SpMhra2YGVjqXQVVeMuylmJQQHm", - "85wVkBF6u1QuGMzTOtYIFcU1aGF3xiTCej0Zr/YDgKc8inIRuiD1Vnu1kP8dBUdveuewVGGEhwS08IjE", - "sOeMqxmcaQ5HafPbBqoqierPcdEEDwkXCCOPrksjAYlyYI0bABB6WRb2ZK5Q9+Xmt+7s6ONrGYE7/gai", - "86ncT0BEfDKpHMD1V5M5wKsyq5ZJ+M96To9cRWefxcUUTxmXikaOGTr8ZV9p/qoQtlIIV69skJonXFw1", - "h9P+wMVVWw3MxRo+LUXMn+EXaNvXwwOI98c38YOB2SgrmmgeXEmr01dxSkHookoWUbYo4WyqT1Fp6H5w", - "S7yv1W0YuEZ9mQrjPy7AtbQSMrI/mqLQXtAyWO0j2+pj8yLd+wP4d37hCtE0S0hKoGh0zxCb3uwSiG28", - "sEBxBRzZ7XilPlU+aoDRBaVx6XedOAR05TZsA6T35e0KMtWET9fDfRadO2zLAN7nkL2TBoj/0nhzLlHB", - "g7VAa4proOsZjWaA/Ql6q27fQIPiLLssYM83D9BrOMg++jt0vmFKamhakzwhBtJznqaXB8tlkd+fnsJH", - "BvbTFEC+PECuFHJxf0j9lo/lqWeRYKnQLxahdKNQxmFHLxXW+mYxv02L8lnC0g9ZCPGTkWvbIJ2gSw/8", - "87IBmc7x25/59NGEsW5zMREzF8WRVR2BNgmLO01xEzQJ435uDwYhjPuWGKRmGPcMQbo0mJ/5tChkUiFl", - "nGVtydcOE6h4nqYraBhteICGUsU8V3+RKiZCwMeWupuIG23gyBaxw1eaUC18pTvYm0B+weggU1kguFSa", - "qXa6HcLytHPwq/3XPE073Y4dj1eR4BbC/Ros13qDy1Esemc8wNavYvltoFirzN7DYq3dHFadbpbI35oX", - "/vTeQmeze0QyBPmgZsT9kkRQb7xVgw/jBaYsjOzlQ4wMgGdRlHBJKg6epwNbZw1dNZmx2VDk1rinhxfn", - "rs5Xm6CQc/vpufvyC9C918WKuDEjN90HDxpZHsFTTsGXS7OZcFHHOlsXTfLFE9Ln25KlqbahkK+0eXsr", - "YyvC1HrCMouwH8Sm7iPOFU+xohHUHItmnEuP7AtgclMd0BqPC8oE04rRcm1Q/qUm1Utrhr60asSBNZkh", - "7D+yffThcxvKH/7CPSq/+MGzChQcv+tEf6jLIRFGY0HJBGU4l0RLdXlKULSINFc0ReYIjmYowpnKBYH6", - "mQSllNE0T33Eeb1jcwzoOJfb6WUXjXOFEiymoJWZhy7YJuJpSlhMwD43ZDOC51SrlAIlWBEWLXqSQN3t", - "OUHXXFwlHMdgYshiDJ4eqNspiKZAgO9PicIxVhgEnUt94kcmL+iyKMVt1HpGbkpqiIdM5OxbU0tEN3vp", - "BnqJCIDlUzkrSrZGOCYsCoLIn3/ZbOzz26LPiapP9JEig+7ESx8zVMi3ubrhfBlRRE8WZaEFm18h9Mpm", - "FbaaUOHI6N/zSJu5ujk+koOpWOJVp/jL8CwVRPfFeJce333EBYpz0513KoHM/6w+oYKh+MFWkKxptvGu", - "jqGiNmWxzLfieVt/uD9P7mDL+0I4YbdRsW+qglZO+ktguXZV78RzH8mIaW1Jvk3u8Viwi+h6NPGJC4/L", - "PRVjawV3rODbPndSAoP2xdlXtl1n2zbg4a5s29lml1z6HiOnrAcxomEObs24jazamg7+TbNRarPzWOaj", - "s8jSc/HgIIKONWZ4kXAc/xmChFf4jyIuhEGUAIyKpwR87FkN/fQAsM2V5RW7Llvz/enpZhOXEGoljxDq", - "CXMILyVHf5bGywbcN3MiBI0douTR6bEN16USiZz10ZuUKqQ4uiIkKzNaIKuwr+fnsDVqw66DaHQ7hCmx", - "yDhlau0oylfvZzDlD9xCjHyBoqRF0v/qDm/tDgfL/tNjZ8BlIGfDTGC1ZqqwWlvhl7IJF6mRy/CY57p1", - "zYP0Mun9NJVyJzQhciEVSU1U4iRP4LhBVRZbedt+Z3a5CzG5+uSYdLmMiJRKSTmTQ2ZzRTIidN/6c92+", - "F2AVdAgoXPDXM8Mkv4zgPT0YE6+GVdOqAQoSVPTtHHS2cJZtxVjhhgAxO7xPGNIPEI2H5CId84RGKKHs", - "SqKNhF4Z9QTNJUr0H5srw/lG8N3nrit+95OlV/qETXiwaqOh2YKY/1RZXZatOcfkk2Nrr4l/WBz/gY0O", - "s7X1lcsFwUkPKoE7LByUK5rQD4bV6UaoVDQyKUe4WLv3pwVT7Q/ZKVFCv4MhtS1JDKIBaJdbmeDR1jAf", - "DHajjAKg2i6BwQHDa36cQo9HZ+9MGipJuVh0h0z/Axq+ODwz3t0JttYEb6C2ZDk62XqzJsD5HJbp3zhC", - "0ExwJXpBcMO/ugRvjzHSeIZkwxHl2SpViWd/+hBWK8F9tSs8TbsCgDwVs9mYChyBUCxnuYr5NQvbEOY8", - "yVP9D/PHyTqoMIWj2Xt49YuRds1w1nbjJvgkDqWdU0xMVdlHcXqYBXuqMat64dwUQIipRAMGb4FD9Wek", - "7s9vvvfX8Qt0d9oVdRWbv5iz9dA3nx2DQ9jw1+OpHHNDaW4miq+2Pl1j2mx9+j7h0ZW0UCy+2VDrbQBZ", - "rn8sIaatixDEBMgMRRbCyABlEdkdspoB0iD+SISRIiKlDCdbMGfTCIBlOysWnnMKCdoR5Kn0JI0BMykB", - "RGyAv9OzAUOVa8Dz6Epb085/x3dGKo7GJOIpcQDimyHV7W+Yqh+4qKKBfyl88cJbf4AExBTs7WsA0Jt7", - "/CRA9FN8A6HScW4dym5EG695+aMxBXUR7M2wszuQw04XDTs76bCjd+AIgwkVK7SPUspyRWQfHRv7FqTg", - "PhsgSSLOYulwzJ0Fb3cgmxJyDVk2ZHc+g+8eUuyxVAVL+dZ2EmIP+j2kv4ekHbThHzh7JuMuHLoY8VwZ", - "c789V/atmCgwj2w+uK/WOyNfdfs2nPxv9vhWeBTssmaX3tYbzp7lckaaTW4/m9pAuRoDPrYr6ytn6O98", - "LLuIkWtjDRdS9Zf4nv76zHTwENj9uqvb4PbbuX8F7W8B2l+uVRis0QRY6ivZUYdBbCQ3GRcKUBxtrr2h", - "IdAkADkCKv69OToZskizIgMtKEjKgTtZiHFzCx/+7Ry9OnrbRcdQYhb9mI83++gNSxau0L/x0QyZkcQM", - "84owQ2NDtSQOXc9m7EA99xksrjt4pJrt5mQEPCtur1yQeLczIzgGieSPzs/cdBZAHX77sz5AAPxrviy2", - "vbNS+Oi8JUoseocTRcRys6c2T4oVmBn2knYQdFZwM8CXukPpkNfKPo1sYKAxdnc6AaSMj1/rKNx/aeKH", - "8ZKZOBFTwW6cqydbChUOYsEcQyzQv66LSgRNWcKWl61UMKDLpsjvL8jkvpJ3VbDl/11PF8z0yTqasso+", - "aSIuKpis9fS65OCZgUO2jqoIZziiatFFOEnsHWVvgiIipVeIv2NB8FXMr1l/yN4WtVNsQi86OnvXdY5a", - "FFN5ZVqwvtg+ejMnQubjYnAIDprxGsOak3jIFEcRTqI80eIGmUxIBLm4UBJFNvhyi6F07vHslJ0E67d4", - "Ue35kysbF6YJ2L2SLOoUt2W2ekuQKME0bQYft4IaBBxCqMFYN8oZomyS2JCqSHApkW2qRxI6pePEBgjJ", - "PrqYESRxSoYsSzBjRKBcmqh4PfReJoiUuUnw1g0ASK+hqC4qgQUzwZUNTUg4F9JEE2gKf3+KpCLZCjJ7", - "a1o+hTnfk2xrGrc9PZKRujaGZlOIfQXpDTGUYhZc01GeuADGBw1FNwN6bCnxqRz8C0GnUyL0qcCGyZpw", - "PHOs3XKaQ1/JWG4sIXlevNWuhGTRqpeV6GXsrQSGG5VY23HndlF/gc6vaCN2oH10uyzin/RHLfuuZquG", - "B2EffeIs/yyV+c+9JMG2BqySwp+aOckbeeWoVhJt18Nqtc6svc9M19b4WY8Gm/WU0bJwJX22SeH98ghh", - "8LAoDw9dZO1p01YF7aqimzak/K9H0/8iKPB+YPQfGeXkDjD6X1TePeCcPx7+SfCgPlYefcX37OrX/umR", - "8O8rfd7A4QMcW1P6vOF6Nnh1paL03r7TTk2yLf6ZJHgb73gL+d0t+1etv4XK4C3WOhe0JniSZmrhAtqs", - "r7IMOpP0A+k3OIKLuNX7cwXfIaTz85GHo9PGgM4/Z7n5R4kZtaUDqUQnx4E67k8MY9A/c5WLZUvfOj0s", - "ohmdk2aje/UE2yXKBOllPAPnSmwWzK6Hu8sUFv3pB2Sbt5ir9l9QexKg+kmMYipIpJKFqQOqOYLp4xuJ", - "BNeaADznYtEcJWKOyA+Cp4d2NmvuQ3umrDGsjDNMF70YK9ybO26zwoT2CdGdLp5SMzxEGXr9PdogN0qY", - "ChdoojUfRCfFkpoC/hJoctMf8PagwbJJP5DRdNxmlCtqlbyxtWBQlEvFU7f3J8doA2qfTQnTe6FF/QlI", - "spngcxqTuDLGzpwnZlW3Gxb0tnZXLVQUheuccmEG9ygyTJsLafqBZlW2UITEjCnDMLi1VUGqZ8ok8ev+", - "MGUuAMfukRvF1yvMan4bTtnRlAh1OO0iKs4NxPPm12vuKV9zfjKUu9Mqt50Lz1ltvG6XH9Uybek+Cj8U", - "uXMPa7Z+/+Wk9FD5JLN5rOl8XiikTWbzL4sEBw93Pzy0ufz9E04BfU2c8u2ZyqEB3WKIYH6GmO6YzEnC", - "sxTqocO7nW4nF0nnoDNTKjvY2oLY7xmX6mDv5fPdzsffPv7/AQAA///BBhRZHPQBAA==", + "WwpP/TzUKmU6+NrbUGYFdtYZu4tEms9GfZ9fyKtj9baS8B6M7rUm9dCIJSeuLIHZUi4celUNkvjfW+g7", + "rxD0g9vrKt1TCSMAwe4JRilz4VBXsYWtpo64LW9iU8D1XmOMKN5qaY9w7/+pTBJm0rcyShTr+tUu0cYu", + "4S/XStOE3Yp7NU6YPh7J+14QW2i14dFXuK0HcOhYivTgtioe7hJwa8algkdPD3vBXiW0oDj/2mjpmSwP", + "5Mrrw5HuyXEXFhLK/kPZIZva/EB+SjeOB1e8bb8PL/QcpmM6zXku/azpFKtoRqRFFEhIlQE/NZNAeT03", + "GgW+YCodPOTV8eA6/1e6vydrRH1DDfM2wQbrZH73VluZ376vZX6DtmxRF2xtpK6rm7fZkATi8JbbknEF", + "lno5OSU0rpAugt5pRaVUFxBoEAdD9r+1/vGrIjj97TuX3p0PBjvP4HfC5r995zK82akjFcKUoLbM6eEv", + "xxDhMYUgfqiEWoJJ1MeB0lwqQ3qu5Mm/nYJUBrm015AcFX7VkFppSN5yrdaQ7F7cr4pULZv04DqSo7fQ", + "gtt6B39OLelP7rqtaHAyn0xoRAmD4lNg/JNLscpGk/vqtb0jWAKzsRJeoGNFEmmtRhZca42EXhb+/5yR", + "hN3GGhQcYaVImik0FTgikzwxVVuQnOUq5tfMlaSACbrqZrScT+h6d02NXCPhBNlwifa2mm5RlvGhVV3b", + "8RPNUOWZrTBulctStGnWLh+XeO9Xp2xx1T68VvmUScyob8tLl2kNIVBizbh50tyk8xZfluiMfXRx8bNL", + "3dXqiXAF+xR3VfpcJech86v09dGrsvyhecG1oNUHEttUf0hotnXvYoLjhDICuQ5EhrJsq7U1H/VYfH4J", + "OFw49KFd3G2Opa2J/XgS8KOxggeRNQufvrIO2eJoejVFi9Pi5E04NU+KX1kGFGA8IVlvC+eK9ywYwNaM", + "G4TIMEjuWYIjwMjVrxn4Rou/YvBa/aYAVEXwJCHCwHJmuXLi1pAVg6NM8aLwtJXMLnXzo5wpmlx2Tagh", + "YCtJhNnCYtMNWaUzK/MBRgLgf8AIBcnMiGtVdPWgKc8lvAVwBn6XCCfXeCGHzKIqmM+h9LogkUGwTZI+", + "+pEDoI2pJu0xXlPK9Rs5ZJc0TsjI4tFcIiqRnHGhCCMxSvmcyGq/BIuEEgGTOMJ65SRK8QKAIQ1Grlkf", + "nhEDvlhBveH635jFFIqC6p6LKR8MGUY7gwFKCWbSYlhIPIELx7aBYBCVAX2LMNobvLRf1fYNwMvd8m/o", + "0yQEmfMIj5MFIpqKTbHtTdjA1BbpNVXf9fZNqJBmvwr7pq2+WNlYKl3N2biLclaidICtP2cFqIbeLpUL", + "BvO0XkBCRXENWmCiMYmwXk/Gq/0AJCyPolyELki91V616H9HwdGb3jksVRgDIwGTQURi2HPG1QzONIej", + "tPltA1WVRPXnuGiCh4QLhJFH16VFg0Q5sMYNgFC9LEufMlfK/HLzW3d29PG1jMAdfwNi+lTuJyAiPplU", + "DuD6q8kc4FW5Z8sk/Gc9p0eu5rXP4mKKp4xLRSPHDB1Cta80f1UIWymEq1c2SM0TLq6aA45/4OKqrQbm", + "AiOfliLmz/ALdETo4QEI/uP7I8AabpQVTTQPrqTV6as4pSB0USWLkGCUcDbVp6i0yj+428DX6jYMoKW+", + "TIVxdhfwY1oJGdkfTdlsL6wbXAyRbfWxeZHu/QGcUb9whWiaJSQlUFa7Z4hNb3YJVTdeWCi9ArDtdrxS", + "nyofV8HogtLEH3SdOAR05TZsA6T35e0KMtWET9cDohadO/TPACLqkL2TplTBpXE9XaKCB2uB1pQfQdcz", + "Gs0AHRX0Vt2+AU/FWXZZAMNvHqDXcJB9fHzofMMUHdG0JnlCDOjpPE0vD5YLR78/PYWPDDCqKRF9eYBc", + "seji/pD6LR/tVM8iwVKhXyyG60ahjMOOXiqs9c1ifpsWB7UE7h+yECYqI9e2QTpBlx486mUDdp/jtz/z", + "qfxiXEVluRUzF8WRVR2BNgmLO01BHjQJO362B4NQFYCWKK1mGPcM0ro0mJ/5tCj1UiFlnGVtydcOE6h4", + "nqYraBhteJCPUsU8V3+RKiZCwMeWupuIG23gyJb5w1eaUC3ApzvYm0B+wVAmU3shuFSaqXa6HcLytHPw", + "q/3XPE073Y4dj1ez4RbC/Rq023qDyyE3emc8SNuvYvltwGqrzN5Dq63dHFadbpbI35oX/vTeQmeze0Qy", + "BPmgZsT9kkRQb7xVgw/jBeoujOzlQ4wMoHlRlHBJKg6epwPsZw1dNZmx2VDk1rinhxfnrhJamwiWc/vp", + "ufvyC9C918WKuDEjN90HDxpZHsFTBimQS7OZcFFHg1sXTfLFE9Ln25KlqbahkK+0eXsrYyvC1HrCMouw", + "H8SmMibOFU+xohFUZYtmnEuP7AvodlM/0RqPC8oE04rRcm0GwaUm1Utrhr60asSBNZkh7D+yffThc5t3", + "EP7CPSq/+MGzChQcv+tEf6hcIhFGY0HJBGU4l0RLdXlKULSINFc0ZfgIjmYowpnKBYEKowSllNE0T31M", + "fr1jcwz4QZfb6WUXjXOFEiymoJWZhy7YJuJpSlhMwD43ZDOC51SrlAIlWBEWLXqSQGXyOUHXXFwlHMdg", + "YshiDJ4eqGwqiKZAKHCQEoVjrDAIOpf6xI9MEtNlUazcqPWM3JTUEA+ZyNm3ptqKbvbSDfQSESgnQOWs", + "KGob4ZiwKAizf/5ls7HPb4s+J6o+0UeKDLoTL33MUCHf5uqG82VEET1ZSIgWbH6F0CubVdhq9ocjo3/P", + "I23m6ub4SA6mYolXneIvw7NUEN0X4116fPcRFyjOTXfeqQQy/7P6hAqG4gdbQWap2ca7OoaK6p3FMt+K", + "52394f48uYMt7wvhhN1Gxb6pTlw56S+B5dpVvRPPfSQjprUl+Ta5x2PBLqLr0cQnLjwu91SMrRVktoJv", + "+9xJCQzaF2df2XadbduAh7uybWebXXLpe4ycsh7EiIY5uDXjNrJqazr4N81Gqc3OY5mPziJLz8WDwyw6", + "1pjhRcJx/GcIEl7hP4q4EAb+AgA1nhI0tGc19NMDwDZXFqDsumzN96enm01cQqiVPEKoJ8whvJQc/Vka", + "Lxtw38yJEDR28JdHp8c2XJdKJHLWR29SqpDi6IqQrMxogazCvp6fAwKpDbuO+NHtEKbEIuOUqbWjKF+9", + "n8GUP3CLh/IFipK21sBXd3hrdzhY9p8eOwMuAzkbZgKrNVOF1doayJRNuEiNXIbHPNetax6kl0nvp0Eq", + "mNCEyIVUJDVRiZM8geMGdWtsbXL7ndnlLsTk6pNj0uUyIlIqJeVMDpnNFcmI0H3rz3X7XoBV0CGgcMFf", + "zwyT/DKC9/RgTLwaVk2rBpBNUPO4c9DZwlm2FWOFGwLE7PA+YUg/QDQekot0zBMaoYSyK4k2Enpl1BM0", + "lyjRf2yuDOcbwXefu/L63U+WXukTNuHBupaGZgti/lNldVm25hyTT46tvSb+YXH8BzY6zNbW13YXBCc9", + "qJXugHtQrmhCPxhWpxuhUtHIpBzhYu3enxZMtT9kp0QJ/Q6G1LYkMYgGoF1uZYJHW8N8MNiNMgrob7sE", + "BgcMr/lxCj0enb0zaagk5WLRHTL9D2j44vDMeHcn2FoTvIHaou7oZOvNmgDnc1imf+MIQTPBlegFwQ3/", + "6hK8PcZI4xmSDUeUZ6tUJZ796UNYrQT31a7wNO0KAPJUzGajAPZyaFxhG8KcJ3mq/2H+OFmHa6ZwNHsP", + "r34x0q4Zztpu3ASfxKG0c4qJqbv7KE4Ps2BPNWZVL5ybAggxlWjA4C1wqP6M1P35zff+On6B7k67oq6m", + "9Rdzth765rNjcAgb/no8lWNuKM3NRPHV1qdrTJutT98nPLqSForFNxtqvQ3w1fWPJR62dRGCmACZochC", + "GBmgLCK7Q1YzQBrEH4kwUkSklOFkC+ZsGgFkb2fFwnNOIUE7gjyVnqQxYCYlAN8N8Hd6NmCocg14Hl1p", + "q/757/jOSMXRmEQ8JQ7tfDOkuv0NU/UDF1Xo8i+FL1546w+QgJiCvX0NWntzj5+E3n6KbyBUOs6tQ9mN", + "aOM1L380pqAugr0ZdnYHctjpomFnJx129A4cYTChYoX2UUpZrojso2Nj34IU3GcDJEnEWSwd6Lqz4O0O", + "ZFNCriHLhuzOZ/DdQ4o9lqpgKd/aTkLsQb+H9PeQtIM2/ANnz2TchUMXI54rY+6358q+FRMF5pHNB/fV", + "emfkq27fhpP/zR7fCo+CXdbs0tt6w9mzXM5Is8ntZ1PIKFdjAPN2hY/lDP2dj2UXMXJtrOFCqv4S39Nf", + "n5kOHqLQgO7qNkUG7Ny/VhhoUWGgXKswWKMJsNRXsqMOg9hIbjIuFKA42lx7Q0OgSQByBJQnfHN0MmSR", + "ZkUGWlCQlAN3snjo5hY+/Ns5enX0touOoQgv+jEfb/bRG5YsbO1666MZMiOJGeYVYYbGhmpJHLqezdiB", + "eu4zWFx38EhV7c3JCHhW3F65IPFuZ0ZwDBLJH52fueksgDr89md9gAD413xZbHtnpfDReUuUWPQOJ4qI", + "5WZPbZ4UKzAz7CXtIOis4GaAL3WH0iGvlX0a2cBAY+zudAJIGR+/Fn24/+LND+MlM3EiptzeOFdPtm4r", + "HMSCOYZYoH9dF2UTmrKELS9bqWBAl02R31+QyX0l76pgy/+7ni6Y6ZN1NGWVfdJEXJRbWevpdcnBMwOH", + "bB1VEc5wRNWii3CS2DvK3gRFREqvEH/HguCrmF+z/pC9LQq92IRedHT2rusctSim8sq0YH2xffRmToTM", + "x8XgEBw04zWGNSfxkCmOIpxEeaLFDTKZkAhycaF+i2zw5RZD6dzj2Sk7CRab8aLa8ydX4y5ME7B7JVnU", + "KW7LbPWWIFGCadoMPm4FNQg4hFCDsW6UM0TZJLEhVZHgUiLbVI8kdErHiQ0Qkn10MSNI4pQMWZZgxohA", + "uTRR8XrovUwQKXOT4K0bAJBeQ1FdVAILZoIrG5qQcC6kiSbQFP7+FElFshVk9ta0fApzvifZ1jRue3ok", + "I3VtDM2mEPsK0htiKMUsuKajPHEBjA8aim4G9NhS4lM5+BeCTqdE6FOBDZM14XjmWLvlNIe+krHcWO/y", + "vHirXb3LolUvK9HL2FsJDDcqsbbjzu2i/gKdX9FG7ED76HZZxD/pj1r2Xc1WDQ/CPvrEWYZKd/47Vsk8", + "95IE2xqwSgp/auYkb+SVo1pJtF0Pq9U6s/Y+M11b42c9GmzWU0bLwpX02SaF98sjhMHDojw8dJG1p01b", + "FbSrim7akPK/Hk3/i6DA+4HRf2SUkzvA6H9RefeAc/54+CfBg/pYefQV37MrtvunR8K/r/R5A4cPcGxN", + "6fOG69ng1ZWK0nv7Tjs1ybb4Z5LgbbzjLeR3t+xftf4WKoO3WOtc0JrgSZqphQtos77KMuhM0g+k3+AI", + "LuJW788VfIeQzs9HHo5OGwM6/5y18R8lZtSWDqQSnRwHis4/MYxB/8xVLpYtfev0sIhmdE6aje7VE2yX", + "KBOkl/EMnCuxWTC7Hu4uU1j0px+Qbd5irtp/Qe1JgOonMYqpIJFKFqYOqOYIpo9vJBJcawLwnItFc5SI", + "OSI/CJ4e2tmsuQ/tmbLGsDLOMF30Yqxwb+64zQoT2idEd7p4Ss3wEGXo9fdog9woYSpcoInWfBCdFEtK", + "biJCYgk0uekPeHvQYNmkH8hoOm4zyhW1St7YWjAoyqXiqdv7k2O0AbXPpoTpvdCi/gQk2UzwOY1JXBlj", + "Z84Ts6rbDQt6W7urFiqKwnVOuTCDexQZps2FNP1AsypbKEJixpRhGNzaqiDVM2WS+HV/mDIXgGP3yI3i", + "6xVmNb8Np+xoSoQ6nHYRFecG4nnz6zX3lK85PxnK3WmV286F56w2XrfLj2qZtnQfhR+K3LmHNVu//3JS", + "eqh8ktk81nQ+LxTSJrP5l0WCg4e7Hx7aXP7+CaeAviZO+fZM5dCAbjFEMD9DTHdM5iThWQr10OHdTreT", + "i6Rz0JkplR1sbUHs94xLdbD38vlu5+NvH///AAAA//+/dvb/PvUBAA==", } // GetSwagger returns the content of the embedded swagger specification file From 90f875f0d31565c91152f4b09be7b21bf7351857 Mon Sep 17 00:00:00 2001 From: "kernel-internal[bot]" <260533166+kernel-internal[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:07:37 +0000 Subject: [PATCH 03/13] chore(stlc): seal custom-code tracking files --- .../custom-code/go/2026-08-21T19-11-19-339Z-custom-code.json | 4 ++-- .../python/2026-08-17T14-04-38-940Z-custom-code.json | 4 ++-- .../typescript/2026-08-21T19-11-20-278Z-custom-code.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/stainless/custom-code/go/2026-08-21T19-11-19-339Z-custom-code.json b/stainless/custom-code/go/2026-08-21T19-11-19-339Z-custom-code.json index 3d74ae22a..2e8ad9ebb 100644 --- a/stainless/custom-code/go/2026-08-21T19-11-19-339Z-custom-code.json +++ b/stainless/custom-code/go/2026-08-21T19-11-19-339Z-custom-code.json @@ -1,6 +1,6 @@ { - "base": "fa5489d57854e89229c6955da07d6bab7d6c547a", - "integrated": "ea5a7f8d0d069b51e10189a8d0196f6155aaa4a8", + "base": "05a212dc4688777a3d1df23a3b182487055e77dd", + "integrated": "013dac97bc8391d82f4b583f1a579c3024f23f88", "filename": "2026-08-21T19-11-19-339Z-custom-code.json", "branch": "main" } diff --git a/stainless/custom-code/python/2026-08-17T14-04-38-940Z-custom-code.json b/stainless/custom-code/python/2026-08-17T14-04-38-940Z-custom-code.json index 28b7c63aa..45791a1af 100644 --- a/stainless/custom-code/python/2026-08-17T14-04-38-940Z-custom-code.json +++ b/stainless/custom-code/python/2026-08-17T14-04-38-940Z-custom-code.json @@ -1,6 +1,6 @@ { - "base": "cbc546d01211970ac7bc195f0b260106aad040a8", - "integrated": "da303fcc32f896bcba886a5101a82d8a6ba728cd", + "base": "948291d549091ca6bab9d0ac22a85aeeb90e22be", + "integrated": "cfc7ca84d29e364889896f01c6e10cedd3ec0541", "filename": "2026-08-17T14-04-38-940Z-custom-code.json", "branch": "main" } diff --git a/stainless/custom-code/typescript/2026-08-21T19-11-20-278Z-custom-code.json b/stainless/custom-code/typescript/2026-08-21T19-11-20-278Z-custom-code.json index bd134d415..d2a1c684e 100644 --- a/stainless/custom-code/typescript/2026-08-21T19-11-20-278Z-custom-code.json +++ b/stainless/custom-code/typescript/2026-08-21T19-11-20-278Z-custom-code.json @@ -1,6 +1,6 @@ { - "base": "10ef61196213827ae7ba90504d3015be0a8c0f22", - "integrated": "a6716d0c0b912758dd05cdafa826b9ba35657aca", + "base": "3339edfd85d573ddf79daa9477092f5f4eaa0dec", + "integrated": "4cd115310104722fe9ce8ab2b307d535c16d25dd", "filename": "2026-08-21T19-11-20-278Z-custom-code.json", "branch": "main" } From dd879d90ae586b3942b802c9367d0d91d1bdbb8e Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 31 Aug 2026 18:37:18 +0000 Subject: [PATCH 04/13] Persist OCI manifest content model per image digest --- lib/images/disk_usage_test.go | 8 + lib/images/manager.go | 269 +++++++++++------------------ lib/images/manager_test.go | 4 +- lib/images/manifest_model.go | 166 ++++++++++++++++++ lib/images/manifest_model_test.go | 242 ++++++++++++++++++++++++++ lib/images/oci.go | 99 +++++++---- lib/images/oci_test.go | 11 +- lib/images/storage.go | 77 +++------ lib/images/storage_refs.go | 17 +- lib/images/storage_test.go | 110 ++++-------- lib/images/tag.go | 110 ------------ lib/images/tag_manager.go | 272 ++++++++++++++++++++++++++++++ lib/images/tag_test.go | 30 +--- lib/paths/paths.go | 6 + 14 files changed, 932 insertions(+), 489 deletions(-) create mode 100644 lib/images/manifest_model.go create mode 100644 lib/images/manifest_model_test.go delete mode 100644 lib/images/tag.go create mode 100644 lib/images/tag_manager.go diff --git a/lib/images/disk_usage_test.go b/lib/images/disk_usage_test.go index 0de352d48..bd6056fef 100644 --- a/lib/images/disk_usage_test.go +++ b/lib/images/disk_usage_test.go @@ -9,6 +9,8 @@ import ( ) func TestTotalReadyImageBytesFromMetadata_UsesRootfsFallbackForMalformedMetadata(t *testing.T) { + t.Parallel() + imagesDir := t.TempDir() digestDir := filepath.Join(imagesDir, "docker.io", "library", "alpine", "sha256deadbeef") require.NoError(t, os.MkdirAll(digestDir, 0o755)) @@ -21,6 +23,8 @@ func TestTotalReadyImageBytesFromMetadata_UsesRootfsFallbackForMalformedMetadata } func TestTotalReadyImageBytesFromMetadata_DeduplicatesHardLinkedAliases(t *testing.T) { + t.Parallel() + imagesDir := t.TempDir() sourceDir := filepath.Join(imagesDir, "source", "digest") targetDir := filepath.Join(imagesDir, "target", "digest") @@ -41,6 +45,8 @@ func TestTotalReadyImageBytesFromMetadata_DeduplicatesHardLinkedAliases(t *testi } func TestTotalReadyImageBytesFromMetadata_DeduplicatesMalformedAliases(t *testing.T) { + t.Parallel() + imagesDir := t.TempDir() malformedDir := filepath.Join(imagesDir, "a-malformed", "digest") validDir := filepath.Join(imagesDir, "b-valid", "digest") @@ -59,6 +65,8 @@ func TestTotalReadyImageBytesFromMetadata_DeduplicatesMalformedAliases(t *testin } func TestTotalReadyImageBytesFromMetadata_UsesRootfsFallbackForReadyImageWithoutSize(t *testing.T) { + t.Parallel() + imagesDir := t.TempDir() digestDir := filepath.Join(imagesDir, "docker.io", "library", "alpine", "sha256deadbeef") require.NoError(t, os.MkdirAll(digestDir, 0o755)) diff --git a/lib/images/manager.go b/lib/images/manager.go index aa66abc4f..f4c547886 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -77,6 +77,7 @@ type manager struct { createMu sync.Mutex diskUsageMu sync.RWMutex tagGenerations map[string]uint64 + requestedTags map[string]string // newest pull's digest per requested tag diskUsageLoaded bool readyImageBytes int64 ociCacheBytes int64 @@ -105,6 +106,7 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma borrowedCredentialsTimeout: DefaultBorrowedCredentialsTimeout, readySubscribers: make(map[string][]chan StatusEvent), tagGenerations: make(map[string]uint64), + requestedTags: make(map[string]string), } // Initialize metrics if meter is provided @@ -117,7 +119,15 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma } m.RecoverInterruptedBuilds() - promoteLegacyImages(m.paths) + // Promotion hardlinks disks and repoints tags, which can take a while on a + // large legacy store; scan before starting the background migration so new + // images created after startup are not picked up by the migration. + legacyRefs, err := collectLegacyImages(p) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to scan legacy images for promotion: %v\n", err) + } else { + go promoteLegacyImages(p, legacyRefs) + } return m, nil } @@ -212,7 +222,7 @@ func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Ima m.createMu.Lock() defer m.createMu.Unlock() - if img, found, err := m.reuseExistingImage(ref, req.Credentials, req.Tags); found || err != nil { + if img, found, err := m.reuseExistingImage(ref, req.Credentials); found || err != nil { return img, err } return m.createAndQueueImage(ref, req, platform) @@ -244,14 +254,31 @@ func (m *manager) ImportLocalImage(ctx context.Context, repo, reference, digest m.createMu.Lock() defer m.createMu.Unlock() - if img, found, err := m.reuseExistingImage(ref, nil, nil); found || err != nil { - return img, err + // Check if we already have this digest (deduplication) + if meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()); err == nil { + // Don't cache failed builds - allow retry by falling through to + // re-queue the build. + if meta.Status == StatusFailed { + if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), ref.DigestHex(), false); err != nil { + return nil, fmt.Errorf("remove failed image: %w", err) + } + } else { + if err := m.claimTagForStatus(meta, ref); err != nil { + return nil, fmt.Errorf("create image tag: %w", err) + } + img := meta.toImageFor(ref.String()) + if meta.Status == StatusPending { + img.QueuePosition = m.queue.GetPosition(meta.Digest) + } + return img, nil + } } + // Don't have this digest yet, queue the build return m.createAndQueueImage(ref, CreateImageRequest{Name: imageRef}, hostPlatform()) } -func (m *manager) reuseExistingImage(ref *ResolvedRef, credentials *authn.AuthConfig, resourceTags tags.Tags) (*Image, bool, error) { +func (m *manager) reuseExistingImage(ref *ResolvedRef, credentials *authn.AuthConfig) (*Image, bool, error) { meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()) if err != nil { return nil, false, nil @@ -262,95 +289,25 @@ func (m *manager) reuseExistingImage(ref *ResolvedRef, credentials *authn.AuthCo } return nil, false, nil } - if err := m.updateExistingReference(meta, ref, resourceTags); err != nil { - return nil, true, fmt.Errorf("update image reference: %w", err) + if ref.Tag() != "" { + if meta.Status != StatusReady { + // A pending pull with different credentials does not get to point + // the tag at its digest. + if !m.inflightCredentialsMatch(ref.Digest(), credentials) { + return nil, true, fmt.Errorf("%w: retry after the current pull completes", ErrCredentialConflict) + } + } + if err := m.claimTagForStatus(meta, ref); err != nil { + return nil, true, fmt.Errorf("create image tag: %w", err) + } } img := meta.toImageFor(ref.String()) - if meta.Status == StatusReady { - return img, true, nil - } - if !m.inflightCredentialsMatch(ref.Digest(), credentials) { - return nil, true, fmt.Errorf("%w: retry after the current pull completes", ErrCredentialConflict) - } if meta.Status == StatusPending { img.QueuePosition = m.queue.GetPosition(meta.Digest) } return img, true, nil } -func (m *manager) updateExistingReference(meta *imageMetadata, ref *ResolvedRef, resourceTags tags.Tags) error { - if ref.Tag() == "" { - return nil - } - if meta.Status == StatusReady { - m.nextTagGeneration(ref.Repository(), ref.Tag()) - if err := createTagSymlink(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()); err != nil { - return err - } - } else if err := m.recordPendingTag(meta, ref); err != nil { - return err - } - setReferenceTags(meta, ref.String(), resourceTags) - return writeMetadata(m.paths, ref.Repository(), ref.DigestHex(), meta) -} - -func tagGenerationKey(repository, tag string) string { - return repository + ":" + tag -} - -func (m *manager) nextTagGeneration(repository, tag string) uint64 { - key := tagGenerationKey(repository, tag) - m.tagGenerations[key]++ - return m.tagGenerations[key] -} - -func (m *manager) recordPendingTag(meta *imageMetadata, ref *ResolvedRef) error { - if meta.RequestedTag == ref.Tag() && strings.HasPrefix(meta.Name, ref.Repository()+":") { - return ensurePendingTag(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()) - } - for _, claim := range meta.TagClaims { - if claim.Repository == ref.Repository() && claim.Tag == ref.Tag() { - return ensurePendingTag(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()) - } - } - previous, err := resolveTag(m.paths, ref.Repository(), ref.Tag()) - if err != nil && !errors.Is(err, ErrNotFound) { - return err - } - meta.TagClaims = append(meta.TagClaims, imageTagClaim{ - Repository: ref.Repository(), - Tag: ref.Tag(), - PreviousTagDigest: previous, - TagGeneration: m.nextTagGeneration(ref.Repository(), ref.Tag()), - }) - return ensurePendingTag(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()) -} - -func (m *manager) restoreTagGenerations(metas []*imageMetadata) { - m.createMu.Lock() - defer m.createMu.Unlock() - for _, meta := range metas { - ref, err := ParseNormalizedRef(meta.Name) - if err != nil { - continue - } - m.restoreTagGeneration(ref.Repository(), meta.RequestedTag, meta.TagGeneration) - for _, claim := range meta.TagClaims { - m.restoreTagGeneration(claim.Repository, claim.Tag, claim.TagGeneration) - } - } -} - -func (m *manager) restoreTagGeneration(repository, tag string, generation uint64) { - if tag == "" { - return - } - key := tagGenerationKey(repository, tag) - if generation > m.tagGenerations[key] { - m.tagGenerations[key] = generation - } -} - func (m *manager) inflightCredentialsMatch(digest string, credentials *authn.AuthConfig) bool { inflight := m.inflightPulls[digest] var existingFingerprint [32]byte @@ -442,6 +399,7 @@ func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest, tagGeneration := uint64(0) if ref.Tag() != "" { tagGeneration = m.nextTagGeneration(ref.Repository(), ref.Tag()) + m.trackRequestedTag(ref.Repository(), ref.Tag(), ref.DigestHex()) } meta := &imageMetadata{ Name: ref.String(), @@ -455,16 +413,19 @@ func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest, RequestedTag: ref.Tag(), PreviousTagDigest: previousTagDigest, TagGeneration: tagGeneration, - References: map[string]tags.Tags{ref.String(): tags.Clone(req.Tags)}, CreatedAt: time.Now(), } // Write initial metadata if err := writeMetadata(m.paths, ref.Repository(), ref.DigestHex(), meta); err != nil { + if ref.Tag() != "" { + m.revertTagGeneration(ref.Repository(), ref.Tag()) + } return nil, fmt.Errorf("write initial metadata: %w", err) } if ref.Tag() != "" && previousTagDigest == "" { if err := createTagSymlink(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()); err != nil { + m.revertTagGeneration(ref.Repository(), ref.Tag()) return nil, fmt.Errorf("create pending image tag: %w", err) } } @@ -542,8 +503,7 @@ func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef, credentials // Another build completed first; last-pull-wins repoints the tag. if ref.Tag() != "" { m.createMu.Lock() - m.nextTagGeneration(ref.Repository(), ref.Tag()) - err := createTagSymlink(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()) + err := m.claimReadyTag(ref.Repository(), ref.Tag(), ref.DigestHex()) m.createMu.Unlock() if err != nil { slog.Warn("failed to claim ready image tag", "repository", ref.Repository(), "tag", ref.Tag(), "error", err) @@ -557,6 +517,8 @@ func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef, credentials m.updateStatusByDigest(ref, StatusConverting, nil, buildID) diskPath := resolveImageLayout(m.paths, ref.Repository(), ref.DigestHex()).disk + // Keep the temporary filesystem beside its final path so finalization stays + // atomic even when system/builds and images are on different filesystems. diskTempPath := diskPath + ".tmp-" + buildID defer os.Remove(diskTempPath) // Use default image format (erofs on Linux, ext4 on Darwin) @@ -583,21 +545,18 @@ func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef, credentials } func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize int64, buildID, diskTempPath string) error { - if diskTempPath != "" { - defer os.Remove(diskTempPath) - } - m.createMu.Lock() defer m.createMu.Unlock() + layout := resolveImageLayout(m.paths, ref.Repository(), ref.DigestHex()) + // Read current metadata to preserve request info and reject stale builds. - meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()) + meta, err := readMetadataAt(layout) if err != nil || meta.BuildID != buildID { return errStaleBuild } - finalDiskPath := resolveImageLayout(m.paths, ref.Repository(), ref.DigestHex()).disk - if err := installAtomically(finalDiskPath, func(path string) error { + if err := installAtomically(layout.disk, func(path string) error { return os.Rename(diskTempPath, path) }); err != nil { return fmt.Errorf("install image disk: %w", err) @@ -613,6 +572,17 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i return err } + // Persist the manifest content model beside the shared content so later + // stages can recompose the image from per-layer artifacts and GC can tell + // which OCI blobs are still referenced. + if result.Manifest != nil { + model := *result.Manifest + model.Platform = actualPlatform.String() + if err := writeManifestModel(m.paths, ref.DigestHex(), &model); err != nil { + return fmt.Errorf("write manifest model: %w", err) + } + } + meta.Status = StatusReady meta.Error = nil meta.Platform = actualPlatform.String() @@ -623,51 +593,18 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i meta.Labels = result.Metadata.Labels meta.WorkingDir = result.Metadata.WorkingDir - if err := writeMetadata(m.paths, ref.Repository(), ref.DigestHex(), meta); err != nil { + if err := writeMetadataFile(layout.metadata, meta); err != nil { return fmt.Errorf("write final metadata: %w", err) } m.notifyReady(ref.DigestHex(), StatusReady, nil) - m.claimImageTags(ref, meta) + if !m.claimRequestedTag(ref, meta) { + m.cleanupUnclaimedImage(ref) + } m.refreshDiskUsageTotals() return nil } -func (m *manager) claimImageTags(ref *ResolvedRef, meta *imageMetadata) { - requestedTag := meta.RequestedTag - if requestedTag == "" { - requestedTag = ref.Tag() - } - claimed := m.claimTag(ref.Repository(), ref.DigestHex(), requestedTag, meta.PreviousTagDigest, meta.TagGeneration, meta.RequestedTag == "") - for _, claim := range meta.TagClaims { - claimed = m.claimTag(claim.Repository, ref.DigestHex(), claim.Tag, claim.PreviousTagDigest, claim.TagGeneration, false) || claimed - } - if !claimed { - if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), ref.DigestHex(), true); err != nil { - slog.Warn("failed to collect stale image", "repository", ref.Repository(), "digest", ref.DigestHex(), "error", err) - } - } -} - -func (m *manager) claimTag(repository, digestHex, tag, previous string, generation uint64, allowMissing bool) bool { - if tag == "" || m.tagGenerations[tagGenerationKey(repository, tag)] != generation { - return false - } - current, err := resolveTag(m.paths, repository, tag) - if err != nil { - if !allowMissing || !errors.Is(err, ErrNotFound) { - return false - } - } else if current != digestHex && current != previous { - return false - } - if err := createTagSymlink(m.paths, repository, tag, digestHex); err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to create tag symlink: %v\n", err) - return false - } - return true -} - func phaseStatus(err error) string { if err != nil { return "failed" @@ -712,7 +649,8 @@ func (m *manager) updateStatusByDigest(ref *ResolvedRef, status string, err erro m.createMu.Lock() defer m.createMu.Unlock() - meta, readErr := readMetadata(m.paths, ref.Repository(), ref.DigestHex()) + layout := resolveImageLayout(m.paths, ref.Repository(), ref.DigestHex()) + meta, readErr := readMetadataAt(layout) if readErr != nil || meta.BuildID != buildID { return } @@ -723,13 +661,18 @@ func (m *manager) updateStatusByDigest(ref *ResolvedRef, status string, err erro meta.Error = &errorMsg } - writeMetadata(m.paths, ref.Repository(), ref.DigestHex(), meta) + writeMetadataFile(layout.metadata, meta) // Notify while holding createMu so a delete/recreate cannot race the // metadata write and receive a terminal event for the old build. if status == StatusReady || status == StatusFailed { m.notifyReady(ref.DigestHex(), status, err) } + // A failed pull releases its tag claim so an older in-flight pull of the + // same tag can still repoint it. + if status == StatusFailed && meta.RequestedTag != "" { + m.releaseTagGeneration(ref.Repository(), meta.RequestedTag, meta.TagGeneration) + } } func (m *manager) RecoverInterruptedBuilds() { @@ -737,12 +680,12 @@ func (m *manager) RecoverInterruptedBuilds() { if err != nil { return // Best effort } - m.restoreTagGenerations(metas) // Sort by created_at to maintain FIFO order sort.Slice(metas, func(i, j int) bool { return metas[i].CreatedAt.Before(metas[j].CreatedAt) }) + m.restoreTagState(metas) seenDigests := make(map[string]struct{}) for _, meta := range metas { @@ -815,6 +758,7 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error { if err := deleteTagsForDigest(m.paths, repository, digestHex); err != nil { return err } + m.pruneTagGenerations() if err := removeDigestIfUnreferenced(m.paths, repository, digestHex, false); err != nil { return err } @@ -823,7 +767,6 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error { } tag := ref.Tag() - m.nextTagGeneration(repository, tag) // Resolve the tag to get the digest before deleting digestHex, err := resolveTag(m.paths, repository, tag) @@ -835,6 +778,7 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error { if err := deleteTag(m.paths, repository, tag); err != nil { return err } + m.pruneTagGenerations() // Check if the digest is now orphaned (no other tags reference it) count, err := countTagsForDigest(m.paths, repository, digestHex) @@ -870,30 +814,6 @@ func (m *manager) TotalOCICacheBytes(ctx context.Context) (int64, error) { return ociCacheBytes, nil } -func (m *manager) findRequestedTagImage(ref *NormalizedRef) *Image { - metas, err := listAllMetadata(m.paths) - if err != nil { - return nil - } - var newest *imageMetadata - for _, meta := range metas { - requestedTag := meta.RequestedTag - if requestedTag == "" { - requestedTag = strings.TrimPrefix(meta.Name, ref.Repository()+":") - } - if requestedTag != ref.Tag() || !strings.HasPrefix(meta.Name, ref.Repository()+":") { - continue - } - if newest == nil || newest.CreatedAt.Before(meta.CreatedAt) { - newest = meta - } - } - if newest == nil { - return nil - } - return newest.toImageFor(ref.String()) -} - // WaitForReady blocks until the image reaches a terminal state (ready or failed) // or the context is cancelled. func (m *manager) WaitForReady(ctx context.Context, name string) error { @@ -905,11 +825,8 @@ func (m *manager) WaitForReady(ctx context.Context, name string) error { if err != nil { return err } - if img.Status == StatusReady { - return nil - } - if img.Status == StatusFailed { - return conversionFailedErr(img.Error, nil) + if terminal, err := terminalImageError(img); terminal { + return err } digestHex := strings.TrimPrefix(img.Digest, "sha256:") @@ -922,11 +839,8 @@ func (m *manager) WaitForReady(ctx context.Context, name string) error { // Re-check after subscribing to close the race window img, err = m.GetImage(ctx, ref.Repository()+"@"+img.Digest) if err == nil { - if img.Status == StatusReady { - return nil - } - if img.Status == StatusFailed { - return conversionFailedErr(img.Error, nil) + if terminal, terminalErr := terminalImageError(img); terminal { + return terminalErr } } @@ -950,7 +864,7 @@ func (m *manager) waitForImage(ctx context.Context, name string, ref *Normalized for { var img *Image if !ref.IsDigest() { - img = m.findRequestedTagImage(ref) + img = m.requestedTagImage(ref) } if img == nil { img, lastErr = m.GetImage(ctx, name) @@ -969,6 +883,17 @@ func (m *manager) waitForImage(ctx context.Context, name string, ref *Normalized } } +func terminalImageError(img *Image) (bool, error) { + switch img.Status { + case StatusReady: + return true, nil + case StatusFailed: + return true, conversionFailedErr(img.Error, nil) + default: + return false, nil + } +} + func conversionFailedErr(message *string, cause error) error { if cause != nil { return fmt.Errorf("image conversion failed: %w", cause) diff --git a/lib/images/manager_test.go b/lib/images/manager_test.go index 8abd90a65..13a6835e2 100644 --- a/lib/images/manager_test.go +++ b/lib/images/manager_test.go @@ -734,9 +734,9 @@ func TestDeleteAndRecreateDuringBuildTail(t *testing.T) { require.NoError(t, err) staleRef := NewResolvedRef(normalized, digestStr) m.updateStatusByDigest(staleRef, StatusFailed, errors.New("stale build"), firstMeta.BuildID) - staleResult, _, _, err := m.ociClient.extractOCIImageDetails(digestHex) + staleBundle, err := m.ociClient.extractOCIImageBundle(digestHex) require.NoError(t, err) - require.ErrorIs(t, m.finalizeImage(staleRef, &pullResult{Metadata: staleResult}, 1, firstMeta.BuildID, ""), errStaleBuild) + require.ErrorIs(t, m.finalizeImage(staleRef, &pullResult{Metadata: staleBundle.Meta}, 1, firstMeta.BuildID, ""), errStaleBuild) currentMeta, err = readMetadata(p, repo, digestHex) require.NoError(t, err) require.Equal(t, StatusPending, currentMeta.Status) diff --git a/lib/images/manifest_model.go b/lib/images/manifest_model.go new file mode 100644 index 000000000..b3e7dc25b --- /dev/null +++ b/lib/images/manifest_model.go @@ -0,0 +1,166 @@ +package images + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/kernel/hypeman/lib/paths" + digest "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +// imageManifestModel is the persisted content model for one OCI manifest. It +// records everything needed to recompose the image from shared layer +// artifacts and to decide which OCI blobs are still referenced: the immutable +// manifest digest, the resolved platform, the image config with its diff ids, +// and the ordered layer descriptors. +type imageManifestModel struct { + SchemaVersion int `json:"schema_version"` + Digest string `json:"digest"` // manifest digest, sha256:... + MediaType string `json:"media_type,omitempty"` + Platform string `json:"platform"` // os/arch[/variant] + Config manifestConfigRef `json:"config"` + RootFSType string `json:"rootfs_type,omitempty"` + Layers []layerDescriptor `json:"layers"` // manifest order, base layer first +} + +const manifestModelSchemaVersion = 1 + +type manifestConfigRef struct { + Digest string `json:"digest"` // config blob digest, sha256:... + MediaType string `json:"media_type,omitempty"` + DiffIDs []string `json:"diff_ids,omitempty"` // uncompressed layer ids, matching Layers order +} + +// layerDescriptor describes one compressed layer blob exactly as it appears in +// the manifest, so the blob can be located in the shared OCI cache by digest. +type layerDescriptor struct { + Digest string `json:"digest"` // compressed blob digest, sha256:... + Size int64 `json:"size"` // compressed bytes + MediaType string `json:"media_type,omitempty"` + DiffID string `json:"diff_id,omitempty"` // uncompressed diff id from the image config +} + +// digestFromHex returns the full sha256 digest string for a bare hex value. +func digestFromHex(hex string) string { + if strings.HasPrefix(hex, "sha256:") { + return hex + } + return "sha256:" + hex +} + +// blobReferences returns every OCI blob digest the manifest depends on: the +// config blob plus all layer blobs. GC must keep these while the manifest is +// referenced. +func (m *imageManifestModel) blobReferences() []string { + refs := make([]string, 0, len(m.Layers)+1) + if m.Config.Digest != "" { + refs = append(refs, m.Config.Digest) + } + for _, layer := range m.Layers { + refs = append(refs, layer.Digest) + } + return refs +} + +func validateManifestModel(digestHex string, model *imageManifestModel) error { + if model == nil { + return fmt.Errorf("manifest model is nil") + } + if model.SchemaVersion != manifestModelSchemaVersion { + return fmt.Errorf("unsupported manifest model schema version: %d", model.SchemaVersion) + } + if model.Digest != digestFromHex(digestHex) { + return fmt.Errorf("manifest model digest %q does not match %q", model.Digest, digestFromHex(digestHex)) + } + if model.RootFSType != "" && model.RootFSType != "layers" { + return fmt.Errorf("unsupported manifest rootfs type: %q", model.RootFSType) + } + if err := validateManifestConfig(model); err != nil { + return err + } + return validateManifestLayers(model) +} + +func validateManifestConfig(model *imageManifestModel) error { + if model.Config.Digest == "" { + return fmt.Errorf("manifest model config digest is empty") + } + if _, err := parseSHA256Digest(model.Config.Digest); err != nil { + return fmt.Errorf("invalid manifest model config digest: %q", model.Config.Digest) + } + if model.Config.MediaType != "" && convertToOCIMediaType(model.Config.MediaType) != v1.MediaTypeImageConfig { + return fmt.Errorf("invalid manifest model config media type: %q", model.Config.MediaType) + } + if len(model.Config.DiffIDs) != len(model.Layers) { + return fmt.Errorf("manifest model has %d diff ids for %d layers", len(model.Config.DiffIDs), len(model.Layers)) + } + return nil +} + +func validateManifestLayers(model *imageManifestModel) error { + for i, layer := range model.Layers { + if _, err := parseSHA256Digest(layer.Digest); err != nil { + return fmt.Errorf("invalid manifest model layer %d digest: %q", i, layer.Digest) + } + diffID, err := parseSHA256Digest(model.Config.DiffIDs[i]) + if err != nil || layer.DiffID != diffID.String() { + return fmt.Errorf("invalid manifest model layer %d diff id", i) + } + if layer.Size < 0 { + return fmt.Errorf("invalid manifest model layer %d size: %d", i, layer.Size) + } + } + return nil +} + +func parseSHA256Digest(value string) (digest.Digest, error) { + parsed, err := digest.Parse(value) + if err != nil || parsed.Algorithm() != digest.SHA256 { + return "", fmt.Errorf("not a sha256 digest") + } + return parsed, nil +} + +func writeManifestModel(p *paths.Paths, digestHex string, model *imageManifestModel) error { + data, err := json.MarshalIndent(model, "", " ") + if err != nil { + return fmt.Errorf("marshal manifest model: %w", err) + } + return writeJSONAtomic(p.ImageContentManifestModel(digestHex), data) +} + +// readManifestModel loads the manifest model for a digest, if present. +// Missing models return (nil, nil): images converted before the manifest model +// existed only have a flattened rootfs. +func readManifestModel(p *paths.Paths, digestHex string) (*imageManifestModel, error) { + data, err := os.ReadFile(p.ImageContentManifestModel(digestHex)) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read manifest model: %w", err) + } + var model imageManifestModel + if err := json.Unmarshal(data, &model); err != nil { + return nil, fmt.Errorf("unmarshal manifest model: %w", err) + } + if err := validateManifestModel(digestHex, &model); err != nil { + return nil, err + } + return &model, nil +} + +// writeJSONAtomic writes data to path via a temp file in the same directory +// followed by a rename, so readers never observe a partial document. +func writeJSONAtomic(path string, data []byte) error { + if err := installAtomically(path, func(tempPath string) error { + return os.WriteFile(tempPath, data, 0o644) + }); err != nil { + return fmt.Errorf("write %s: %w", filepath.Base(path), err) + } + return nil +} diff --git a/lib/images/manifest_model_test.go b/lib/images/manifest_model_test.go new file mode 100644 index 000000000..1137e3912 --- /dev/null +++ b/lib/images/manifest_model_test.go @@ -0,0 +1,242 @@ +package images + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "io" + "os" + "strings" + "testing" + "time" + + gcr "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +// syntheticLayer builds a gzipped tar layer containing one file. +func syntheticLayer(t *testing.T, name, content string) gcr.Layer { + t.Helper() + + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: name, + Size: int64(len(content)), + Typeflag: tar.TypeReg, + Mode: 0644, + })) + _, err := tw.Write([]byte(content)) + require.NoError(t, err) + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + + data := buf.Bytes() + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(data)), nil + }) + require.NoError(t, err) + return layer +} + +// writeSyntheticLayout writes img into a fresh OCI layout cache tagged with the +// image's own digest, mirroring pullToOCILayout. +func writeSyntheticLayout(t *testing.T, img gcr.Image) (*ociClient, string) { + t.Helper() + + client, err := newOCIClient(t.TempDir()) + require.NoError(t, err) + + digest, err := img.Digest() + require.NoError(t, err) + layoutPath, err := layout.Write(client.cacheDir, empty.Index) + require.NoError(t, err) + require.NoError(t, layoutPath.AppendImage(img, layout.WithAnnotations(map[string]string{ + "org.opencontainers.image.ref.name": digestToLayoutTag(digest.String()), + }))) + return client, digestToLayoutTag(digest.String()) +} + +func TestExtractManifestModel(t *testing.T) { + base := syntheticLayer(t, "base.txt", "base layer content") + top := syntheticLayer(t, "top.txt", "top layer content") + img, err := mutate.AppendLayers(empty.Image, base, top) + require.NoError(t, err) + img = mutate.MediaType(img, types.OCIManifestSchema1) + + client, layoutTag := writeSyntheticLayout(t, img) + + bundle, err := client.extractOCIImageBundle(layoutTag) + require.NoError(t, err) + model := bundle.Model + + manifest, err := img.Manifest() + require.NoError(t, err) + configFile, err := img.ConfigFile() + require.NoError(t, err) + configDigest, err := img.ConfigName() + require.NoError(t, err) + + require.Equal(t, "sha256:"+layoutTag, model.Digest) + require.Equal(t, manifest.MediaType, types.MediaType(model.MediaType)) + require.Equal(t, configDigest.String(), model.Config.Digest) + require.Len(t, model.Layers, 2) + require.Len(t, model.Config.DiffIDs, 2) + + // Layer order and pairing must match the manifest and config diff ids. + for i, desc := range model.Layers { + require.Equal(t, manifest.Layers[i].Digest.String(), desc.Digest) + require.Equal(t, manifest.Layers[i].Size, desc.Size) + require.Equal(t, configFile.RootFS.DiffIDs[i].String(), desc.DiffID) + require.Equal(t, model.Config.DiffIDs[i], desc.DiffID) + } + + // Digests are content addresses, so identical layer content dedupes. + require.NotEqual(t, model.Layers[0].Digest, model.Layers[1].Digest) +} + +func TestExtractManifestModelPlatform(t *testing.T) { + layer := syntheticLayer(t, "file.txt", "content") + img, err := mutate.AppendLayers(empty.Image, layer) + require.NoError(t, err) + cfgFile, err := img.ConfigFile() + require.NoError(t, err) + cfgFile = cfgFile.DeepCopy() + cfgFile.OS = "linux" + cfgFile.Architecture = "amd64" + img, err = mutate.ConfigFile(img, cfgFile) + require.NoError(t, err) + + client, layoutTag := writeSyntheticLayout(t, img) + + bundle, err := client.extractOCIImageBundle(layoutTag) + require.NoError(t, err) + model := bundle.Model + require.Equal(t, "linux/amd64", model.Platform) +} + +func TestManifestModelWriteReadRoundtrip(t *testing.T) { + p := paths.New(t.TempDir()) + digestHex := "ab01ab01ab01ab01ab01ab01ab01ab01ab01ab01ab01ab01ab01ab01ab01ab01" + + model := &imageManifestModel{ + SchemaVersion: manifestModelSchemaVersion, + Digest: "sha256:" + digestHex, + Platform: "linux/amd64", + Config: manifestConfigRef{ + Digest: "sha256:" + strings.Repeat("c", 64), + DiffIDs: []string{"sha256:" + strings.Repeat("d", 64), "sha256:" + strings.Repeat("e", 64)}, + }, + Layers: []layerDescriptor{ + {Digest: "sha256:" + strings.Repeat("f", 64), Size: 10, DiffID: "sha256:" + strings.Repeat("d", 64)}, + {Digest: "sha256:" + strings.Repeat("0", 64), Size: 20, DiffID: "sha256:" + strings.Repeat("e", 64)}, + }, + } + require.NoError(t, writeManifestModel(p, digestHex, model)) + + read, err := readManifestModel(p, digestHex) + require.NoError(t, err) + require.Equal(t, model, read) + + // No leftover temp files from the atomic write. + entries, err := os.ReadDir(p.ImageContentDir(digestHex)) + require.NoError(t, err) + require.Len(t, entries, 1) + require.Equal(t, "manifest.json", entries[0].Name()) +} + +func TestReadManifestModelRejectsInvalidSchema(t *testing.T) { + p := paths.New(t.TempDir()) + digestHex := strings.Repeat("a", 64) + require.NoError(t, os.MkdirAll(p.ImageContentDir(digestHex), 0755)) + require.NoError(t, os.WriteFile(p.ImageContentManifestModel(digestHex), []byte(`{"schema_version":1,"digest":"sha256:`+digestHex+`"}`), 0644)) + _, err := readManifestModel(p, digestHex) + require.ErrorContains(t, err, "config digest is empty") +} + +func TestReadManifestModelMissing(t *testing.T) { + p := paths.New(t.TempDir()) + model, err := readManifestModel(p, "deadbeef") + require.NoError(t, err) + require.Nil(t, model, "missing model must read as nil for pre-model images") +} + +func TestManifestModelBlobReferences(t *testing.T) { + model := &imageManifestModel{ + Config: manifestConfigRef{Digest: "sha256:cfg"}, + Layers: []layerDescriptor{ + {Digest: "sha256:layer0"}, + {Digest: "sha256:layer1"}, + }, + } + require.Equal(t, []string{"sha256:cfg", "sha256:layer0", "sha256:layer1"}, model.blobReferences()) +} + +func TestImportLocalImagePersistsManifestModel(t *testing.T) { + dataDir := t.TempDir() + p := paths.New(dataDir) + mgr, err := NewManager(p, 1, nil) + require.NoError(t, err) + m := mgr.(*manager) + + ctx := context.Background() + const repo = "kernel.local/test/manifest-model" + const tag = "v1" + + testImg := createTestDockerImage(t) + imgDigest, err := testImg.Digest() + require.NoError(t, err) + digestStr := imgDigest.String() + + cacheDir := p.SystemOCICache() + layoutPath, err := layout.Write(cacheDir, empty.Index) + require.NoError(t, err) + require.NoError(t, layoutPath.AppendImage(testImg, layout.WithAnnotations(map[string]string{ + "org.opencontainers.image.ref.name": digestToLayoutTag(digestStr), + }))) + + digestHex := digestToLayoutTag(digestStr) + events := make(chan StatusEvent, 2) + m.subscribeToReady(digestHex, events) + defer m.unsubscribeFromReady(digestHex, events) + + _, err = m.ImportLocalImage(ctx, repo, tag, digestStr) + require.NoError(t, err) + select { + case event := <-events: + require.Equal(t, StatusReady, event.Status) + case <-time.After(30 * time.Second): + t.Fatal("build did not become ready") + } + + model, err := readManifestModel(p, digestHex) + require.NoError(t, err) + require.NotNil(t, model, "ready image must persist a manifest model") + require.Equal(t, "sha256:"+digestHex, model.Digest) + + configFile, err := testImg.ConfigFile() + require.NoError(t, err) + manifest, err := testImg.Manifest() + require.NoError(t, err) + require.Len(t, model.Layers, len(manifest.Layers)) + for i, desc := range model.Layers { + require.Equal(t, manifest.Layers[i].Digest.String(), desc.Digest) + require.Equal(t, configFile.RootFS.DiffIDs[i].String(), desc.DiffID) + } + require.Equal(t, StatusReady, mustReadContentStatus(t, p, digestHex)) +} + +func mustReadContentStatus(t *testing.T, p *paths.Paths, digestHex string) string { + t.Helper() + meta, err := readContentMetadata(p, digestHex) + require.NoError(t, err) + return meta.Status +} diff --git a/lib/images/oci.go b/lib/images/oci.go index 33206d134..2a1a23223 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -194,6 +194,7 @@ func (c *ociClient) inspectDigestPlatformAuth(ctx context.Context, imageRef stri // pullResult contains the metadata and digest from pulling an image type pullResult struct { Metadata *containerMetadata + Manifest *imageManifestModel Digest string // sha256:abc123... CacheHit bool LayerCount int @@ -207,6 +208,14 @@ type imageBuildPhaseMeasurement struct { Status string } +// ociImageBundle is the extracted content of one OCI image in the cache. +type ociImageBundle struct { + Meta *containerMetadata + Model *imageManifestModel + LayerCount int + CompressedBytes int64 +} + func (r *pullResult) measure(phase string, operation func() error) error { start := time.Now() err := operation() @@ -260,20 +269,19 @@ func (c *ociClient) pullAndExportWithPlatformAuth(ctx context.Context, imageRef, // If cached, we skip the pull entirely // Extract metadata (from cache or freshly pulled) - var meta *containerMetadata - var layerCount int - var compressedBytes int64 + var bundle *ociImageBundle err := result.measure("metadata_extract", func() error { var err error - meta, layerCount, compressedBytes, err = c.extractOCIImageDetails(layoutTag) + bundle, err = c.extractOCIImageBundle(layoutTag) return err }) if err != nil { return result, fmt.Errorf("extract metadata: %w", err) } - result.Metadata = meta - result.LayerCount = layerCount - result.CompressedBytes = compressedBytes + result.Metadata = bundle.Meta + result.Manifest = bundle.Model + result.LayerCount = bundle.LayerCount + result.CompressedBytes = bundle.CompressedBytes // Unpack layers to the export directory if err := result.measure("layer_unpack", func() error { @@ -395,44 +403,31 @@ func imageByAnnotation(path layout.Path, layoutTag string) (gcr.Image, error) { return nil, fmt.Errorf("no image found with tag %s", layoutTag) } -// extractOCIMetadata reads metadata from OCI layout config.json -// Uses go-containerregistry which handles both Docker v2 and OCI v1 manifests. -func (c *ociClient) extractOCIMetadata(layoutTag string) (*containerMetadata, error) { - meta, _, _, err := c.extractOCIImageDetails(layoutTag) - return meta, err -} - -func (c *ociClient) extractOCIImageDetails(layoutTag string) (*containerMetadata, int, int64, error) { - // Open OCI layout using go-containerregistry (handles Docker v2 and OCI v1) +// extractOCIImageBundle reads metadata, the manifest content model, and layer +// stats from the cached OCI layout. Uses go-containerregistry which handles +// both Docker v2 and OCI v1 manifests. +func (c *ociClient) extractOCIImageBundle(layoutTag string) (*ociImageBundle, error) { path, err := layout.FromPath(c.cacheDir) if err != nil { - return nil, 0, 0, fmt.Errorf("open oci layout: %w", err) + return nil, fmt.Errorf("open oci layout: %w", err) } - - // Get the image by annotation tag from the layout img, err := imageByAnnotation(path, layoutTag) if err != nil { - return nil, 0, 0, fmt.Errorf("find image by tag %s: %w", layoutTag, err) + return nil, fmt.Errorf("find image by tag %s: %w", layoutTag, err) } - - // Get config file (go-containerregistry handles manifest format automatically) configFile, err := img.ConfigFile() if err != nil { - return nil, 0, 0, fmt.Errorf("get config file: %w", err) + return nil, fmt.Errorf("get config file: %w", err) } - manifest, err := img.Manifest() if err != nil { - return nil, 0, 0, fmt.Errorf("get manifest: %w", err) + return nil, fmt.Errorf("get manifest: %w", err) } - var compressedBytes int64 - for _, layer := range manifest.Layers { - compressedBytes += layer.Size + configDigest, err := img.ConfigName() + if err != nil { + return nil, fmt.Errorf("get config digest: %w", err) } - // Extract metadata from config. OS/Architecture/Variant come straight from - // the pulled image config, so they reflect the manifest actually fetched - // rather than what the caller requested. meta := &containerMetadata{ OS: configFile.OS, Architecture: configFile.Architecture, @@ -443,7 +438,6 @@ func (c *ociClient) extractOCIImageDetails(layoutTag string) (*containerMetadata Labels: make(map[string]string), WorkingDir: configFile.Config.WorkingDir, } - // Parse environment variables for _, env := range configFile.Config.Env { for i := 0; i < len(env); i++ { @@ -455,12 +449,49 @@ func (c *ociClient) extractOCIImageDetails(layoutTag string) (*containerMetadata } } } - for key, value := range configFile.Config.Labels { meta.Labels[key] = value } - return meta, len(manifest.Layers), compressedBytes, nil + model := &imageManifestModel{ + SchemaVersion: manifestModelSchemaVersion, + Digest: digestFromHex(layoutTag), + MediaType: string(manifest.MediaType), + Platform: Platform{ + OS: configFile.OS, + Architecture: configFile.Architecture, + Variant: configFile.Variant, + }.String(), + Config: manifestConfigRef{ + Digest: configDigest.String(), + MediaType: string(manifest.Config.MediaType), + DiffIDs: make([]string, 0, len(configFile.RootFS.DiffIDs)), + }, + RootFSType: configFile.RootFS.Type, + Layers: make([]layerDescriptor, 0, len(manifest.Layers)), + } + for _, diffID := range configFile.RootFS.DiffIDs { + model.Config.DiffIDs = append(model.Config.DiffIDs, diffID.String()) + } + var compressedBytes int64 + for i, descriptor := range manifest.Layers { + compressedBytes += descriptor.Size + layer := layerDescriptor{ + Digest: descriptor.Digest.String(), + Size: descriptor.Size, + MediaType: string(descriptor.MediaType), + } + if i < len(model.Config.DiffIDs) { + layer.DiffID = model.Config.DiffIDs[i] + } + model.Layers = append(model.Layers, layer) + } + return &ociImageBundle{ + Meta: meta, + Model: model, + LayerCount: len(manifest.Layers), + CompressedBytes: compressedBytes, + }, nil } // unpackLayers unpacks all OCI layers to a target directory using umoci diff --git a/lib/images/oci_test.go b/lib/images/oci_test.go index e076156ff..2cb765cc5 100644 --- a/lib/images/oci_test.go +++ b/lib/images/oci_test.go @@ -89,11 +89,11 @@ func TestExtractMetadataSucceedsOnBuildKitCache(t *testing.T) { // This succeeds because go-containerregistry doesn't validate config mediatype // The failure only happens in unpackLayers when umoci validates the config - meta, err := client.extractOCIMetadata("test-cache") - require.NoError(t, err, "extractOCIMetadata succeeds - go-containerregistry is lenient") + bundle, err := client.extractOCIImageBundle("test-cache") + require.NoError(t, err, "extractOCIImageBundle succeeds - go-containerregistry is lenient") // But the metadata will be empty/invalid since it's not a real OCI config - t.Logf("Got metadata (likely empty): %+v", meta) + t.Logf("Got metadata (likely empty): %+v", bundle.Meta) } // createBuildKitCacheLayout creates an OCI layout that mimics what BuildKit @@ -337,9 +337,10 @@ func TestDockerSaveTarballToOCILayoutRoundtrip(t *testing.T) { require.NoError(t, err) assert.True(t, client.existsInLayout(layoutTag), "image should exist in layout after AppendImage") - // Step 6: Verify extractOCIMetadata reads correct config - meta, err := client.extractOCIMetadata(layoutTag) + // Step 6: Verify extractOCIImageBundle reads correct config + bundle, err := client.extractOCIImageBundle(layoutTag) require.NoError(t, err) + meta := bundle.Meta assert.Equal(t, []string{"/usr/local/bin/guest-agent"}, meta.Entrypoint) assert.Equal(t, "/app", meta.WorkingDir) assert.Contains(t, meta.Env, "PATH") diff --git a/lib/images/storage.go b/lib/images/storage.go index 65d433c47..91ce646f7 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -14,53 +14,29 @@ import ( ) type imageMetadata struct { - Name string `json:"name"` // Normalized ref (tag or digest) - Digest string `json:"digest"` // Always present: sha256:... - Platform string `json:"platform,omitempty"` - Status string `json:"status"` - Error *string `json:"error,omitempty"` - Request *CreateImageRequest `json:"request,omitempty"` - SizeBytes int64 `json:"size_bytes"` - Entrypoint []string `json:"entrypoint,omitempty"` - Cmd []string `json:"cmd,omitempty"` - Env map[string]string `json:"env,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - Tags tags.Tags `json:"tags,omitempty"` - WorkingDir string `json:"working_dir,omitempty"` - CreatedAt time.Time `json:"created_at"` - BorrowedAuth bool `json:"borrowed_auth,omitempty"` - BuildID string `json:"build_id,omitempty"` - RequestedTag string `json:"requested_tag,omitempty"` - PreviousTagDigest string `json:"previous_tag_digest,omitempty"` - TagGeneration uint64 `json:"tag_generation,omitempty"` - References map[string]tags.Tags `json:"references,omitempty"` - TagClaims []imageTagClaim `json:"tag_claims,omitempty"` -} - -type imageTagClaim struct { - Repository string `json:"repository"` - Tag string `json:"tag"` - PreviousTagDigest string `json:"previous_tag_digest,omitempty"` - TagGeneration uint64 `json:"tag_generation,omitempty"` -} - -func referenceTags(meta *imageMetadata, reference string) (tags.Tags, bool) { - resourceTags, ok := meta.References[reference] - return tags.Clone(resourceTags), ok -} - -func setReferenceTags(meta *imageMetadata, reference string, resourceTags tags.Tags) { - if meta.References == nil { - meta.References = make(map[string]tags.Tags) - } - meta.References[reference] = tags.Clone(resourceTags) + Name string `json:"name"` // Normalized ref (tag or digest) + Digest string `json:"digest"` // Always present: sha256:... + Platform string `json:"platform,omitempty"` + Status string `json:"status"` + Error *string `json:"error,omitempty"` + Request *CreateImageRequest `json:"request,omitempty"` + SizeBytes int64 `json:"size_bytes"` + Entrypoint []string `json:"entrypoint,omitempty"` + Cmd []string `json:"cmd,omitempty"` + Env map[string]string `json:"env,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Tags tags.Tags `json:"tags,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + CreatedAt time.Time `json:"created_at"` + BorrowedAuth bool `json:"borrowed_auth,omitempty"` + BuildID string `json:"build_id,omitempty"` + RequestedTag string `json:"requested_tag,omitempty"` + PreviousTagDigest string `json:"previous_tag_digest,omitempty"` + TagGeneration uint64 `json:"tag_generation,omitempty"` } func (m *imageMetadata) toImageFor(reference string) *Image { img := m.toImage() - if resourceTags, ok := referenceTags(m, reference); ok { - img.Tags = resourceTags - } img.Name = reference return img } @@ -216,24 +192,11 @@ func writeMetadata(p *paths.Paths, repository, digestHex string, meta *imageMeta } func writeMetadataFile(path string, meta *imageMetadata) error { - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - return fmt.Errorf("create metadata directory: %w", err) - } - data, err := json.MarshalIndent(meta, "", " ") if err != nil { return fmt.Errorf("marshal metadata: %w", err) } - - tempPath := path + ".tmp" - if err := os.WriteFile(tempPath, data, 0644); err != nil { - return fmt.Errorf("write temp metadata: %w", err) - } - if err := os.Rename(tempPath, path); err != nil { - _ = os.Remove(tempPath) - return fmt.Errorf("rename metadata: %w", err) - } - return nil + return writeJSONAtomic(path, data) } func readMetadata(p *paths.Paths, repository, digestHex string) (*imageMetadata, error) { diff --git a/lib/images/storage_refs.go b/lib/images/storage_refs.go index ac1864c2f..bc25b5eda 100644 --- a/lib/images/storage_refs.go +++ b/lib/images/storage_refs.go @@ -47,12 +47,13 @@ func listTags(p *paths.Paths, repository string) ([]string, error) { return tags, nil } -func promoteLegacyImages(p *paths.Paths) { +type legacyRef struct { + repository string + digestHex string +} + +func collectLegacyImages(p *paths.Paths) ([]legacyRef, error) { imagesDir := p.ImagesDir() - type legacyRef struct { - repository string - digestHex string - } refs := make([]legacyRef, 0) err := filepath.Walk(imagesDir, func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() || info.Name() != "metadata.json" { @@ -73,10 +74,12 @@ func promoteLegacyImages(p *paths.Paths) { return nil }) if err != nil && !os.IsNotExist(err) { - fmt.Fprintf(os.Stderr, "Warning: failed to scan legacy images for promotion: %v\n", err) - return + return nil, err } + return refs, nil +} +func promoteLegacyImages(p *paths.Paths, refs []legacyRef) { for _, ref := range refs { layout := resolveImageLayout(p, ref.repository, ref.digestHex) meta, readErr := readMetadataAt(layout) diff --git a/lib/images/storage_test.go b/lib/images/storage_test.go index b34c47d54..d7c8fd994 100644 --- a/lib/images/storage_test.go +++ b/lib/images/storage_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/kernel/hypeman/lib/paths" - "github.com/kernel/hypeman/lib/tags" "github.com/stretchr/testify/require" ) @@ -162,79 +161,6 @@ func TestContentLayoutResolvesDiskByDigest(t *testing.T) { require.Equal(t, p.ImageContentPath(digest), got) } -func TestSharedContentKeepsReferenceTags(t *testing.T) { - p := paths.New(t.TempDir()) - digest := "abababababababababababababababababababababababababababababababab" - first := "docker.io/library/alpine:latest" - second := "registry.example.com/app:v1" - meta := &imageMetadata{ - Name: first, - Digest: "sha256:" + digest, - Status: StatusReady, - References: map[string]tags.Tags{ - first: {"team": "one"}, - second: {"team": "two"}, - }, - } - require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digest), meta)) - require.NoError(t, os.WriteFile(p.ImageContentPath(digest), []byte("rootfs"), 0o644)) - require.NoError(t, createTagSymlink(p, "docker.io/library/alpine", "latest", digest)) - require.NoError(t, createTagSymlink(p, "registry.example.com/app", "v1", digest)) - - mgr := &manager{paths: p} - image, err := mgr.GetImage(nil, second) - require.NoError(t, err) - require.Equal(t, tags.Tags{"team": "two"}, image.Tags) - - images, err := mgr.ListImages(nil) - require.NoError(t, err) - got := make(map[string]tags.Tags, len(images)) - for _, image := range images { - got[image.Name] = image.Tags - } - require.Equal(t, map[string]tags.Tags{ - first: {"team": "one"}, - second: {"team": "two"}, - }, got) -} - -func TestPendingTagClaimSurvivesSharedBuild(t *testing.T) { - p := paths.New(t.TempDir()) - const repository = "docker.io/library/alpine" - const otherRepository = "registry.example.com/app" - const tag = "v1" - const digest = "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd" - const previous = "efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef" - - require.NoError(t, createTagSymlink(p, otherRepository, tag, previous)) - meta := &imageMetadata{ - Name: repository + ":latest", Digest: "sha256:" + digest, - Status: StatusPending, RequestedTag: "latest", - } - normalized, err := ParseNormalizedRef(otherRepository + ":" + tag) - require.NoError(t, err) - m := &manager{paths: p, tagGenerations: make(map[string]uint64)} - ref := NewResolvedRef(normalized, "sha256:"+digest) - require.NoError(t, m.recordPendingTag(meta, ref)) - require.Len(t, meta.TagClaims, 1) - claim := meta.TagClaims[0] - require.Equal(t, previous, claim.PreviousTagDigest) - require.Equal(t, previous, mustResolveTag(t, p, otherRepository, tag)) - - meta.Status = StatusReady - require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digest), meta)) - require.NoError(t, os.WriteFile(p.ImageContentPath(digest), []byte("rootfs"), 0o644)) - m.claimTag(claim.Repository, digest, claim.Tag, claim.PreviousTagDigest, claim.TagGeneration, false) - require.Equal(t, digest, mustResolveTag(t, p, otherRepository, tag)) -} - -func mustResolveTag(t *testing.T, p *paths.Paths, repository, tag string) string { - t.Helper() - resolved, err := resolveTag(p, repository, tag) - require.NoError(t, err) - return resolved -} - func TestListAllMetadataContentLayout(t *testing.T) { p := paths.New(t.TempDir()) digest := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" @@ -266,12 +192,46 @@ func TestListAllMetadataContentLayout(t *testing.T) { }, names) } +func TestImageMetadataToImage_ClonesMetadata(t *testing.T) { + createdAt := time.Now().UTC().Truncate(time.Second) + source := &imageMetadata{ + Name: "docker.io/library/alpine:latest", + Digest: "sha256:abc", + Status: StatusReady, + Tags: map[string]string{"team": "backend", "env": "staging"}, + SizeBytes: 123, + CreatedAt: createdAt, + } + + img := source.toImageFor(source.Name) + require.Equal(t, source.Name, img.Name) + require.Equal(t, source.Digest, img.Digest) + require.Equal(t, map[string]string{"team": "backend", "env": "staging"}, img.Tags) + require.NotNil(t, img.SizeBytes) + require.Equal(t, int64(123), *img.SizeBytes) + + source.Tags["team"] = "mutated" + require.Equal(t, "backend", img.Tags["team"]) +} + +func TestImageMetadataToImage_EmptyMetadataOmitted(t *testing.T) { + img := (&imageMetadata{ + Name: "docker.io/library/alpine:latest", + Digest: "sha256:abc", + Status: StatusPending, + CreatedAt: time.Now().UTC(), + }).toImageFor("docker.io/library/alpine:latest") + + require.Nil(t, img.Tags) +} + func TestPromoteLegacyImagesMovesContentAndTags(t *testing.T) { p := paths.New(t.TempDir()) repository := "docker.io/library/alpine" tag := "latest" digest := "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + // Ready legacy image with a legacy tag symlink. legacyDir := p.ImageDigestDir(repository, digest) require.NoError(t, os.MkdirAll(legacyDir, 0o755)) meta := &imageMetadata{ @@ -289,6 +249,7 @@ func TestPromoteLegacyImagesMovesContentAndTags(t *testing.T) { promoteLegacyImages(p) + // Content exists with the same bytes and is ready. contentMeta, err := readContentMetadata(p, digest) require.NoError(t, err) require.Equal(t, StatusReady, contentMeta.Status) @@ -298,6 +259,7 @@ func TestPromoteLegacyImagesMovesContentAndTags(t *testing.T) { require.FileExists(t, p.ImageDigestPath(repository, digest)) + // The tag now resolves through the shared content layout. resolved, err := resolveTag(p, repository, tag) require.NoError(t, err) require.Equal(t, digest, resolved) diff --git a/lib/images/tag.go b/lib/images/tag.go deleted file mode 100644 index 9b092f595..000000000 --- a/lib/images/tag.go +++ /dev/null @@ -1,110 +0,0 @@ -package images - -import ( - "context" - "errors" - "fmt" - "log/slog" -) - -// TagImage creates a ready-image tag without pulling or converting content. -// Cross-repository tags promote legacy content into the shared layout. A -// failed call leaves no side effects: the target tag's generation is only -// bumped after the new tag is on disk, so pending pulls that claimed the -// target tag keep their claim. When the target previously pointed at -// different content, that digest is collected after the new tag is live; -// cleanup failures are logged and do not fail the call, since the tag is -// already installed at that point. -// -// Promotion deliberately runs before the symlink install, so a symlink -// failure after a cross-repo promotion leaves the content promoted with no -// target tag. That state is gc-consistent (unreferenced content is -// collected) and retry is idempotent (promoteImageToContent short-circuits -// on ready content), which beats rolling back a completed promotion. -func (m *manager) TagImage(ctx context.Context, source, target string) (*Image, error) { - sourceRef, targetRef, err := parseTagReferences(source, target) - if err != nil { - return nil, err - } - - m.createMu.Lock() - defer m.createMu.Unlock() - - digestHex, meta, err := m.readyTagImage(sourceRef) - if err != nil { - return nil, err - } - - // A dangling or malformed target symlink is treated like a missing tag so - // the retag self-heals; createTagSymlink replaces the link either way. - previousDigest, err := resolveTag(m.paths, targetRef.Repository(), targetRef.Tag()) - if err != nil && !errors.Is(err, ErrNotFound) && !errors.Is(err, errInvalidSymlinkTarget) { - return nil, fmt.Errorf("resolve existing target tag: %w", err) - } - if sourceRef.Repository() != targetRef.Repository() { - if err := promoteImageToContent(m.paths, sourceRef.Repository(), digestHex, meta); err != nil { - return nil, fmt.Errorf("promote image to content: %w", err) - } - } - if err := createTagSymlink(m.paths, targetRef.Repository(), targetRef.Tag(), digestHex); err != nil { - return nil, fmt.Errorf("create image tag: %w", err) - } - setReferenceTags(meta, targetRef.String(), nil) - if err := writeMetadata(m.paths, targetRef.Repository(), digestHex, meta); err != nil { - return nil, fmt.Errorf("write tagged image metadata: %w", err) - } - // Unlike updateExistingReference, which bumps the generation before - // installing the symlink, the bump happens after the install here so a - // failed tag call cannot invalidate a pending pull's claim on the target - // tag (pinned by TestTagImageFailureLeavesNoSideEffects). - m.nextTagGeneration(targetRef.Repository(), targetRef.Tag()) - m.cleanupReplacedTag(targetRef, previousDigest, digestHex) - - return meta.toImageFor(targetRef.String()), nil -} - -func (m *manager) cleanupReplacedTag(ref *NormalizedRef, previousDigest, digestHex string) { - if previousDigest == "" || previousDigest == digestHex { - return - } - // Sibling tags in this repository may still reference the previous - // digest; only collect when this was the last reference. - count, err := countTagsForDigest(m.paths, ref.Repository(), previousDigest) - if err != nil { - slog.Warn("failed to count tags for replaced image", "repository", ref.Repository(), "digest", previousDigest, "error", err) - return - } - if count > 0 { - return - } - if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), previousDigest, true); err != nil { - slog.Warn("failed to collect replaced image content", "repository", ref.Repository(), "digest", previousDigest, "error", err) - } - m.refreshDiskUsageTotals() -} - -func parseTagReferences(source, target string) (*NormalizedRef, *NormalizedRef, error) { - sourceRef, err := ParseNormalizedRef(source) - if err != nil { - return nil, nil, fmt.Errorf("%w: invalid source reference: %s", ErrInvalidName, err) - } - targetRef, err := ParseNormalizedRef(target) - if err != nil { - return nil, nil, fmt.Errorf("%w: invalid target reference: %s", ErrInvalidName, err) - } - if targetRef.IsDigest() { - return nil, nil, fmt.Errorf("%w: target must be a tag reference, not a digest", ErrInvalidName) - } - return sourceRef, targetRef, nil -} - -func (m *manager) readyTagImage(ref *NormalizedRef) (string, *imageMetadata, error) { - digestHex, meta, err := resolveRefMetadata(m.paths, ref) - if err != nil { - return "", nil, err - } - if meta.Status != StatusReady { - return "", nil, fmt.Errorf("%w: %s", ErrImageNotReady, meta.Status) - } - return digestHex, meta, nil -} diff --git a/lib/images/tag_manager.go b/lib/images/tag_manager.go new file mode 100644 index 000000000..a700e59c1 --- /dev/null +++ b/lib/images/tag_manager.go @@ -0,0 +1,272 @@ +package images + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "strings" +) + +func tagGenerationKey(repository, tag string) string { + return repository + ":" + tag +} + +// nextTagGeneration bumps the mutation generation for a tag and returns the +// new value. Pulls record the value with their metadata and only repoint the +// tag on completion when no later operation mutated it. +func (m *manager) nextTagGeneration(repository, tag string) uint64 { + if m.tagGenerations == nil { + m.tagGenerations = make(map[string]uint64) + } + key := tagGenerationKey(repository, tag) + m.tagGenerations[key]++ + return m.tagGenerations[key] +} + +// revertTagGeneration undoes a nextTagGeneration bump for an operation that +// failed before mutating the tag, so an in-flight pull of the tag is not +// permanently blocked from repointing it. +func (m *manager) revertTagGeneration(repository, tag string) { + key := tagGenerationKey(repository, tag) + if m.tagGenerations[key] <= 1 { + delete(m.tagGenerations, key) + return + } + m.tagGenerations[key]-- +} + +// releaseTagGeneration drops a failed pull's claim on its tag so an older +// in-flight pull of the same tag can still repoint it. +func (m *manager) releaseTagGeneration(repository, tag string, generation uint64) { + if m.tagGenerations[tagGenerationKey(repository, tag)] == generation { + m.revertTagGeneration(repository, tag) + } +} + +// pruneTagGenerations drops entries for tags that no longer resolve, keeping +// the map bounded over the process lifetime. Deleting a tag still suppresses +// an in-flight pull's repoint: the pull's recorded generation can no longer +// match the missing entry. +func (m *manager) pruneTagGenerations() { + for key := range m.tagGenerations { + // tagGenerationKey is repo+":"+tag and tags never contain ":", but + // repositories may (host:port), so split on the last colon. + colon := strings.LastIndexByte(key, ':') + if colon < 0 { + continue + } + if _, err := resolveTag(m.paths, key[:colon], key[colon+1:]); err != nil { + delete(m.tagGenerations, key) + } + } +} + +// restoreTagState re-seeds the tag indexes from recovered metadata. metas must +// be sorted oldest first so the newest requested pull wins. +func (m *manager) restoreTagState(metas []*imageMetadata) { + if m.tagGenerations == nil { + m.tagGenerations = make(map[string]uint64) + } + if m.requestedTags == nil { + m.requestedTags = make(map[string]string) + } + for _, meta := range metas { + if meta.RequestedTag == "" || meta.Digest == "" { + continue + } + ref, err := ParseNormalizedRef(meta.Name) + if err != nil { + continue + } + key := tagGenerationKey(ref.Repository(), meta.RequestedTag) + if meta.TagGeneration > m.tagGenerations[key] { + m.tagGenerations[key] = meta.TagGeneration + } + m.requestedTags[key] = strings.TrimPrefix(meta.Digest, "sha256:") + } +} + +// claimTagForStatus repoints ref's tag at the digest when the image is ready, +// or records a pending tag when the build is still in flight. It is a no-op +// for digest-only references. +func (m *manager) claimTagForStatus(meta *imageMetadata, ref *ResolvedRef) error { + if ref.Tag() == "" { + return nil + } + if meta.Status == StatusReady { + return m.claimReadyTag(ref.Repository(), ref.Tag(), ref.DigestHex()) + } + return ensurePendingTag(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()) +} + +// claimReadyTag repoints an existing tag at a ready digest, last pull wins. +func (m *manager) claimReadyTag(repository, tag, digestHex string) error { + m.nextTagGeneration(repository, tag) + if err := createTagSymlink(m.paths, repository, tag, digestHex); err != nil { + m.revertTagGeneration(repository, tag) + return err + } + return nil +} + +// trackRequestedTag records the digest of the newest pull requested for a tag +// so readiness waits can find it without walking the metadata tree. +func (m *manager) trackRequestedTag(repository, tag, digestHex string) { + if m.requestedTags == nil { + m.requestedTags = make(map[string]string) + } + m.requestedTags[tagGenerationKey(repository, tag)] = digestHex +} + +// requestedTagImage returns the newest image requested for a tag, so a +// readiness wait tracks the latest pull rather than the digest the tag +// currently points at. +func (m *manager) requestedTagImage(ref *NormalizedRef) *Image { + m.createMu.Lock() + digestHex, ok := m.requestedTags[tagGenerationKey(ref.Repository(), ref.Tag())] + m.createMu.Unlock() + if !ok { + return nil + } + meta, err := readMetadata(m.paths, ref.Repository(), digestHex) + if err != nil { + return nil + } + return meta.toImageFor(ref.String()) +} + +// claimRequestedTag repoints the pull's requested tag at the finished digest. +// The tag is only repointed when its generation still matches the pull's +// recorded claim, so a later mutation of the tag wins over the in-flight +// pull. A recovered build whose metadata predates requested-tag tracking has +// no recorded tag: fall back to the reference's own tag and recreate a +// missing symlink, matching the pre-tracking behavior. Otherwise an +// already-missing tag stays missing so a concurrent delete wins. +func (m *manager) claimRequestedTag(ref *ResolvedRef, meta *imageMetadata) bool { + requestedTag := meta.RequestedTag + allowMissing := requestedTag == "" + if requestedTag == "" { + requestedTag = ref.Tag() + } + if requestedTag == "" || m.tagGenerations[tagGenerationKey(ref.Repository(), requestedTag)] != meta.TagGeneration { + return false + } + current, err := resolveTag(m.paths, ref.Repository(), requestedTag) + if err != nil { + if !allowMissing || !errors.Is(err, ErrNotFound) { + return false + } + } else if current != ref.DigestHex() && current != meta.PreviousTagDigest { + return false + } + if err := createTagSymlink(m.paths, ref.Repository(), requestedTag, ref.DigestHex()); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to create tag symlink: %v\n", err) + return false + } + return true +} + +// TagImage creates a ready-image tag without pulling or converting content. +// Cross-repository tags promote legacy content into the shared layout. A +// failed call leaves no side effects: the target tag's generation is only +// bumped after the new tag is on disk, so pending pulls that claimed the +// target tag keep their claim. When the target previously pointed at +// different content, that digest is collected after the new tag is live; +// cleanup failures are logged and do not fail the call, since the tag is +// already installed at that point. +// +// Promotion deliberately runs before the symlink install, so a symlink +// failure after a cross-repo promotion leaves the content promoted with no +// target tag. That state is gc-consistent (unreferenced content is +// collected) and retry is idempotent (promoteImageToContent short-circuits +// on ready content), which beats rolling back a completed promotion. +func (m *manager) TagImage(ctx context.Context, source, target string) (*Image, error) { + sourceRef, targetRef, err := parseTagReferences(source, target) + if err != nil { + return nil, err + } + + m.createMu.Lock() + defer m.createMu.Unlock() + + // A dangling or malformed target symlink is treated like a missing tag so + // the retag self-heals; createTagSymlink replaces the link either way. + previousDigest, err := resolveTag(m.paths, targetRef.Repository(), targetRef.Tag()) + if err != nil && !errors.Is(err, ErrNotFound) && !errors.Is(err, errInvalidSymlinkTarget) { + return nil, fmt.Errorf("resolve existing target tag: %w", err) + } + + digestHex, meta, err := m.readyTagImage(sourceRef) + if err != nil { + return nil, err + } + if sourceRef.Repository() != targetRef.Repository() { + if err := promoteImageToContent(m.paths, sourceRef.Repository(), digestHex, meta); err != nil { + return nil, fmt.Errorf("promote image to content: %w", err) + } + } + if err := createTagSymlink(m.paths, targetRef.Repository(), targetRef.Tag(), digestHex); err != nil { + return nil, fmt.Errorf("create image tag: %w", err) + } + if err := writeMetadata(m.paths, targetRef.Repository(), digestHex, meta); err != nil { + return nil, fmt.Errorf("write tagged image metadata: %w", err) + } + // Unlike updateExistingReference, which bumps the generation before + // installing the symlink, the bump happens after the install here so a + // failed tag call cannot invalidate a pending pull's claim on the target + // tag (pinned by TestTagImageFailureLeavesNoSideEffects). + m.nextTagGeneration(targetRef.Repository(), targetRef.Tag()) + m.cleanupReplacedTag(targetRef, previousDigest, digestHex) + + return meta.toImageFor(targetRef.String()), nil +} + +func (m *manager) cleanupUnclaimedImage(ref *ResolvedRef) { + if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), ref.DigestHex(), true); err != nil { + slog.Warn("failed to collect stale image", "repository", ref.Repository(), "digest", ref.DigestHex(), "error", err) + } +} + +func (m *manager) cleanupReplacedTag(ref *NormalizedRef, previousDigest, digestHex string) { + if previousDigest != "" && previousDigest != digestHex { + // Sibling tags in this repository may still reference the previous + // digest; only collect when this was the last reference. + count, err := countTagsForDigest(m.paths, ref.Repository(), previousDigest) + if err != nil { + slog.Warn("failed to count tags for replaced image", "repository", ref.Repository(), "digest", previousDigest, "error", err) + } else if count == 0 { + if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), previousDigest, true); err != nil { + slog.Warn("failed to collect replaced image content", "repository", ref.Repository(), "digest", previousDigest, "error", err) + } + m.refreshDiskUsageTotals() + } + } +} + +func parseTagReferences(source, target string) (*NormalizedRef, *NormalizedRef, error) { + sourceRef, err := ParseNormalizedRef(source) + if err != nil { + return nil, nil, fmt.Errorf("%w: invalid source reference: %s", ErrInvalidName, err) + } + targetRef, err := ParseNormalizedRef(target) + if err != nil { + return nil, nil, fmt.Errorf("%w: invalid target reference: %s", ErrInvalidName, err) + } + if targetRef.IsDigest() { + return nil, nil, fmt.Errorf("%w: target must be a tag reference, not a digest", ErrInvalidName) + } + return sourceRef, targetRef, nil +} + +func (m *manager) readyTagImage(ref *NormalizedRef) (string, *imageMetadata, error) { + digestHex, meta, err := resolveRefMetadata(m.paths, ref) + if err != nil { + return "", nil, err + } + if meta.Status != StatusReady { + return "", nil, fmt.Errorf("%w: %s", ErrImageNotReady, meta.Status) + } + return digestHex, meta, nil +} diff --git a/lib/images/tag_test.go b/lib/images/tag_test.go index c91f13f5b..75796ab2f 100644 --- a/lib/images/tag_test.go +++ b/lib/images/tag_test.go @@ -8,7 +8,6 @@ import ( "time" "github.com/kernel/hypeman/lib/paths" - "github.com/kernel/hypeman/lib/tags" "github.com/stretchr/testify/require" ) @@ -89,32 +88,6 @@ func TestTagImageAliasesReadyImage(t *testing.T) { } } -func TestTagImagePersistsTargetReference(t *testing.T) { - p, m, repository := newTagTestCase(t) - digest := "f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1" - seedContent(t, p, repository, "latest", digest) - - meta, err := readContentMetadata(p, digest) - require.NoError(t, err) - meta.Tags = tags.Tags{"source": "tag"} - meta.References = map[string]tags.Tags{ - repository + ":latest": {"source": "tag"}, - } - require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digest), meta)) - - target := repository + ":stable" - tagImage(t, m, repository+":latest", target, digest) - - image, err := m.GetImage(context.Background(), target) - require.NoError(t, err) - require.Empty(t, image.Tags) - - meta, err = readContentMetadata(p, digest) - require.NoError(t, err) - _, ok := meta.References[target] - require.True(t, ok, "tagging must persist an explicit target reference") -} - func TestTagImageSameRepositoryDeletesContent(t *testing.T) { p, m, repository := newTagTestCase(t) digest := "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" @@ -180,7 +153,8 @@ func TestStaleTagClaimCollectsContent(t *testing.T) { meta.RequestedTag = "latest" meta.TagGeneration = 1 - m.claimImageTags(ref, meta) + require.False(t, m.claimRequestedTag(ref, meta)) + m.cleanupUnclaimedImage(ref) _, err = os.Stat(p.ImageContentDir(digest)) require.ErrorIs(t, err, os.ErrNotExist) diff --git a/lib/paths/paths.go b/lib/paths/paths.go index 814dc1432..bc9403cd6 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -161,6 +161,12 @@ func (p *Paths) ImageContentMetadata(digestHex string) string { return filepath.Join(p.ImageContentDir(digestHex), "metadata.json") } +// ImageContentManifestModel returns the path to the persisted OCI manifest +// model (layer descriptors, config, platform) for content-addressed image data. +func (p *Paths) ImageContentManifestModel(digestHex string) string { + return filepath.Join(p.ImageContentDir(digestHex), "manifest.json") +} + // ImageRepositoriesDir returns the root directory for repository tag references. func (p *Paths) ImageRepositoriesDir() string { return filepath.Join(p.dataDir, "images", "repositories") From a6e571e8e62df0facce8bad33032cb2b6a52ca7e Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:15:57 +0000 Subject: [PATCH 05/13] Simplify image tag management flow --- lib/images/manager.go | 22 +---- lib/images/storage_test.go | 10 ++- lib/images/tag_manager.go | 173 +++++++++++++++---------------------- 3 files changed, 81 insertions(+), 124 deletions(-) diff --git a/lib/images/manager.go b/lib/images/manager.go index f4c547886..fb202f273 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -254,27 +254,9 @@ func (m *manager) ImportLocalImage(ctx context.Context, repo, reference, digest m.createMu.Lock() defer m.createMu.Unlock() - // Check if we already have this digest (deduplication) - if meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()); err == nil { - // Don't cache failed builds - allow retry by falling through to - // re-queue the build. - if meta.Status == StatusFailed { - if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), ref.DigestHex(), false); err != nil { - return nil, fmt.Errorf("remove failed image: %w", err) - } - } else { - if err := m.claimTagForStatus(meta, ref); err != nil { - return nil, fmt.Errorf("create image tag: %w", err) - } - img := meta.toImageFor(ref.String()) - if meta.Status == StatusPending { - img.QueuePosition = m.queue.GetPosition(meta.Digest) - } - return img, nil - } + if img, found, err := m.reuseExistingImage(ref, nil); found || err != nil { + return img, err } - - // Don't have this digest yet, queue the build return m.createAndQueueImage(ref, CreateImageRequest{Name: imageRef}, hostPlatform()) } diff --git a/lib/images/storage_test.go b/lib/images/storage_test.go index d7c8fd994..0f3444b78 100644 --- a/lib/images/storage_test.go +++ b/lib/images/storage_test.go @@ -247,7 +247,9 @@ func TestPromoteLegacyImagesMovesContentAndTags(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Dir(tagPath), 0o755)) require.NoError(t, os.Symlink(digest, tagPath)) - promoteLegacyImages(p) + refs, err := collectLegacyImages(p) + require.NoError(t, err) + promoteLegacyImages(p, refs) // Content exists with the same bytes and is ready. contentMeta, err := readContentMetadata(p, digest) @@ -279,9 +281,11 @@ func TestPromoteLegacyImagesSkipsNonReady(t *testing.T) { CreatedAt: time.Now().UTC(), })) - promoteLegacyImages(p) + refs, err := collectLegacyImages(p) + require.NoError(t, err) + promoteLegacyImages(p, refs) - _, err := os.Stat(p.ImageContentMetadata(digest)) + _, err = os.Stat(p.ImageContentMetadata(digest)) require.True(t, os.IsNotExist(err), "non-ready legacy image must not be promoted") _, err = os.Stat(legacyDir) require.NoError(t, err, "non-ready legacy tree must be preserved") diff --git a/lib/images/tag_manager.go b/lib/images/tag_manager.go index a700e59c1..296c3058f 100644 --- a/lib/images/tag_manager.go +++ b/lib/images/tag_manager.go @@ -7,27 +7,30 @@ import ( "log/slog" "os" "strings" + + "github.com/kernel/hypeman/lib/paths" ) func tagGenerationKey(repository, tag string) string { return repository + ":" + tag } -// nextTagGeneration bumps the mutation generation for a tag and returns the -// new value. Pulls record the value with their metadata and only repoint the -// tag on completion when no later operation mutated it. -func (m *manager) nextTagGeneration(repository, tag string) uint64 { +func (m *manager) ensureTagState() { if m.tagGenerations == nil { m.tagGenerations = make(map[string]uint64) } + if m.requestedTags == nil { + m.requestedTags = make(map[string]string) + } +} + +func (m *manager) nextTagGeneration(repository, tag string) uint64 { + m.ensureTagState() key := tagGenerationKey(repository, tag) m.tagGenerations[key]++ return m.tagGenerations[key] } -// revertTagGeneration undoes a nextTagGeneration bump for an operation that -// failed before mutating the tag, so an in-flight pull of the tag is not -// permanently blocked from repointing it. func (m *manager) revertTagGeneration(repository, tag string) { key := tagGenerationKey(repository, tag) if m.tagGenerations[key] <= 1 { @@ -37,22 +40,14 @@ func (m *manager) revertTagGeneration(repository, tag string) { m.tagGenerations[key]-- } -// releaseTagGeneration drops a failed pull's claim on its tag so an older -// in-flight pull of the same tag can still repoint it. func (m *manager) releaseTagGeneration(repository, tag string, generation uint64) { if m.tagGenerations[tagGenerationKey(repository, tag)] == generation { m.revertTagGeneration(repository, tag) } } -// pruneTagGenerations drops entries for tags that no longer resolve, keeping -// the map bounded over the process lifetime. Deleting a tag still suppresses -// an in-flight pull's repoint: the pull's recorded generation can no longer -// match the missing entry. func (m *manager) pruneTagGenerations() { for key := range m.tagGenerations { - // tagGenerationKey is repo+":"+tag and tags never contain ":", but - // repositories may (host:port), so split on the last colon. colon := strings.LastIndexByte(key, ':') if colon < 0 { continue @@ -63,15 +58,8 @@ func (m *manager) pruneTagGenerations() { } } -// restoreTagState re-seeds the tag indexes from recovered metadata. metas must -// be sorted oldest first so the newest requested pull wins. func (m *manager) restoreTagState(metas []*imageMetadata) { - if m.tagGenerations == nil { - m.tagGenerations = make(map[string]uint64) - } - if m.requestedTags == nil { - m.requestedTags = make(map[string]string) - } + m.ensureTagState() for _, meta := range metas { if meta.RequestedTag == "" || meta.Digest == "" { continue @@ -88,9 +76,6 @@ func (m *manager) restoreTagState(metas []*imageMetadata) { } } -// claimTagForStatus repoints ref's tag at the digest when the image is ready, -// or records a pending tag when the build is still in flight. It is a no-op -// for digest-only references. func (m *manager) claimTagForStatus(meta *imageMetadata, ref *ResolvedRef) error { if ref.Tag() == "" { return nil @@ -101,7 +86,6 @@ func (m *manager) claimTagForStatus(meta *imageMetadata, ref *ResolvedRef) error return ensurePendingTag(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()) } -// claimReadyTag repoints an existing tag at a ready digest, last pull wins. func (m *manager) claimReadyTag(repository, tag, digestHex string) error { m.nextTagGeneration(repository, tag) if err := createTagSymlink(m.paths, repository, tag, digestHex); err != nil { @@ -111,18 +95,11 @@ func (m *manager) claimReadyTag(repository, tag, digestHex string) error { return nil } -// trackRequestedTag records the digest of the newest pull requested for a tag -// so readiness waits can find it without walking the metadata tree. func (m *manager) trackRequestedTag(repository, tag, digestHex string) { - if m.requestedTags == nil { - m.requestedTags = make(map[string]string) - } + m.ensureTagState() m.requestedTags[tagGenerationKey(repository, tag)] = digestHex } -// requestedTagImage returns the newest image requested for a tag, so a -// readiness wait tracks the latest pull rather than the digest the tag -// currently points at. func (m *manager) requestedTagImage(ref *NormalizedRef) *Image { m.createMu.Lock() digestHex, ok := m.requestedTags[tagGenerationKey(ref.Repository(), ref.Tag())] @@ -137,51 +114,35 @@ func (m *manager) requestedTagImage(ref *NormalizedRef) *Image { return meta.toImageFor(ref.String()) } -// claimRequestedTag repoints the pull's requested tag at the finished digest. -// The tag is only repointed when its generation still matches the pull's -// recorded claim, so a later mutation of the tag wins over the in-flight -// pull. A recovered build whose metadata predates requested-tag tracking has -// no recorded tag: fall back to the reference's own tag and recreate a -// missing symlink, matching the pre-tracking behavior. Otherwise an -// already-missing tag stays missing so a concurrent delete wins. func (m *manager) claimRequestedTag(ref *ResolvedRef, meta *imageMetadata) bool { - requestedTag := meta.RequestedTag - allowMissing := requestedTag == "" - if requestedTag == "" { - requestedTag = ref.Tag() + tag := meta.RequestedTag + allowMissing := tag == "" + if allowMissing { + tag = ref.Tag() } - if requestedTag == "" || m.tagGenerations[tagGenerationKey(ref.Repository(), requestedTag)] != meta.TagGeneration { + if !m.tagClaimIsCurrent(ref.Repository(), tag, ref.DigestHex(), meta, allowMissing) { return false } - current, err := resolveTag(m.paths, ref.Repository(), requestedTag) - if err != nil { - if !allowMissing || !errors.Is(err, ErrNotFound) { - return false - } - } else if current != ref.DigestHex() && current != meta.PreviousTagDigest { - return false - } - if err := createTagSymlink(m.paths, ref.Repository(), requestedTag, ref.DigestHex()); err != nil { + if err := createTagSymlink(m.paths, ref.Repository(), tag, ref.DigestHex()); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to create tag symlink: %v\n", err) return false } return true } +func (m *manager) tagClaimIsCurrent(repository, tag, digest string, meta *imageMetadata, allowMissing bool) bool { + if tag == "" || m.tagGenerations[tagGenerationKey(repository, tag)] != meta.TagGeneration { + return false + } + current, err := resolveTag(m.paths, repository, tag) + if err == nil { + return current == digest || current == meta.PreviousTagDigest + } + return allowMissing && errors.Is(err, ErrNotFound) +} + // TagImage creates a ready-image tag without pulling or converting content. -// Cross-repository tags promote legacy content into the shared layout. A -// failed call leaves no side effects: the target tag's generation is only -// bumped after the new tag is on disk, so pending pulls that claimed the -// target tag keep their claim. When the target previously pointed at -// different content, that digest is collected after the new tag is live; -// cleanup failures are logged and do not fail the call, since the tag is -// already installed at that point. -// -// Promotion deliberately runs before the symlink install, so a symlink -// failure after a cross-repo promotion leaves the content promoted with no -// target tag. That state is gc-consistent (unreferenced content is -// collected) and retry is idempotent (promoteImageToContent short-circuits -// on ready content), which beats rolling back a completed promotion. +// Cross-repository tags promote content into the shared layout. func (m *manager) TagImage(ctx context.Context, source, target string) (*Image, error) { sourceRef, targetRef, err := parseTagReferences(source, target) if err != nil { @@ -191,38 +152,46 @@ func (m *manager) TagImage(ctx context.Context, source, target string) (*Image, m.createMu.Lock() defer m.createMu.Unlock() - // A dangling or malformed target symlink is treated like a missing tag so - // the retag self-heals; createTagSymlink replaces the link either way. - previousDigest, err := resolveTag(m.paths, targetRef.Repository(), targetRef.Tag()) - if err != nil && !errors.Is(err, ErrNotFound) && !errors.Is(err, errInvalidSymlinkTarget) { - return nil, fmt.Errorf("resolve existing target tag: %w", err) + previousDigest, err := existingTagDigest(m.paths, targetRef) + if err != nil { + return nil, err } - digestHex, meta, err := m.readyTagImage(sourceRef) if err != nil { return nil, err } - if sourceRef.Repository() != targetRef.Repository() { - if err := promoteImageToContent(m.paths, sourceRef.Repository(), digestHex, meta); err != nil { - return nil, fmt.Errorf("promote image to content: %w", err) - } - } - if err := createTagSymlink(m.paths, targetRef.Repository(), targetRef.Tag(), digestHex); err != nil { - return nil, fmt.Errorf("create image tag: %w", err) - } - if err := writeMetadata(m.paths, targetRef.Repository(), digestHex, meta); err != nil { - return nil, fmt.Errorf("write tagged image metadata: %w", err) + if err := m.installTag(sourceRef, targetRef, digestHex, meta); err != nil { + return nil, err } - // Unlike updateExistingReference, which bumps the generation before - // installing the symlink, the bump happens after the install here so a - // failed tag call cannot invalidate a pending pull's claim on the target - // tag (pinned by TestTagImageFailureLeavesNoSideEffects). m.nextTagGeneration(targetRef.Repository(), targetRef.Tag()) m.cleanupReplacedTag(targetRef, previousDigest, digestHex) return meta.toImageFor(targetRef.String()), nil } +func existingTagDigest(p *paths.Paths, ref *NormalizedRef) (string, error) { + digest, err := resolveTag(p, ref.Repository(), ref.Tag()) + if err == nil || errors.Is(err, ErrNotFound) || errors.Is(err, errInvalidSymlinkTarget) { + return digest, nil + } + return "", fmt.Errorf("resolve existing target tag: %w", err) +} + +func (m *manager) installTag(source, target *NormalizedRef, digest string, meta *imageMetadata) error { + if source.Repository() != target.Repository() { + if err := promoteImageToContent(m.paths, source.Repository(), digest, meta); err != nil { + return fmt.Errorf("promote image to content: %w", err) + } + } + if err := createTagSymlink(m.paths, target.Repository(), target.Tag(), digest); err != nil { + return fmt.Errorf("create image tag: %w", err) + } + if err := writeMetadata(m.paths, target.Repository(), digest, meta); err != nil { + return fmt.Errorf("write tagged image metadata: %w", err) + } + return nil +} + func (m *manager) cleanupUnclaimedImage(ref *ResolvedRef) { if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), ref.DigestHex(), true); err != nil { slog.Warn("failed to collect stale image", "repository", ref.Repository(), "digest", ref.DigestHex(), "error", err) @@ -230,19 +199,21 @@ func (m *manager) cleanupUnclaimedImage(ref *ResolvedRef) { } func (m *manager) cleanupReplacedTag(ref *NormalizedRef, previousDigest, digestHex string) { - if previousDigest != "" && previousDigest != digestHex { - // Sibling tags in this repository may still reference the previous - // digest; only collect when this was the last reference. - count, err := countTagsForDigest(m.paths, ref.Repository(), previousDigest) - if err != nil { - slog.Warn("failed to count tags for replaced image", "repository", ref.Repository(), "digest", previousDigest, "error", err) - } else if count == 0 { - if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), previousDigest, true); err != nil { - slog.Warn("failed to collect replaced image content", "repository", ref.Repository(), "digest", previousDigest, "error", err) - } - m.refreshDiskUsageTotals() - } + if previousDigest == "" || previousDigest == digestHex { + return + } + count, err := countTagsForDigest(m.paths, ref.Repository(), previousDigest) + if err != nil { + slog.Warn("failed to count tags for replaced image", "repository", ref.Repository(), "digest", previousDigest, "error", err) + return + } + if count > 0 { + return + } + if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), previousDigest, true); err != nil { + slog.Warn("failed to collect replaced image content", "repository", ref.Repository(), "digest", previousDigest, "error", err) } + m.refreshDiskUsageTotals() } func parseTagReferences(source, target string) (*NormalizedRef, *NormalizedRef, error) { From d5195e45ad918d9cc7caea4621fbe3d76f193982 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:29:33 +0000 Subject: [PATCH 06/13] Harden image promotion and tag cleanup --- lib/images/manager.go | 43 +++++++++++++++++++++++------- lib/images/storage.go | 55 +++++++++++++++++++++------------------ lib/images/tag_manager.go | 21 +++++++++++++++ 3 files changed, 85 insertions(+), 34 deletions(-) diff --git a/lib/images/manager.go b/lib/images/manager.go index fb202f273..2c066e5ef 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -126,7 +126,9 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma if err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to scan legacy images for promotion: %v\n", err) } else { - go promoteLegacyImages(p, legacyRefs) + m.createMu.Lock() + promoteLegacyImages(p, legacyRefs) + m.createMu.Unlock() } return m, nil } @@ -402,12 +404,14 @@ func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest, if err := writeMetadata(m.paths, ref.Repository(), ref.DigestHex(), meta); err != nil { if ref.Tag() != "" { m.revertTagGeneration(ref.Repository(), ref.Tag()) + m.clearRequestedTag(ref.Repository(), ref.Tag(), ref.DigestHex()) } return nil, fmt.Errorf("write initial metadata: %w", err) } if ref.Tag() != "" && previousTagDigest == "" { if err := createTagSymlink(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()); err != nil { m.revertTagGeneration(ref.Repository(), ref.Tag()) + m.clearRequestedTag(ref.Repository(), ref.Tag(), ref.DigestHex()) return nil, fmt.Errorf("create pending image tag: %w", err) } } @@ -538,12 +542,6 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i return errStaleBuild } - if err := installAtomically(layout.disk, func(path string) error { - return os.Rename(diskTempPath, path) - }); err != nil { - return fmt.Errorf("install image disk: %w", err) - } - // The pulled image config is the source of truth for the platform. var requestedPlatform string if meta.Request != nil { @@ -554,6 +552,29 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i return err } + diskInstalled := false + modelWritten := false + rollback := func(cause error) error { + var rollbackErr error + if modelWritten { + rollbackErr = errors.Join(rollbackErr, os.Remove(m.paths.ImageContentManifestModel(ref.DigestHex()))) + } + if diskInstalled { + rollbackErr = errors.Join(rollbackErr, os.Remove(layout.disk)) + } + if rollbackErr != nil { + return errors.Join(cause, fmt.Errorf("rollback finalization: %w", rollbackErr)) + } + return cause + } + + if err := installAtomically(layout.disk, func(path string) error { + return os.Rename(diskTempPath, path) + }); err != nil { + return fmt.Errorf("install image disk: %w", err) + } + diskInstalled = true + // Persist the manifest content model beside the shared content so later // stages can recompose the image from per-layer artifacts and GC can tell // which OCI blobs are still referenced. @@ -561,8 +582,9 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i model := *result.Manifest model.Platform = actualPlatform.String() if err := writeManifestModel(m.paths, ref.DigestHex(), &model); err != nil { - return fmt.Errorf("write manifest model: %w", err) + return rollback(fmt.Errorf("write manifest model: %w", err)) } + modelWritten = true } meta.Status = StatusReady @@ -576,7 +598,7 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i meta.WorkingDir = result.Metadata.WorkingDir if err := writeMetadataFile(layout.metadata, meta); err != nil { - return fmt.Errorf("write final metadata: %w", err) + return rollback(fmt.Errorf("write final metadata: %w", err)) } m.notifyReady(ref.DigestHex(), StatusReady, nil) @@ -654,6 +676,7 @@ func (m *manager) updateStatusByDigest(ref *ResolvedRef, status string, err erro // same tag can still repoint it. if status == StatusFailed && meta.RequestedTag != "" { m.releaseTagGeneration(ref.Repository(), meta.RequestedTag, meta.TagGeneration) + m.clearRequestedTag(ref.Repository(), meta.RequestedTag, strings.TrimPrefix(meta.Digest, "sha256:")) } } @@ -744,6 +767,7 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error { if err := removeDigestIfUnreferenced(m.paths, repository, digestHex, false); err != nil { return err } + m.clearRequestedDigest(digestHex) m.refreshDiskUsageTotals() return nil } @@ -760,6 +784,7 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error { if err := deleteTag(m.paths, repository, tag); err != nil { return err } + m.clearRequestedTag(repository, tag, digestHex) m.pruneTagGenerations() // Check if the digest is now orphaned (no other tags reference it) diff --git a/lib/images/storage.go b/lib/images/storage.go index 99fd9cd15..cd1702371 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -253,32 +253,37 @@ func readMetadataAt(layout imageLayout) (*imageMetadata, error) { } func promoteImageToContent(p *paths.Paths, sourceRepository, digestHex string, sourceMeta *imageMetadata) error { - contentReady := false - if contentMeta, err := readContentMetadata(p, digestHex); err == nil { - contentReady = contentMeta.Status == StatusReady - } - if !contentReady { - sourceLayout := resolveImageLayout(p, sourceRepository, digestHex) - sourceDiskPath := sourceLayout.disk - if _, err := os.Stat(sourceDiskPath); err != nil { - return fmt.Errorf("stat source disk: %w", err) - } - if err := os.MkdirAll(p.ImageContentDir(digestHex), 0755); err != nil { - return fmt.Errorf("create content directory: %w", err) - } - contentMeta := *sourceMeta - contentMeta.Status = StatusConverting - if err := writeMetadataFile(p.ImageContentMetadata(digestHex), &contentMeta); err != nil { - return fmt.Errorf("write content metadata: %w", err) - } - if err := installAtomically(p.ImageContentPath(digestHex), func(path string) error { - return os.Link(sourceDiskPath, path) - }); err != nil { - return fmt.Errorf("link source disk: %w", err) - } - if err := writeMetadataFile(p.ImageContentMetadata(digestHex), sourceMeta); err != nil { - return fmt.Errorf("finalize content metadata: %w", err) + contentMeta, err := readContentMetadata(p, digestHex) + if err == nil { + if contentMeta.Status == StatusReady { + return promoteLegacyTags(p, sourceRepository, digestHex) } + return fmt.Errorf("%w: content status is %s", ErrImageNotReady, contentMeta.Status) + } + if !errors.Is(err, ErrNotFound) { + return err + } + + sourceLayout := resolveImageLayout(p, sourceRepository, digestHex) + sourceDiskPath := sourceLayout.disk + if _, err := os.Stat(sourceDiskPath); err != nil { + return fmt.Errorf("stat source disk: %w", err) + } + if err := os.MkdirAll(p.ImageContentDir(digestHex), 0755); err != nil { + return fmt.Errorf("create content directory: %w", err) + } + pendingMeta := *sourceMeta + pendingMeta.Status = StatusConverting + if err := writeMetadataFile(p.ImageContentMetadata(digestHex), &pendingMeta); err != nil { + return fmt.Errorf("write content metadata: %w", err) + } + if err := installAtomically(p.ImageContentPath(digestHex), func(path string) error { + return os.Link(sourceDiskPath, path) + }); err != nil { + return fmt.Errorf("link source disk: %w", err) + } + if err := writeMetadataFile(p.ImageContentMetadata(digestHex), sourceMeta); err != nil { + return fmt.Errorf("finalize content metadata: %w", err) } return promoteLegacyTags(p, sourceRepository, digestHex) diff --git a/lib/images/tag_manager.go b/lib/images/tag_manager.go index b7b11aeab..43b3a53db 100644 --- a/lib/images/tag_manager.go +++ b/lib/images/tag_manager.go @@ -61,6 +61,9 @@ func (m *manager) pruneTagGenerations() { func (m *manager) restoreTagState(metas []*imageMetadata) { m.ensureTagState() for _, meta := range metas { + if meta.Status != StatusPending && meta.Status != StatusPulling && meta.Status != StatusConverting { + continue + } if meta.RequestedTag == "" || meta.Digest == "" { continue } @@ -100,6 +103,21 @@ func (m *manager) trackRequestedTag(repository, tag, digestHex string) { m.requestedTags[tagGenerationKey(repository, tag)] = digestHex } +func (m *manager) clearRequestedTag(repository, tag, digestHex string) { + key := tagGenerationKey(repository, tag) + if current, ok := m.requestedTags[key]; ok && (digestHex == "" || current == digestHex) { + delete(m.requestedTags, key) + } +} + +func (m *manager) clearRequestedDigest(digestHex string) { + for key, current := range m.requestedTags { + if current == digestHex { + delete(m.requestedTags, key) + } + } +} + func (m *manager) requestedTagImage(ref *NormalizedRef) *Image { m.createMu.Lock() digestHex, ok := m.requestedTags[tagGenerationKey(ref.Repository(), ref.Tag())] @@ -121,12 +139,14 @@ func (m *manager) claimRequestedTag(ref *ResolvedRef, meta *imageMetadata) bool tag = ref.Tag() } if !m.tagClaimIsCurrent(ref.Repository(), tag, ref.DigestHex(), meta, allowMissing) { + m.clearRequestedTag(ref.Repository(), tag, ref.DigestHex()) return false } if err := createTagSymlink(m.paths, ref.Repository(), tag, ref.DigestHex()); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to create tag symlink: %v\n", err) return false } + m.clearRequestedTag(ref.Repository(), tag, ref.DigestHex()) return true } @@ -164,6 +184,7 @@ func (m *manager) TagImage(ctx context.Context, source, target string) (*Image, return nil, err } m.nextTagGeneration(targetRef.Repository(), targetRef.Tag()) + m.clearRequestedTag(targetRef.Repository(), targetRef.Tag(), "") m.cleanupReplacedTag(targetRef, previousDigest, digestHex) return meta.toImageFor(targetRef.String()), nil From 497610cec6786d8737b07eb3d09d56978bfe1272 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:40:03 +0000 Subject: [PATCH 07/13] Fix shared image tag claims and labels --- lib/images/manager.go | 50 +++++++++++------------ lib/images/storage_refs.go | 11 ----- lib/images/storage_test.go | 26 ++++++++++++ lib/images/tag_manager.go | 35 +++++++++++++++- lib/images/tag_test.go | 82 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 167 insertions(+), 37 deletions(-) diff --git a/lib/images/manager.go b/lib/images/manager.go index 2c066e5ef..56d8f7135 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -119,17 +119,9 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma } m.RecoverInterruptedBuilds() - // Promotion hardlinks disks and repoints tags, which can take a while on a - // large legacy store; scan before starting the background migration so new - // images created after startup are not picked up by the migration. - legacyRefs, err := collectLegacyImages(p) - if err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to scan legacy images for promotion: %v\n", err) - } else { - m.createMu.Lock() - promoteLegacyImages(p, legacyRefs) - m.createMu.Unlock() - } + // Legacy images remain in place until a later operation needs to promote + // their content. This keeps startup reads available even if migration work + // cannot complete. return m, nil } @@ -224,7 +216,7 @@ func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Ima m.createMu.Lock() defer m.createMu.Unlock() - if img, found, err := m.reuseExistingImage(ref, req.Credentials); found || err != nil { + if img, found, err := m.reuseExistingImage(ref, req.Credentials, req.Tags); found || err != nil { return img, err } return m.createAndQueueImage(ref, req, platform) @@ -256,13 +248,13 @@ func (m *manager) ImportLocalImage(ctx context.Context, repo, reference, digest m.createMu.Lock() defer m.createMu.Unlock() - if img, found, err := m.reuseExistingImage(ref, nil); found || err != nil { + if img, found, err := m.reuseExistingImage(ref, nil, nil); found || err != nil { return img, err } return m.createAndQueueImage(ref, CreateImageRequest{Name: imageRef}, hostPlatform()) } -func (m *manager) reuseExistingImage(ref *ResolvedRef, credentials *authn.AuthConfig) (*Image, bool, error) { +func (m *manager) reuseExistingImage(ref *ResolvedRef, credentials *authn.AuthConfig, resourceTags tags.Tags) (*Image, bool, error) { meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()) if err != nil { return nil, false, nil @@ -273,14 +265,20 @@ func (m *manager) reuseExistingImage(ref *ResolvedRef, credentials *authn.AuthCo } return nil, false, nil } - if ref.Tag() != "" { - if meta.Status != StatusReady { - // A pending pull with different credentials does not get to point - // the tag at its digest. - if !m.inflightCredentialsMatch(ref.Digest(), credentials) { - return nil, true, fmt.Errorf("%w: retry after the current pull completes", ErrCredentialConflict) - } + if ref.Tag() != "" && meta.Status != StatusReady { + // A pending pull with different credentials does not get to point + // the tag at its digest or update its labels. + if !m.inflightCredentialsMatch(ref.Digest(), credentials) { + return nil, true, fmt.Errorf("%w: retry after the current pull completes", ErrCredentialConflict) } + } + if resourceTags != nil { + meta.Tags = tags.Clone(resourceTags) + if err := writeMetadata(m.paths, ref.Repository(), ref.DigestHex(), meta); err != nil { + return nil, true, fmt.Errorf("update image tags: %w", err) + } + } + if ref.Tag() != "" { if err := m.claimTagForStatus(meta, ref); err != nil { return nil, true, fmt.Errorf("create image tag: %w", err) } @@ -602,7 +600,7 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i } m.notifyReady(ref.DigestHex(), StatusReady, nil) - if !m.claimRequestedTag(ref, meta) { + if !m.claimRequestedTags(ref, meta) { m.cleanupUnclaimedImage(ref) } m.refreshDiskUsageTotals() @@ -674,9 +672,11 @@ func (m *manager) updateStatusByDigest(ref *ResolvedRef, status string, err erro } // A failed pull releases its tag claim so an older in-flight pull of the // same tag can still repoint it. - if status == StatusFailed && meta.RequestedTag != "" { - m.releaseTagGeneration(ref.Repository(), meta.RequestedTag, meta.TagGeneration) - m.clearRequestedTag(ref.Repository(), meta.RequestedTag, strings.TrimPrefix(meta.Digest, "sha256:")) + if status == StatusFailed { + if meta.RequestedTag != "" { + m.releaseTagGeneration(ref.Repository(), meta.RequestedTag, meta.TagGeneration) + } + m.clearRequestedDigest(strings.TrimPrefix(meta.Digest, "sha256:")) } } diff --git a/lib/images/storage_refs.go b/lib/images/storage_refs.go index bc25b5eda..7502eb33d 100644 --- a/lib/images/storage_refs.go +++ b/lib/images/storage_refs.go @@ -10,17 +10,6 @@ import ( "github.com/kernel/hypeman/lib/paths" ) -func ensurePendingTag(p *paths.Paths, repository, tag, digestHex string) error { - _, err := resolveTag(p, repository, tag) - if err == nil { - return nil - } - if !errors.Is(err, ErrNotFound) { - return err - } - return createTagSymlink(p, repository, tag, digestHex) -} - func listTags(p *paths.Paths, repository string) ([]string, error) { dirs := []string{filepath.Join(p.ImageRepositoriesDir(), repository), p.ImageRepositoryDir(repository)} seen := make(map[string]struct{}) diff --git a/lib/images/storage_test.go b/lib/images/storage_test.go index 0f3444b78..24a2a9db3 100644 --- a/lib/images/storage_test.go +++ b/lib/images/storage_test.go @@ -225,6 +225,32 @@ func TestImageMetadataToImage_EmptyMetadataOmitted(t *testing.T) { require.Nil(t, img.Tags) } +func TestNewManagerLeavesLegacyImagesInPlace(t *testing.T) { + p := paths.New(t.TempDir()) + repository := "docker.io/library/alpine" + tag := "latest" + digest := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + legacyDir := p.ImageDigestDir(repository, digest) + require.NoError(t, os.MkdirAll(legacyDir, 0o755)) + require.NoError(t, writeMetadataFile(p.ImageMetadata(repository, digest), &imageMetadata{ + Name: repository + ":" + tag, Digest: "sha256:" + digest, + Status: StatusReady, SizeBytes: 7, CreatedAt: time.Now().UTC(), + })) + require.NoError(t, os.WriteFile(p.ImageDigestPath(repository, digest), []byte("rootfs!"), 0o644)) + tagPath := p.ImageTagSymlink(repository, tag) + require.NoError(t, os.MkdirAll(filepath.Dir(tagPath), 0o755)) + require.NoError(t, os.Symlink(digest, tagPath)) + + _, err := NewManager(p, 1, nil) + require.NoError(t, err) + require.DirExists(t, legacyDir) + require.FileExists(t, p.ImageMetadata(repository, digest)) + require.FileExists(t, p.ImageDigestPath(repository, digest)) + _, err = os.Stat(p.ImageContentDir(digest)) + require.ErrorIs(t, err, os.ErrNotExist) + requireTagResolvesTo(t, p, repository, tag, digest) +} + func TestPromoteLegacyImagesMovesContentAndTags(t *testing.T) { p := paths.New(t.TempDir()) repository := "docker.io/library/alpine" diff --git a/lib/images/tag_manager.go b/lib/images/tag_manager.go index 43b3a53db..39103bb80 100644 --- a/lib/images/tag_manager.go +++ b/lib/images/tag_manager.go @@ -86,7 +86,12 @@ func (m *manager) claimTagForStatus(meta *imageMetadata, ref *ResolvedRef) error if meta.Status == StatusReady { return m.claimReadyTag(ref.Repository(), ref.Tag(), ref.DigestHex()) } - return ensurePendingTag(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()) + // Keep the existing tag visible until the build succeeds, but remember this + // digest as the newest request for the tag. Finalization will claim it along + // with every other tag waiting for the same digest. + m.nextTagGeneration(ref.Repository(), ref.Tag()) + m.trackRequestedTag(ref.Repository(), ref.Tag(), ref.DigestHex()) + return nil } func (m *manager) claimReadyTag(repository, tag, digestHex string) error { @@ -95,6 +100,7 @@ func (m *manager) claimReadyTag(repository, tag, digestHex string) error { m.revertTagGeneration(repository, tag) return err } + m.clearRequestedTag(repository, tag, digestHex) return nil } @@ -150,6 +156,33 @@ func (m *manager) claimRequestedTag(ref *ResolvedRef, meta *imageMetadata) bool return true } +func (m *manager) claimRequestedTags(ref *ResolvedRef, meta *imageMetadata) bool { + digestHex := ref.DigestHex() + claimed := false + for key, requestedDigest := range m.requestedTags { + if requestedDigest != digestHex { + continue + } + separator := strings.LastIndexByte(key, ':') + if separator <= 0 || separator == len(key)-1 { + continue + } + repository, tag := key[:separator], key[separator+1:] + if err := createTagSymlink(m.paths, repository, tag, digestHex); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to create tag symlink: %v\n", err) + continue + } + delete(m.requestedTags, key) + claimed = true + } + if claimed { + return true + } + // Digest-only images have no tag claim but must remain addressable by their + // digest after finalization. + return meta.RequestedTag == "" && ref.Tag() == "" +} + func (m *manager) tagClaimIsCurrent(repository, tag, digest string, meta *imageMetadata, allowMissing bool) bool { if tag == "" || m.tagGenerations[tagGenerationKey(repository, tag)] != meta.TagGeneration { return false diff --git a/lib/images/tag_test.go b/lib/images/tag_test.go index 75796ab2f..72bfd0ef8 100644 --- a/lib/images/tag_test.go +++ b/lib/images/tag_test.go @@ -4,10 +4,13 @@ import ( "context" "os" "path/filepath" + "strings" "testing" "time" "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/queue" + "github.com/kernel/hypeman/lib/tags" "github.com/stretchr/testify/require" ) @@ -160,6 +163,85 @@ func TestStaleTagClaimCollectsContent(t *testing.T) { require.ErrorIs(t, err, os.ErrNotExist) } +func TestPendingDigestClaimsAllRequestedTags(t *testing.T) { + p, m, repository := newTagTestCase(t) + oldDigest := "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" + newDigest := "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2" + seedLegacy(t, p, repository, "latest", oldDigest) + seedLegacy(t, p, repository, "stable", oldDigest) + + latest, err := ParseNormalizedRef(repository + ":latest") + require.NoError(t, err) + stable, err := ParseNormalizedRef(repository + ":stable") + require.NoError(t, err) + latestRef := NewResolvedRef(latest, "sha256:"+newDigest) + stableRef := NewResolvedRef(stable, "sha256:"+newDigest) + meta := &imageMetadata{RequestedTag: "latest", TagGeneration: 1} + + require.NoError(t, m.claimTagForStatus(meta, latestRef)) + require.NoError(t, m.claimTagForStatus(meta, stableRef)) + require.True(t, m.claimRequestedTags(latestRef, meta)) + requireTagResolvesTo(t, p, repository, "latest", newDigest) + requireTagResolvesTo(t, p, repository, "stable", newDigest) +} + +func TestWaitForReadyUsesNewestPendingTag(t *testing.T) { + p, m, repository := newTagTestCase(t) + m.queue = queue.New(1) + m.readySubscribers = make(map[string][]chan StatusEvent) + m.requestedTags = make(map[string]string) + oldDigest := strings.Repeat("a1", 32) + newDigest := strings.Repeat("b2", 32) + seedContent(t, p, repository, "latest", oldDigest) + + require.NoError(t, os.MkdirAll(p.ImageContentDir(newDigest), 0o755)) + pending := &imageMetadata{ + Name: repository + ":latest", Digest: "sha256:" + newDigest, + Status: StatusPending, CreatedAt: time.Now().UTC(), + } + require.NoError(t, writeMetadataFile(p.ImageContentMetadata(newDigest), pending)) + m.requestedTags[tagGenerationKey(repository, "latest")] = newDigest + + result := make(chan error, 1) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { result <- m.WaitForReady(ctx, repository+":latest") }() + select { + case err := <-result: + t.Fatalf("wait returned before the pending digest completed: %v", err) + case <-time.After(100 * time.Millisecond): + } + + pending.Status = StatusReady + require.NoError(t, os.WriteFile(p.ImageContentPath(newDigest), []byte("rootfs!"), 0o644)) + require.NoError(t, writeMetadataFile(p.ImageContentMetadata(newDigest), pending)) + m.notifyReady(newDigest, StatusReady, nil) + select { + case err := <-result: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("wait did not observe the newest digest") + } +} + +func TestReuseExistingImageUpdatesResourceTags(t *testing.T) { + p, m, repository := newTagTestCase(t) + digest := strings.Repeat("c3", 32) + seedContent(t, p, repository, "latest", digest) + + ref, err := ParseNormalizedRef(repository + "@sha256:" + digest) + require.NoError(t, err) + resolved := NewResolvedRef(ref, "sha256:"+digest) + img, found, err := m.reuseExistingImage(resolved, nil, tags.Tags{"team": "payments"}) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, "payments", img.Tags["team"]) + + meta, err := readContentMetadata(p, digest) + require.NoError(t, err) + require.Equal(t, tags.Tags{"team": "payments"}, meta.Tags) +} + func TestTagImageRejectsNotReady(t *testing.T) { p, m, repository := newTagTestCase(t) digest := "d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4" From bc4f6a5751116ac87bb542605fd24598afc561bc Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:03:34 +0000 Subject: [PATCH 08/13] Preserve tags for recovered image builds --- lib/images/tag_manager.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/images/tag_manager.go b/lib/images/tag_manager.go index 39103bb80..f431d2d0c 100644 --- a/lib/images/tag_manager.go +++ b/lib/images/tag_manager.go @@ -175,6 +175,23 @@ func (m *manager) claimRequestedTags(ref *ResolvedRef, meta *imageMetadata) bool delete(m.requestedTags, key) claimed = true } + + // Older metadata did not persist RequestedTag or a requestedTags entry. Keep + // those recovered builds addressable by falling back to the build reference. + primaryTag := meta.RequestedTag + if primaryTag == "" { + primaryTag = ref.Tag() + } + primaryKey := tagGenerationKey(ref.Repository(), primaryTag) + if primaryTag != "" { + if _, hasRequest := m.requestedTags[primaryKey]; !hasRequest && m.tagClaimIsCurrent(ref.Repository(), primaryTag, digestHex, meta, meta.RequestedTag == "") { + if err := createTagSymlink(m.paths, ref.Repository(), primaryTag, digestHex); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to create tag symlink: %v\n", err) + } else { + claimed = true + } + } + } if claimed { return true } From ddca8d65bc9611024d2506d409cae81165cb4533 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:16:30 +0000 Subject: [PATCH 09/13] Harden image tag claim recovery --- lib/images/manager.go | 34 ++++++--- lib/images/manifest_model.go | 9 ++- lib/images/storage.go | 9 +++ lib/images/tag_manager.go | 134 +++++++++++++++++++++++++---------- lib/images/tag_test.go | 43 ++++++++++- 5 files changed, 179 insertions(+), 50 deletions(-) diff --git a/lib/images/manager.go b/lib/images/manager.go index 56d8f7135..5179906be 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -119,9 +119,10 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma } m.RecoverInterruptedBuilds() - // Legacy images remain in place until a later operation needs to promote - // their content. This keeps startup reads available even if migration work - // cannot complete. + // Keep legacy images readable in their existing layout and promote them only + // when an operation needs shared content, such as a cross-repository tag. + // Avoiding a startup-wide migration keeps startup bounded and independent of + // migration success. return m, nil } @@ -265,7 +266,7 @@ func (m *manager) reuseExistingImage(ref *ResolvedRef, credentials *authn.AuthCo } return nil, false, nil } - if ref.Tag() != "" && meta.Status != StatusReady { + if meta.Status != StatusReady { // A pending pull with different credentials does not get to point // the tag at its digest or update its labels. if !m.inflightCredentialsMatch(ref.Digest(), credentials) { @@ -551,11 +552,15 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i } diskInstalled := false + modelPath := m.paths.ImageContentManifestModel(ref.DigestHex()) + if !layout.content { + modelPath = filepath.Join(layout.dir, "manifest.json") + } modelWritten := false rollback := func(cause error) error { var rollbackErr error if modelWritten { - rollbackErr = errors.Join(rollbackErr, os.Remove(m.paths.ImageContentManifestModel(ref.DigestHex()))) + rollbackErr = errors.Join(rollbackErr, os.Remove(modelPath)) } if diskInstalled { rollbackErr = errors.Join(rollbackErr, os.Remove(layout.disk)) @@ -579,7 +584,7 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i if result.Manifest != nil { model := *result.Manifest model.Platform = actualPlatform.String() - if err := writeManifestModel(m.paths, ref.DigestHex(), &model); err != nil { + if err := writeManifestModelAt(modelPath, ref.DigestHex(), &model); err != nil { return rollback(fmt.Errorf("write manifest model: %w", err)) } modelWritten = true @@ -760,10 +765,19 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error { if _, err := readMetadata(m.paths, repository, digestHex); err != nil { return err } + tags, err := tagsForDigest(m.paths, repository, digestHex) + if err != nil { + return err + } if err := deleteTagsForDigest(m.paths, repository, digestHex); err != nil { return err } - m.pruneTagGenerations() + for _, tag := range tags { + m.forgetTagState(repository, tag) + if err := m.cancelPendingTag(repository, tag); err != nil { + return fmt.Errorf("cancel pending image tag: %w", err) + } + } if err := removeDigestIfUnreferenced(m.paths, repository, digestHex, false); err != nil { return err } @@ -784,8 +798,10 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error { if err := deleteTag(m.paths, repository, tag); err != nil { return err } - m.clearRequestedTag(repository, tag, digestHex) - m.pruneTagGenerations() + m.forgetTagState(repository, tag) + if err := m.cancelPendingTag(repository, tag); err != nil { + return fmt.Errorf("cancel pending image tag: %w", err) + } // Check if the digest is now orphaned (no other tags reference it) count, err := countTagsForDigest(m.paths, repository, digestHex) diff --git a/lib/images/manifest_model.go b/lib/images/manifest_model.go index b3e7dc25b..2e4fc3e09 100644 --- a/lib/images/manifest_model.go +++ b/lib/images/manifest_model.go @@ -126,11 +126,18 @@ func parseSHA256Digest(value string) (digest.Digest, error) { } func writeManifestModel(p *paths.Paths, digestHex string, model *imageManifestModel) error { + return writeManifestModelAt(p.ImageContentManifestModel(digestHex), digestHex, model) +} + +func writeManifestModelAt(path, digestHex string, model *imageManifestModel) error { + if err := validateManifestModel(digestHex, model); err != nil { + return err + } data, err := json.MarshalIndent(model, "", " ") if err != nil { return fmt.Errorf("marshal manifest model: %w", err) } - return writeJSONAtomic(p.ImageContentManifestModel(digestHex), data) + return writeJSONAtomic(path, data) } // readManifestModel loads the manifest model for a digest, if present. diff --git a/lib/images/storage.go b/lib/images/storage.go index cd1702371..eaaafbe02 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -33,6 +33,15 @@ type imageMetadata struct { RequestedTag string `json:"requested_tag,omitempty"` PreviousTagDigest string `json:"previous_tag_digest,omitempty"` TagGeneration uint64 `json:"tag_generation,omitempty"` + TagClaimCanceled bool `json:"tag_claim_canceled,omitempty"` + TagClaims []imageTagClaim `json:"tag_claims,omitempty"` +} + +type imageTagClaim struct { + Repository string `json:"repository"` + Tag string `json:"tag"` + PreviousTagDigest string `json:"previous_tag_digest,omitempty"` + TagGeneration uint64 `json:"tag_generation,omitempty"` } func (m *imageMetadata) toImageFor(reference string) *Image { diff --git a/lib/images/tag_manager.go b/lib/images/tag_manager.go index f431d2d0c..fadcd9d79 100644 --- a/lib/images/tag_manager.go +++ b/lib/images/tag_manager.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "log/slog" - "os" "strings" "github.com/kernel/hypeman/lib/paths" @@ -46,16 +45,9 @@ func (m *manager) releaseTagGeneration(repository, tag string, generation uint64 } } -func (m *manager) pruneTagGenerations() { - for key := range m.tagGenerations { - colon := strings.LastIndexByte(key, ':') - if colon < 0 { - continue - } - if _, err := resolveTag(m.paths, key[:colon], key[colon+1:]); err != nil { - delete(m.tagGenerations, key) - } - } +func (m *manager) forgetTagState(repository, tag string) { + delete(m.tagGenerations, tagGenerationKey(repository, tag)) + m.clearRequestedTag(repository, tag, "") } func (m *manager) restoreTagState(metas []*imageMetadata) { @@ -64,21 +56,34 @@ func (m *manager) restoreTagState(metas []*imageMetadata) { if meta.Status != StatusPending && meta.Status != StatusPulling && meta.Status != StatusConverting { continue } - if meta.RequestedTag == "" || meta.Digest == "" { + if meta.Digest == "" { continue } + digestHex := strings.TrimPrefix(meta.Digest, "sha256:") ref, err := ParseNormalizedRef(meta.Name) if err != nil { continue } - key := tagGenerationKey(ref.Repository(), meta.RequestedTag) - if meta.TagGeneration > m.tagGenerations[key] { - m.tagGenerations[key] = meta.TagGeneration + if meta.RequestedTag != "" && !meta.TagClaimCanceled { + m.restoreRequestedTag(ref.Repository(), meta.RequestedTag, digestHex, meta.TagGeneration) + } + for _, claim := range meta.TagClaims { + m.restoreRequestedTag(claim.Repository, claim.Tag, digestHex, claim.TagGeneration) } - m.requestedTags[key] = strings.TrimPrefix(meta.Digest, "sha256:") } } +func (m *manager) restoreRequestedTag(repository, tag, digestHex string, generation uint64) { + if tag == "" { + return + } + key := tagGenerationKey(repository, tag) + if generation > m.tagGenerations[key] { + m.tagGenerations[key] = generation + } + m.requestedTags[key] = digestHex +} + func (m *manager) claimTagForStatus(meta *imageMetadata, ref *ResolvedRef) error { if ref.Tag() == "" { return nil @@ -89,7 +94,37 @@ func (m *manager) claimTagForStatus(meta *imageMetadata, ref *ResolvedRef) error // Keep the existing tag visible until the build succeeds, but remember this // digest as the newest request for the tag. Finalization will claim it along // with every other tag waiting for the same digest. - m.nextTagGeneration(ref.Repository(), ref.Tag()) + if meta.RequestedTag == ref.Tag() && strings.HasPrefix(meta.Name, ref.Repository()+":") { + wasCanceled := meta.TagClaimCanceled + meta.TagClaimCanceled = false + m.trackRequestedTag(ref.Repository(), ref.Tag(), ref.DigestHex()) + if wasCanceled { + return writeMetadata(m.paths, ref.Repository(), ref.DigestHex(), meta) + } + return nil + } + for _, claim := range meta.TagClaims { + if claim.Repository == ref.Repository() && claim.Tag == ref.Tag() { + m.restoreRequestedTag(claim.Repository, claim.Tag, ref.DigestHex(), claim.TagGeneration) + return nil + } + } + previous, err := resolveTag(m.paths, ref.Repository(), ref.Tag()) + if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + generation := m.nextTagGeneration(ref.Repository(), ref.Tag()) + meta.TagClaims = append(meta.TagClaims, imageTagClaim{ + Repository: ref.Repository(), + Tag: ref.Tag(), + PreviousTagDigest: previous, + TagGeneration: generation, + }) + if err := writeMetadata(m.paths, ref.Repository(), ref.DigestHex(), meta); err != nil { + m.revertTagGeneration(ref.Repository(), ref.Tag()) + meta.TagClaims = meta.TagClaims[:len(meta.TagClaims)-1] + return err + } m.trackRequestedTag(ref.Repository(), ref.Tag(), ref.DigestHex()) return nil } @@ -124,6 +159,42 @@ func (m *manager) clearRequestedDigest(digestHex string) { } } +func (m *manager) cancelPendingTag(repository, tag string) error { + metas, err := listAllMetadata(m.paths) + if err != nil { + return err + } + for _, meta := range metas { + if meta.Status != StatusPending && meta.Status != StatusPulling && meta.Status != StatusConverting { + continue + } + ref, err := ParseNormalizedRef(meta.Name) + if err != nil { + continue + } + changed := false + if ref.Repository() == repository && meta.RequestedTag == tag { + meta.TagClaimCanceled = true + changed = true + } + claims := meta.TagClaims[:0] + for _, claim := range meta.TagClaims { + if claim.Repository == repository && claim.Tag == tag { + changed = true + continue + } + claims = append(claims, claim) + } + meta.TagClaims = claims + if changed { + if err := writeMetadata(m.paths, ref.Repository(), strings.TrimPrefix(meta.Digest, "sha256:"), meta); err != nil { + return err + } + } + } + return nil +} + func (m *manager) requestedTagImage(ref *NormalizedRef) *Image { m.createMu.Lock() digestHex, ok := m.requestedTags[tagGenerationKey(ref.Repository(), ref.Tag())] @@ -138,24 +209,6 @@ func (m *manager) requestedTagImage(ref *NormalizedRef) *Image { return meta.toImageFor(ref.String()) } -func (m *manager) claimRequestedTag(ref *ResolvedRef, meta *imageMetadata) bool { - tag := meta.RequestedTag - allowMissing := tag == "" - if allowMissing { - tag = ref.Tag() - } - if !m.tagClaimIsCurrent(ref.Repository(), tag, ref.DigestHex(), meta, allowMissing) { - m.clearRequestedTag(ref.Repository(), tag, ref.DigestHex()) - return false - } - if err := createTagSymlink(m.paths, ref.Repository(), tag, ref.DigestHex()); err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to create tag symlink: %v\n", err) - return false - } - m.clearRequestedTag(ref.Repository(), tag, ref.DigestHex()) - return true -} - func (m *manager) claimRequestedTags(ref *ResolvedRef, meta *imageMetadata) bool { digestHex := ref.DigestHex() claimed := false @@ -169,7 +222,7 @@ func (m *manager) claimRequestedTags(ref *ResolvedRef, meta *imageMetadata) bool } repository, tag := key[:separator], key[separator+1:] if err := createTagSymlink(m.paths, repository, tag, digestHex); err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to create tag symlink: %v\n", err) + slog.Warn("failed to claim image tag", "repository", repository, "tag", tag, "digest", digestHex, "error", err) continue } delete(m.requestedTags, key) @@ -182,11 +235,11 @@ func (m *manager) claimRequestedTags(ref *ResolvedRef, meta *imageMetadata) bool if primaryTag == "" { primaryTag = ref.Tag() } - primaryKey := tagGenerationKey(ref.Repository(), primaryTag) - if primaryTag != "" { + if primaryTag != "" && !meta.TagClaimCanceled { + primaryKey := tagGenerationKey(ref.Repository(), primaryTag) if _, hasRequest := m.requestedTags[primaryKey]; !hasRequest && m.tagClaimIsCurrent(ref.Repository(), primaryTag, digestHex, meta, meta.RequestedTag == "") { if err := createTagSymlink(m.paths, ref.Repository(), primaryTag, digestHex); err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to create tag symlink: %v\n", err) + slog.Warn("failed to claim image tag", "repository", ref.Repository(), "tag", primaryTag, "digest", digestHex, "error", err) } else { claimed = true } @@ -230,6 +283,9 @@ func (m *manager) TagImage(ctx context.Context, source, target string) (*Image, if err != nil { return nil, err } + if err := m.cancelPendingTag(targetRef.Repository(), targetRef.Tag()); err != nil { + return nil, fmt.Errorf("cancel pending image tag: %w", err) + } if err := m.installTag(sourceRef, targetRef, digestHex, meta); err != nil { return nil, err } diff --git a/lib/images/tag_test.go b/lib/images/tag_test.go index 72bfd0ef8..4d30a5f8c 100644 --- a/lib/images/tag_test.go +++ b/lib/images/tag_test.go @@ -156,13 +156,54 @@ func TestStaleTagClaimCollectsContent(t *testing.T) { meta.RequestedTag = "latest" meta.TagGeneration = 1 - require.False(t, m.claimRequestedTag(ref, meta)) + require.False(t, m.claimRequestedTags(ref, meta)) m.cleanupUnclaimedImage(ref) _, err = os.Stat(p.ImageContentDir(digest)) require.ErrorIs(t, err, os.ErrNotExist) } +func TestDeletingPendingTagDoesNotReclaimIt(t *testing.T) { + p, m, repository := newTagTestCase(t) + oldDigest := strings.Repeat("a1", 32) + newDigest := strings.Repeat("b2", 32) + seedContent(t, p, repository, "latest", oldDigest) + + pending := &imageMetadata{ + Name: repository + ":latest", Digest: "sha256:" + newDigest, + Status: StatusPending, RequestedTag: "latest", TagGeneration: 1, + PreviousTagDigest: oldDigest, CreatedAt: time.Now().UTC(), + } + require.NoError(t, os.MkdirAll(p.ImageContentDir(newDigest), 0o755)) + require.NoError(t, writeMetadataFile(p.ImageContentMetadata(newDigest), pending)) + m.tagGenerations[tagGenerationKey(repository, "latest")] = 1 + m.requestedTags = map[string]string{tagGenerationKey(repository, "latest"): newDigest} + + require.NoError(t, m.DeleteImage(context.Background(), repository+":latest")) + pending, err := readContentMetadata(p, newDigest) + require.NoError(t, err) + require.True(t, pending.TagClaimCanceled) + ref, err := ParseNormalizedRef(repository + ":latest") + require.NoError(t, err) + require.False(t, m.claimRequestedTags(NewResolvedRef(ref, "sha256:"+newDigest), pending)) + _, err = resolveTag(p, repository, "latest") + require.ErrorIs(t, err, ErrNotFound) +} + +func TestRestoreTagStateRestoresSecondaryClaims(t *testing.T) { + _, m, repository := newTagTestCase(t) + digest := strings.Repeat("c3", 32) + meta := &imageMetadata{ + Name: repository + ":latest", Digest: "sha256:" + digest, + Status: StatusConverting, RequestedTag: "latest", TagGeneration: 1, + TagClaims: []imageTagClaim{{Repository: repository, Tag: "stable", TagGeneration: 2}}, + } + m.restoreTagState([]*imageMetadata{meta}) + require.Equal(t, digest, m.requestedTags[tagGenerationKey(repository, "latest")]) + require.Equal(t, digest, m.requestedTags[tagGenerationKey(repository, "stable")]) + require.Equal(t, uint64(2), m.tagGenerations[tagGenerationKey(repository, "stable")]) +} + func TestPendingDigestClaimsAllRequestedTags(t *testing.T) { p, m, repository := newTagTestCase(t) oldDigest := "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" From b4837acb0e71e09b3557fc1180fde426a67e05c0 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:26:28 +0000 Subject: [PATCH 10/13] Reduce image manager complexity --- lib/images/manager.go | 236 +++++++++++++++++++------------------ lib/images/oci.go | 31 ++--- lib/images/storage_refs.go | 169 +++++++++++++++----------- lib/images/tag_manager.go | 91 ++++++++------ 4 files changed, 292 insertions(+), 235 deletions(-) diff --git a/lib/images/manager.go b/lib/images/manager.go index 5179906be..fb5f76bc5 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -361,42 +361,9 @@ func (m *manager) borrowedAuth(digest string, expected *inflightImagePull) (*aut } func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest, requestedPlatform Platform) (*Image, error) { - // Build one request value and reuse it for both the persisted metadata and - // the queue entry so they never diverge. - storedReq := CreateImageRequest{ - Name: ref.String(), - Tags: tags.Clone(req.Tags), - Platform: req.Platform, - } - // Record the requested platform optimistically while the build is pending; - // buildImage overwrites it with the authoritative manifest platform once the - // image config is pulled. - previousTagDigest := "" - var err error - if ref.Tag() != "" { - previousTagDigest, err = resolveTag(m.paths, ref.Repository(), ref.Tag()) - if err != nil && !errors.Is(err, ErrNotFound) { - return nil, fmt.Errorf("resolve existing image tag: %w", err) - } - } - tagGeneration := uint64(0) - if ref.Tag() != "" { - tagGeneration = m.nextTagGeneration(ref.Repository(), ref.Tag()) - m.trackRequestedTag(ref.Repository(), ref.Tag(), ref.DigestHex()) - } - meta := &imageMetadata{ - Name: ref.String(), - Digest: ref.Digest(), - Platform: requestedPlatform.String(), - Status: StatusPending, - Request: &storedReq, - BorrowedAuth: req.Credentials != nil, - BuildID: uuid.New().String(), - Tags: tags.Clone(req.Tags), - RequestedTag: ref.Tag(), - PreviousTagDigest: previousTagDigest, - TagGeneration: tagGeneration, - CreatedAt: time.Now(), + meta, previousTagDigest, err := m.newPendingImageMetadata(ref, req, requestedPlatform) + if err != nil { + return nil, err } // Write initial metadata @@ -444,6 +411,41 @@ func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest, return img, nil } +func (m *manager) newPendingImageMetadata(ref *ResolvedRef, req CreateImageRequest, requestedPlatform Platform) (*imageMetadata, string, error) { + previousTagDigest := "" + if ref.Tag() != "" { + var err error + previousTagDigest, err = resolveTag(m.paths, ref.Repository(), ref.Tag()) + if err != nil && !errors.Is(err, ErrNotFound) { + return nil, "", fmt.Errorf("resolve existing image tag: %w", err) + } + } + tagGeneration := uint64(0) + if ref.Tag() != "" { + tagGeneration = m.nextTagGeneration(ref.Repository(), ref.Tag()) + m.trackRequestedTag(ref.Repository(), ref.Tag(), ref.DigestHex()) + } + storedReq := CreateImageRequest{ + Name: ref.String(), + Tags: tags.Clone(req.Tags), + Platform: req.Platform, + } + return &imageMetadata{ + Name: ref.String(), + Digest: ref.Digest(), + Platform: requestedPlatform.String(), + Status: StatusPending, + Request: &storedReq, + BorrowedAuth: req.Credentials != nil, + BuildID: uuid.New().String(), + Tags: tags.Clone(req.Tags), + RequestedTag: ref.Tag(), + PreviousTagDigest: previousTagDigest, + TagGeneration: tagGeneration, + CreatedAt: time.Now(), + }, previousTagDigest, nil +} + func (m *manager) buildImage(ctx context.Context, ref *ResolvedRef, credentials *authn.AuthConfig, buildID string) { buildStart := time.Now() buildStatus := "failed" @@ -551,25 +553,9 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i return err } + modelPath := manifestModelPath(m.paths, layout, ref.DigestHex()) diskInstalled := false - modelPath := m.paths.ImageContentManifestModel(ref.DigestHex()) - if !layout.content { - modelPath = filepath.Join(layout.dir, "manifest.json") - } modelWritten := false - rollback := func(cause error) error { - var rollbackErr error - if modelWritten { - rollbackErr = errors.Join(rollbackErr, os.Remove(modelPath)) - } - if diskInstalled { - rollbackErr = errors.Join(rollbackErr, os.Remove(layout.disk)) - } - if rollbackErr != nil { - return errors.Join(cause, fmt.Errorf("rollback finalization: %w", rollbackErr)) - } - return cause - } if err := installAtomically(layout.disk, func(path string) error { return os.Rename(diskTempPath, path) @@ -585,7 +571,7 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i model := *result.Manifest model.Platform = actualPlatform.String() if err := writeManifestModelAt(modelPath, ref.DigestHex(), &model); err != nil { - return rollback(fmt.Errorf("write manifest model: %w", err)) + return rollbackFinalization(layout, modelPath, diskInstalled, modelWritten, fmt.Errorf("write manifest model: %w", err)) } modelWritten = true } @@ -601,7 +587,7 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i meta.WorkingDir = result.Metadata.WorkingDir if err := writeMetadataFile(layout.metadata, meta); err != nil { - return rollback(fmt.Errorf("write final metadata: %w", err)) + return rollbackFinalization(layout, modelPath, diskInstalled, modelWritten, fmt.Errorf("write final metadata: %w", err)) } m.notifyReady(ref.DigestHex(), StatusReady, nil) @@ -612,6 +598,27 @@ func (m *manager) finalizeImage(ref *ResolvedRef, result *pullResult, diskSize i return nil } +func manifestModelPath(p *paths.Paths, layout imageLayout, digestHex string) string { + if layout.content { + return p.ImageContentManifestModel(digestHex) + } + return filepath.Join(layout.dir, "manifest.json") +} + +func rollbackFinalization(layout imageLayout, modelPath string, diskInstalled, modelWritten bool, cause error) error { + var rollbackErr error + if modelWritten { + rollbackErr = errors.Join(rollbackErr, os.Remove(modelPath)) + } + if diskInstalled { + rollbackErr = errors.Join(rollbackErr, os.Remove(layout.disk)) + } + if rollbackErr != nil { + return errors.Join(cause, fmt.Errorf("rollback finalization: %w", rollbackErr)) + } + return cause +} + func phaseStatus(err error) string { if err != nil { return "failed" @@ -691,7 +698,6 @@ func (m *manager) RecoverInterruptedBuilds() { return // Best effort } - // Sort by created_at to maintain FIFO order sort.Slice(metas, func(i, j int) bool { return metas[i].CreatedAt.Before(metas[j].CreatedAt) }) @@ -699,32 +705,38 @@ func (m *manager) RecoverInterruptedBuilds() { seenDigests := make(map[string]struct{}) for _, meta := range metas { - if meta.Status != StatusPending && meta.Status != StatusPulling && meta.Status != StatusConverting { - continue - } - if meta.Request == nil || meta.Digest == "" { - continue - } if _, seen := seenDigests[meta.Digest]; seen { continue } - seenDigests[meta.Digest] = struct{}{} - normalized, err := ParseNormalizedRef(meta.Name) - if err != nil { - continue - } - ref := NewResolvedRef(normalized, meta.Digest) - if meta.BorrowedAuth && (meta.Status != StatusConverting || !m.ociClient.existsInLayout(digestToLayoutTag(meta.Digest))) { - m.updateStatusByDigest(ref, StatusFailed, ErrBorrowedCredentialsExpired, meta.BuildID) - continue + if m.recoverInterruptedBuild(meta) { + seenDigests[meta.Digest] = struct{}{} } - buildID := meta.BuildID - m.queue.Enqueue(meta.Digest, func() { - m.buildImage(context.Background(), ref, nil, buildID) - }, nil) } } +func (m *manager) recoverInterruptedBuild(meta *imageMetadata) bool { + if meta.Status != StatusPending && meta.Status != StatusPulling && meta.Status != StatusConverting { + return false + } + if meta.Request == nil || meta.Digest == "" { + return false + } + normalized, err := ParseNormalizedRef(meta.Name) + if err != nil { + return false + } + ref := NewResolvedRef(normalized, meta.Digest) + if meta.BorrowedAuth && (meta.Status != StatusConverting || !m.ociClient.existsInLayout(digestToLayoutTag(meta.Digest))) { + m.updateStatusByDigest(ref, StatusFailed, ErrBorrowedCredentialsExpired, meta.BuildID) + return true + } + buildID := meta.BuildID + m.queue.Enqueue(meta.Digest, func() { + m.buildImage(context.Background(), ref, nil, buildID) + }, nil) + return true +} + func (m *manager) GetImage(ctx context.Context, name string) (*Image, error) { // Parse and normalize the reference ref, err := ParseNormalizedRef(name) @@ -747,54 +759,50 @@ func (m *manager) GetImage(ctx context.Context, name string) (*Image, error) { } func (m *manager) DeleteImage(ctx context.Context, name string) error { - // Parse and normalize the reference ref, err := ParseNormalizedRef(name) if err != nil { return fmt.Errorf("%w: %s", ErrInvalidName, err.Error()) } - repository := ref.Repository() - - // Hold createMu during delete so tag resolution, tag removal, and orphan checks - // stay consistent with concurrent creates for the same repository digest. m.createMu.Lock() defer m.createMu.Unlock() if ref.IsDigest() { - digestHex := ref.DigestHex() - if _, err := readMetadata(m.paths, repository, digestHex); err != nil { - return err - } - tags, err := tagsForDigest(m.paths, repository, digestHex) - if err != nil { - return err - } - if err := deleteTagsForDigest(m.paths, repository, digestHex); err != nil { - return err - } - for _, tag := range tags { - m.forgetTagState(repository, tag) - if err := m.cancelPendingTag(repository, tag); err != nil { - return fmt.Errorf("cancel pending image tag: %w", err) - } - } - if err := removeDigestIfUnreferenced(m.paths, repository, digestHex, false); err != nil { - return err - } - m.clearRequestedDigest(digestHex) - m.refreshDiskUsageTotals() - return nil + return m.deleteDigestImage(ref.Repository(), ref.DigestHex()) } + return m.deleteTaggedImage(ref.Repository(), ref.Tag()) +} - tag := ref.Tag() +func (m *manager) deleteDigestImage(repository, digestHex string) error { + if _, err := readMetadata(m.paths, repository, digestHex); err != nil { + return err + } + tags, err := tagsForDigest(m.paths, repository, digestHex) + if err != nil { + return err + } + if err := deleteTagsForDigest(m.paths, repository, digestHex); err != nil { + return err + } + for _, tag := range tags { + m.forgetTagState(repository, tag) + if err := m.cancelPendingTag(repository, tag); err != nil { + return fmt.Errorf("cancel pending image tag: %w", err) + } + } + if err := removeDigestIfUnreferenced(m.paths, repository, digestHex, false); err != nil { + return err + } + m.clearRequestedDigest(digestHex) + m.refreshDiskUsageTotals() + return nil +} - // Resolve the tag to get the digest before deleting +func (m *manager) deleteTaggedImage(repository, tag string) error { digestHex, err := resolveTag(m.paths, repository, tag) if err != nil { return err } - - // Delete the tag symlink if err := deleteTag(m.paths, repository, tag); err != nil { return err } @@ -803,19 +811,17 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error { return fmt.Errorf("cancel pending image tag: %w", err) } - // Check if the digest is now orphaned (no other tags reference it) count, err := countTagsForDigest(m.paths, repository, digestHex) if err != nil { return fmt.Errorf("count tags for digest %s: %w", digestHex, err) } - - if count == 0 { - if err := removeDigestIfUnreferenced(m.paths, repository, digestHex, true); err != nil { - return fmt.Errorf("delete orphaned digest %s: %w", digestHex, err) - } - m.refreshDiskUsageTotals() + if count > 0 { + return nil } - + if err := removeDigestIfUnreferenced(m.paths, repository, digestHex, true); err != nil { + return fmt.Errorf("delete orphaned digest %s: %w", digestHex, err) + } + m.refreshDiskUsageTotals() return nil } diff --git a/lib/images/oci.go b/lib/images/oci.go index 2a1a23223..88bd6de24 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -428,6 +428,17 @@ func (c *ociClient) extractOCIImageBundle(layoutTag string) (*ociImageBundle, er return nil, fmt.Errorf("get config digest: %w", err) } + meta := containerMetadataFromConfig(configFile) + model, compressedBytes := manifestModelFromImage(layoutTag, configFile, manifest, configDigest) + return &ociImageBundle{ + Meta: meta, + Model: model, + LayerCount: len(manifest.Layers), + CompressedBytes: compressedBytes, + }, nil +} + +func containerMetadataFromConfig(configFile *gcr.ConfigFile) *containerMetadata { meta := &containerMetadata{ OS: configFile.OS, Architecture: configFile.Architecture, @@ -438,21 +449,18 @@ func (c *ociClient) extractOCIImageBundle(layoutTag string) (*ociImageBundle, er Labels: make(map[string]string), WorkingDir: configFile.Config.WorkingDir, } - // Parse environment variables for _, env := range configFile.Config.Env { - for i := 0; i < len(env); i++ { - if env[i] == '=' { - key := env[:i] - val := env[i+1:] - meta.Env[key] = val - break - } + if key, value, ok := strings.Cut(env, "="); ok { + meta.Env[key] = value } } for key, value := range configFile.Config.Labels { meta.Labels[key] = value } + return meta +} +func manifestModelFromImage(layoutTag string, configFile *gcr.ConfigFile, manifest *gcr.Manifest, configDigest gcr.Hash) (*imageManifestModel, int64) { model := &imageManifestModel{ SchemaVersion: manifestModelSchemaVersion, Digest: digestFromHex(layoutTag), @@ -486,12 +494,7 @@ func (c *ociClient) extractOCIImageBundle(layoutTag string) (*ociImageBundle, er } model.Layers = append(model.Layers, layer) } - return &ociImageBundle{ - Meta: meta, - Model: model, - LayerCount: len(manifest.Layers), - CompressedBytes: compressedBytes, - }, nil + return model, compressedBytes } // unpackLayers unpacks all OCI layers to a target directory using umoci diff --git a/lib/images/storage_refs.go b/lib/images/storage_refs.go index 7502eb33d..8ac89e6f9 100644 --- a/lib/images/storage_refs.go +++ b/lib/images/storage_refs.go @@ -81,93 +81,122 @@ func promoteLegacyImages(p *paths.Paths, refs []legacyRef) { } } +type metadataIndex struct { + seen map[string]struct{} + contentDigests map[string]struct{} + taggedDigests map[string]struct{} + taggedContentDigests map[string]struct{} + metadataRefs []metadataReference + seenMetadataRefs map[string]struct{} + metas []*imageMetadata +} + func listAllMetadata(p *paths.Paths) ([]*imageMetadata, error) { - imagesDir := p.ImagesDir() - seen := make(map[string]struct{}) - contentDigests := make(map[string]struct{}) - taggedDigests := make(map[string]struct{}) - taggedContentDigests := make(map[string]struct{}) - metadataRefs := make([]metadataReference, 0) - seenMetadataRefs := make(map[string]struct{}) - metas := make([]*imageMetadata, 0) + index := metadataIndex{ + seen: make(map[string]struct{}), + contentDigests: make(map[string]struct{}), + taggedDigests: make(map[string]struct{}), + taggedContentDigests: make(map[string]struct{}), + metadataRefs: make([]metadataReference, 0), + seenMetadataRefs: make(map[string]struct{}), + metas: make([]*imageMetadata, 0), + } + if err := index.walk(p); err != nil { + return nil, err + } + index.appendUnreferenced(p) + return index.metas, nil +} +func (i *metadataIndex) walk(p *paths.Paths) error { + imagesDir := p.ImagesDir() err := filepath.Walk(imagesDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return nil // Skip errors - } - rel, err := filepath.Rel(imagesDir, path) - if err != nil { - return nil - } - parts := strings.Split(rel, string(filepath.Separator)) - if len(parts) > 0 && parts[0] == "content" { - if info.IsDir() { - return nil - } - if info.Name() != "metadata.json" { - return nil - } - digestHex := filepath.Base(filepath.Dir(path)) - contentDigests[digestHex] = struct{}{} - return nil - } + return i.visit(p, imagesDir, path, info, err) + }) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("walk images directory: %w", err) + } + return nil +} - switch { - case info.Mode()&os.ModeSymlink != 0: - digestHex, err := os.Readlink(path) - if err != nil { - return nil // Skip invalid symlinks - } - digestHex = filepath.Base(digestHex) +func (i *metadataIndex) visit(p *paths.Paths, imagesDir, path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return nil + } + rel, err := filepath.Rel(imagesDir, path) + if err != nil { + return nil + } + parts := strings.Split(rel, string(filepath.Separator)) + if len(parts) > 0 && parts[0] == "content" { + return i.visitContent(path, info) + } + if info.Mode()&os.ModeSymlink != 0 { + return i.visitTag(p, path, rel, parts) + } + if !info.IsDir() && info.Name() == "metadata.json" { + i.recordMetadataRef(imagesDir, path) + } + return nil +} - var repository, tag string - if len(parts) > 1 && parts[0] == "repositories" { - repository = filepath.Join(parts[1 : len(parts)-1]...) - tag = parts[len(parts)-1] - } else { - repository = filepath.Dir(rel) - tag = filepath.Base(path) - } - return appendMetadataForTag(p, repository, tag, digestHex, seen, taggedDigests, taggedContentDigests, &metas) - case !info.IsDir() && info.Name() == "metadata.json": - digestHex := filepath.Base(filepath.Dir(path)) - repository, err := filepath.Rel(imagesDir, filepath.Dir(filepath.Dir(path))) - if err != nil { - return nil - } - key := repository + "@" + digestHex - if _, ok := seenMetadataRefs[key]; !ok { - seenMetadataRefs[key] = struct{}{} - metadataRefs = append(metadataRefs, metadataReference{repository: repository, digestHex: digestHex}) - } - return nil - default: - return nil - } - }) +func (i *metadataIndex) visitContent(path string, info os.FileInfo) error { + if info.IsDir() || info.Name() != "metadata.json" { + return nil + } + i.contentDigests[filepath.Base(filepath.Dir(path))] = struct{}{} + return nil +} - if err != nil && !os.IsNotExist(err) { - return nil, fmt.Errorf("walk images directory: %w", err) +func (i *metadataIndex) visitTag(p *paths.Paths, path, rel string, parts []string) error { + digestHex, err := os.Readlink(path) + if err != nil { + return nil + } + digestHex = filepath.Base(digestHex) + var repository, tag string + if len(parts) > 1 && parts[0] == "repositories" { + repository = filepath.Join(parts[1 : len(parts)-1]...) + tag = parts[len(parts)-1] + } else { + repository = filepath.Dir(rel) + tag = filepath.Base(path) } - seenDigests := make(map[string]struct{}, len(metas)) - for _, ref := range metadataRefs { - if _, tagged := taggedDigests[ref.repository+"@"+ref.digestHex]; tagged { + return appendMetadataForTag(p, repository, tag, digestHex, i.seen, i.taggedDigests, i.taggedContentDigests, &i.metas) +} + +func (i *metadataIndex) recordMetadataRef(imagesDir, path string) { + digestHex := filepath.Base(filepath.Dir(path)) + repository, err := filepath.Rel(imagesDir, filepath.Dir(filepath.Dir(path))) + if err != nil { + return + } + key := repository + "@" + digestHex + if _, ok := i.seenMetadataRefs[key]; ok { + return + } + i.seenMetadataRefs[key] = struct{}{} + i.metadataRefs = append(i.metadataRefs, metadataReference{repository: repository, digestHex: digestHex}) +} + +func (i *metadataIndex) appendUnreferenced(p *paths.Paths) { + seenDigests := make(map[string]struct{}, len(i.metas)) + for _, ref := range i.metadataRefs { + if _, tagged := i.taggedDigests[ref.repository+"@"+ref.digestHex]; tagged { continue } - appendMetadataIfNew(p, ref.repository, ref.digestHex, seen, &metas) + appendMetadataIfNew(p, ref.repository, ref.digestHex, i.seen, &i.metas) seenDigests[ref.digestHex] = struct{}{} } - for digestHex := range contentDigests { - if _, found := taggedContentDigests[digestHex]; found { + for digestHex := range i.contentDigests { + if _, found := i.taggedContentDigests[digestHex]; found { continue } if _, found := seenDigests[digestHex]; found { continue } - appendContentMetadataIfNew(p, digestHex, seen, &metas) + appendContentMetadataIfNew(p, digestHex, i.seen, &i.metas) } - - return metas, nil } type metadataReference struct { diff --git a/lib/images/tag_manager.go b/lib/images/tag_manager.go index fadcd9d79..def203d10 100644 --- a/lib/images/tag_manager.go +++ b/lib/images/tag_manager.go @@ -165,36 +165,42 @@ func (m *manager) cancelPendingTag(repository, tag string) error { return err } for _, meta := range metas { - if meta.Status != StatusPending && meta.Status != StatusPulling && meta.Status != StatusConverting { + if !isPendingImageStatus(meta.Status) { continue } ref, err := ParseNormalizedRef(meta.Name) - if err != nil { + if err != nil || !cancelTagClaim(meta, ref, repository, tag) { continue } - changed := false - if ref.Repository() == repository && meta.RequestedTag == tag { - meta.TagClaimCanceled = true - changed = true - } - claims := meta.TagClaims[:0] - for _, claim := range meta.TagClaims { - if claim.Repository == repository && claim.Tag == tag { - changed = true - continue - } - claims = append(claims, claim) - } - meta.TagClaims = claims - if changed { - if err := writeMetadata(m.paths, ref.Repository(), strings.TrimPrefix(meta.Digest, "sha256:"), meta); err != nil { - return err - } + if err := writeMetadata(m.paths, ref.Repository(), strings.TrimPrefix(meta.Digest, "sha256:"), meta); err != nil { + return err } } return nil } +func isPendingImageStatus(status string) bool { + return status == StatusPending || status == StatusPulling || status == StatusConverting +} + +func cancelTagClaim(meta *imageMetadata, ref *NormalizedRef, repository, tag string) bool { + changed := false + if ref.Repository() == repository && meta.RequestedTag == tag { + meta.TagClaimCanceled = true + changed = true + } + claims := meta.TagClaims[:0] + for _, claim := range meta.TagClaims { + if claim.Repository == repository && claim.Tag == tag { + changed = true + continue + } + claims = append(claims, claim) + } + meta.TagClaims = claims + return changed +} + func (m *manager) requestedTagImage(ref *NormalizedRef) *Image { m.createMu.Lock() digestHex, ok := m.requestedTags[tagGenerationKey(ref.Repository(), ref.Tag())] @@ -211,6 +217,19 @@ func (m *manager) requestedTagImage(ref *NormalizedRef) *Image { func (m *manager) claimRequestedTags(ref *ResolvedRef, meta *imageMetadata) bool { digestHex := ref.DigestHex() + claimed := m.claimMatchingTags(digestHex) + if m.claimPrimaryTag(ref, meta, digestHex) { + claimed = true + } + if claimed { + return true + } + // Digest-only images have no tag claim but must remain addressable by their + // digest after finalization. + return meta.RequestedTag == "" && ref.Tag() == "" +} + +func (m *manager) claimMatchingTags(digestHex string) bool { claimed := false for key, requestedDigest := range m.requestedTags { if requestedDigest != digestHex { @@ -228,29 +247,29 @@ func (m *manager) claimRequestedTags(ref *ResolvedRef, meta *imageMetadata) bool delete(m.requestedTags, key) claimed = true } + return claimed +} - // Older metadata did not persist RequestedTag or a requestedTags entry. Keep - // those recovered builds addressable by falling back to the build reference. +func (m *manager) claimPrimaryTag(ref *ResolvedRef, meta *imageMetadata, digestHex string) bool { + if meta.TagClaimCanceled { + return false + } primaryTag := meta.RequestedTag if primaryTag == "" { primaryTag = ref.Tag() } - if primaryTag != "" && !meta.TagClaimCanceled { - primaryKey := tagGenerationKey(ref.Repository(), primaryTag) - if _, hasRequest := m.requestedTags[primaryKey]; !hasRequest && m.tagClaimIsCurrent(ref.Repository(), primaryTag, digestHex, meta, meta.RequestedTag == "") { - if err := createTagSymlink(m.paths, ref.Repository(), primaryTag, digestHex); err != nil { - slog.Warn("failed to claim image tag", "repository", ref.Repository(), "tag", primaryTag, "digest", digestHex, "error", err) - } else { - claimed = true - } - } + if primaryTag == "" { + return false } - if claimed { - return true + primaryKey := tagGenerationKey(ref.Repository(), primaryTag) + if _, hasRequest := m.requestedTags[primaryKey]; hasRequest || !m.tagClaimIsCurrent(ref.Repository(), primaryTag, digestHex, meta, meta.RequestedTag == "") { + return false } - // Digest-only images have no tag claim but must remain addressable by their - // digest after finalization. - return meta.RequestedTag == "" && ref.Tag() == "" + if err := createTagSymlink(m.paths, ref.Repository(), primaryTag, digestHex); err != nil { + slog.Warn("failed to claim image tag", "repository", ref.Repository(), "tag", primaryTag, "digest", digestHex, "error", err) + return false + } + return true } func (m *manager) tagClaimIsCurrent(repository, tag, digest string, meta *imageMetadata, allowMissing bool) bool { From f19b3ad7958018b18d33369c3bfe023da3b062dc Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:39:50 +0000 Subject: [PATCH 11/13] Fix image tag recovery regressions --- lib/images/manager.go | 20 ++++++------ lib/images/storage.go | 12 +++++-- lib/images/storage_refs.go | 65 +++++------------------------------- lib/images/storage_test.go | 67 +++++++------------------------------- lib/images/tag_manager.go | 57 +++++++++++++++++++++++--------- lib/images/tag_test.go | 31 ++++++++++++++---- 6 files changed, 104 insertions(+), 148 deletions(-) diff --git a/lib/images/manager.go b/lib/images/manager.go index fb5f76bc5..ca5203901 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -77,7 +77,7 @@ type manager struct { createMu sync.Mutex diskUsageMu sync.RWMutex tagGenerations map[string]uint64 - requestedTags map[string]string // newest pull's digest per requested tag + requestedTags map[requestedTagKey]string // newest pull's digest per requested tag diskUsageLoaded bool readyImageBytes int64 ociCacheBytes int64 @@ -106,7 +106,7 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma borrowedCredentialsTimeout: DefaultBorrowedCredentialsTimeout, readySubscribers: make(map[string][]chan StatusEvent), tagGenerations: make(map[string]uint64), - requestedTags: make(map[string]string), + requestedTags: make(map[requestedTagKey]string), } // Initialize metrics if meter is provided @@ -688,7 +688,7 @@ func (m *manager) updateStatusByDigest(ref *ResolvedRef, status string, err erro if meta.RequestedTag != "" { m.releaseTagGeneration(ref.Repository(), meta.RequestedTag, meta.TagGeneration) } - m.clearRequestedDigest(strings.TrimPrefix(meta.Digest, "sha256:")) + m.clearRequestedDigest(meta.digestHex()) } } @@ -715,7 +715,7 @@ func (m *manager) RecoverInterruptedBuilds() { } func (m *manager) recoverInterruptedBuild(meta *imageMetadata) bool { - if meta.Status != StatusPending && meta.Status != StatusPulling && meta.Status != StatusConverting { + if !isPendingImageStatus(meta.Status) { return false } if meta.Request == nil || meta.Digest == "" { @@ -777,18 +777,18 @@ func (m *manager) deleteDigestImage(repository, digestHex string) error { if _, err := readMetadata(m.paths, repository, digestHex); err != nil { return err } - tags, err := tagsForDigest(m.paths, repository, digestHex) + digestTags, err := tagsForDigest(m.paths, repository, digestHex) if err != nil { return err } - if err := deleteTagsForDigest(m.paths, repository, digestHex); err != nil { + if err := deleteTagsForDigest(m.paths, repository, digestTags); err != nil { return err } - for _, tag := range tags { + for _, tag := range digestTags { m.forgetTagState(repository, tag) - if err := m.cancelPendingTag(repository, tag); err != nil { - return fmt.Errorf("cancel pending image tag: %w", err) - } + } + if err := m.cancelPendingTags(repository, digestTags); err != nil { + return fmt.Errorf("cancel pending image tags: %w", err) } if err := removeDigestIfUnreferenced(m.paths, repository, digestHex, false); err != nil { return err diff --git a/lib/images/storage.go b/lib/images/storage.go index eaaafbe02..72c57e213 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -44,6 +44,10 @@ type imageTagClaim struct { TagGeneration uint64 `json:"tag_generation,omitempty"` } +func (m *imageMetadata) digestHex() string { + return strings.TrimPrefix(m.Digest, "sha256:") +} + func (m *imageMetadata) toImageFor(reference string) *Image { img := m.toImage() img.Name = reference @@ -264,12 +268,14 @@ func readMetadataAt(layout imageLayout) (*imageMetadata, error) { func promoteImageToContent(p *paths.Paths, sourceRepository, digestHex string, sourceMeta *imageMetadata) error { contentMeta, err := readContentMetadata(p, digestHex) if err == nil { - if contentMeta.Status == StatusReady { + switch contentMeta.Status { + case StatusReady: return promoteLegacyTags(p, sourceRepository, digestHex) + case StatusPending, StatusPulling, StatusConverting: + return fmt.Errorf("%w: content status is %s", ErrImageNotReady, contentMeta.Status) } - return fmt.Errorf("%w: content status is %s", ErrImageNotReady, contentMeta.Status) } - if !errors.Is(err, ErrNotFound) { + if err != nil && !errors.Is(err, ErrNotFound) { return err } diff --git a/lib/images/storage_refs.go b/lib/images/storage_refs.go index 8ac89e6f9..78eecef70 100644 --- a/lib/images/storage_refs.go +++ b/lib/images/storage_refs.go @@ -36,51 +36,6 @@ func listTags(p *paths.Paths, repository string) ([]string, error) { return tags, nil } -type legacyRef struct { - repository string - digestHex string -} - -func collectLegacyImages(p *paths.Paths) ([]legacyRef, error) { - imagesDir := p.ImagesDir() - refs := make([]legacyRef, 0) - err := filepath.Walk(imagesDir, func(path string, info os.FileInfo, err error) error { - if err != nil || info.IsDir() || info.Name() != "metadata.json" { - return nil - } - rel, relErr := filepath.Rel(imagesDir, path) - if relErr != nil { - return nil - } - parts := strings.Split(rel, string(filepath.Separator)) - if len(parts) < 3 || parts[0] == "content" || parts[0] == "repositories" { - return nil - } - refs = append(refs, legacyRef{ - repository: filepath.Join(parts[:len(parts)-2]...), - digestHex: parts[len(parts)-2], - }) - return nil - }) - if err != nil && !os.IsNotExist(err) { - return nil, err - } - return refs, nil -} - -func promoteLegacyImages(p *paths.Paths, refs []legacyRef) { - for _, ref := range refs { - layout := resolveImageLayout(p, ref.repository, ref.digestHex) - meta, readErr := readMetadataAt(layout) - if readErr != nil || meta.Status != StatusReady { - continue - } - if promoteErr := promoteImageToContent(p, ref.repository, ref.digestHex, meta); promoteErr != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to promote legacy image %s@%s: %v\n", ref.repository, ref.digestHex, promoteErr) - } - } -} - type metadataIndex struct { seen map[string]struct{} contentDigests map[string]struct{} @@ -162,7 +117,7 @@ func (i *metadataIndex) visitTag(p *paths.Paths, path, rel string, parts []strin repository = filepath.Dir(rel) tag = filepath.Base(path) } - return appendMetadataForTag(p, repository, tag, digestHex, i.seen, i.taggedDigests, i.taggedContentDigests, &i.metas) + return i.appendMetadataForTag(p, repository, tag, digestHex) } func (i *metadataIndex) recordMetadataRef(imagesDir, path string) { @@ -232,9 +187,9 @@ func appendContentMetadataIfNew(p *paths.Paths, digestHex string, seen map[strin *metas = append(*metas, meta) } -func appendMetadataForTag(p *paths.Paths, repository, tag, digestHex string, seen, taggedDigests, taggedContentDigests map[string]struct{}, metas *[]*imageMetadata) error { +func (i *metadataIndex) appendMetadataForTag(p *paths.Paths, repository, tag, digestHex string) error { tagKey := repository + ":" + tag - if _, ok := seen[tagKey]; ok { + if _, ok := i.seen[tagKey]; ok { return nil } meta, err := readMetadata(p, repository, digestHex) @@ -242,10 +197,10 @@ func appendMetadataForTag(p *paths.Paths, repository, tag, digestHex string, see return nil } meta.Name = repository + ":" + tag - seen[tagKey] = struct{}{} - taggedDigests[repository+"@"+digestHex] = struct{}{} - taggedContentDigests[digestHex] = struct{}{} - *metas = append(*metas, meta) + i.seen[tagKey] = struct{}{} + i.taggedDigests[repository+"@"+digestHex] = struct{}{} + i.taggedContentDigests[digestHex] = struct{}{} + i.metas = append(i.metas, meta) return nil } @@ -293,11 +248,7 @@ func countTagsForDigest(p *paths.Paths, repository, digestHex string) (int, erro return len(tags), err } -func deleteTagsForDigest(p *paths.Paths, repository, digestHex string) error { - tags, err := tagsForDigest(p, repository, digestHex) - if err != nil { - return err - } +func deleteTagsForDigest(p *paths.Paths, repository string, tags []string) error { for _, tag := range tags { if err := deleteTag(p, repository, tag); err != nil && !errors.Is(err, ErrNotFound) { return err diff --git a/lib/images/storage_test.go b/lib/images/storage_test.go index 24a2a9db3..003bdf26c 100644 --- a/lib/images/storage_test.go +++ b/lib/images/storage_test.go @@ -251,68 +251,23 @@ func TestNewManagerLeavesLegacyImagesInPlace(t *testing.T) { requireTagResolvesTo(t, p, repository, tag, digest) } -func TestPromoteLegacyImagesMovesContentAndTags(t *testing.T) { +func TestPromoteImageToContentReplacesFailedContent(t *testing.T) { p := paths.New(t.TempDir()) repository := "docker.io/library/alpine" - tag := "latest" digest := "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" - - // Ready legacy image with a legacy tag symlink. - legacyDir := p.ImageDigestDir(repository, digest) - require.NoError(t, os.MkdirAll(legacyDir, 0o755)) - meta := &imageMetadata{ - Name: repository + ":" + tag, - Digest: "sha256:" + digest, - Status: StatusReady, - SizeBytes: 7, - CreatedAt: time.Now().UTC(), - } - require.NoError(t, writeMetadataFile(p.ImageMetadata(repository, digest), meta)) - require.NoError(t, os.WriteFile(p.ImageDigestPath(repository, digest), []byte("rootfs!"), 0o644)) - tagPath := p.ImageTagSymlink(repository, tag) - require.NoError(t, os.MkdirAll(filepath.Dir(tagPath), 0o755)) - require.NoError(t, os.Symlink(digest, tagPath)) - - refs, err := collectLegacyImages(p) - require.NoError(t, err) - promoteLegacyImages(p, refs) - - // Content exists with the same bytes and is ready. - contentMeta, err := readContentMetadata(p, digest) - require.NoError(t, err) - require.Equal(t, StatusReady, contentMeta.Status) - data, err := os.ReadFile(p.ImageContentPath(digest)) + seedLegacy(t, p, repository, "latest", digest) + legacyMeta, err := readMetadata(p, repository, digest) require.NoError(t, err) - require.Equal(t, "rootfs!", string(data)) - require.FileExists(t, p.ImageDigestPath(repository, digest)) - - // The tag now resolves through the shared content layout. - resolved, err := resolveTag(p, repository, tag) - require.NoError(t, err) - require.Equal(t, digest, resolved) -} - -func TestPromoteLegacyImagesSkipsNonReady(t *testing.T) { - p := paths.New(t.TempDir()) - repository := "docker.io/library/alpine" - digest := "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" - - legacyDir := p.ImageDigestDir(repository, digest) - require.NoError(t, os.MkdirAll(legacyDir, 0o755)) - require.NoError(t, writeMetadataFile(p.ImageMetadata(repository, digest), &imageMetadata{ - Name: repository + ":latest", - Digest: "sha256:" + digest, - Status: StatusConverting, - CreatedAt: time.Now().UTC(), + require.NoError(t, os.MkdirAll(p.ImageContentDir(digest), 0o755)) + require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digest), &imageMetadata{ + Digest: "sha256:" + digest, + Status: StatusFailed, })) - refs, err := collectLegacyImages(p) + require.NoError(t, promoteImageToContent(p, repository, digest, legacyMeta)) + contentMeta, err := readContentMetadata(p, digest) require.NoError(t, err) - promoteLegacyImages(p, refs) - - _, err = os.Stat(p.ImageContentMetadata(digest)) - require.True(t, os.IsNotExist(err), "non-ready legacy image must not be promoted") - _, err = os.Stat(legacyDir) - require.NoError(t, err, "non-ready legacy tree must be preserved") + require.Equal(t, StatusReady, contentMeta.Status) + require.FileExists(t, p.ImageContentPath(digest)) } diff --git a/lib/images/tag_manager.go b/lib/images/tag_manager.go index def203d10..eafce2d3b 100644 --- a/lib/images/tag_manager.go +++ b/lib/images/tag_manager.go @@ -10,16 +10,25 @@ import ( "github.com/kernel/hypeman/lib/paths" ) +type requestedTagKey struct { + repository string + tag string +} + func tagGenerationKey(repository, tag string) string { return repository + ":" + tag } +func requestedTagKeyFor(repository, tag string) requestedTagKey { + return requestedTagKey{repository: repository, tag: tag} +} + func (m *manager) ensureTagState() { if m.tagGenerations == nil { m.tagGenerations = make(map[string]uint64) } if m.requestedTags == nil { - m.requestedTags = make(map[string]string) + m.requestedTags = make(map[requestedTagKey]string) } } @@ -53,13 +62,13 @@ func (m *manager) forgetTagState(repository, tag string) { func (m *manager) restoreTagState(metas []*imageMetadata) { m.ensureTagState() for _, meta := range metas { - if meta.Status != StatusPending && meta.Status != StatusPulling && meta.Status != StatusConverting { + if !isPendingImageStatus(meta.Status) { continue } if meta.Digest == "" { continue } - digestHex := strings.TrimPrefix(meta.Digest, "sha256:") + digestHex := meta.digestHex() ref, err := ParseNormalizedRef(meta.Name) if err != nil { continue @@ -81,7 +90,7 @@ func (m *manager) restoreRequestedTag(repository, tag, digestHex string, generat if generation > m.tagGenerations[key] { m.tagGenerations[key] = generation } - m.requestedTags[key] = digestHex + m.requestedTags[requestedTagKeyFor(repository, tag)] = digestHex } func (m *manager) claimTagForStatus(meta *imageMetadata, ref *ResolvedRef) error { @@ -135,17 +144,17 @@ func (m *manager) claimReadyTag(repository, tag, digestHex string) error { m.revertTagGeneration(repository, tag) return err } - m.clearRequestedTag(repository, tag, digestHex) + m.clearRequestedTag(repository, tag, "") return nil } func (m *manager) trackRequestedTag(repository, tag, digestHex string) { m.ensureTagState() - m.requestedTags[tagGenerationKey(repository, tag)] = digestHex + m.requestedTags[requestedTagKeyFor(repository, tag)] = digestHex } func (m *manager) clearRequestedTag(repository, tag, digestHex string) { - key := tagGenerationKey(repository, tag) + key := requestedTagKeyFor(repository, tag) if current, ok := m.requestedTags[key]; ok && (digestHex == "" || current == digestHex) { delete(m.requestedTags, key) } @@ -160,6 +169,17 @@ func (m *manager) clearRequestedDigest(digestHex string) { } func (m *manager) cancelPendingTag(repository, tag string) error { + return m.cancelPendingTags(repository, []string{tag}) +} + +func (m *manager) cancelPendingTags(repository string, tags []string) error { + if len(tags) == 0 { + return nil + } + tagSet := make(map[string]struct{}, len(tags)) + for _, tag := range tags { + tagSet[tag] = struct{}{} + } metas, err := listAllMetadata(m.paths) if err != nil { return err @@ -169,10 +189,19 @@ func (m *manager) cancelPendingTag(repository, tag string) error { continue } ref, err := ParseNormalizedRef(meta.Name) - if err != nil || !cancelTagClaim(meta, ref, repository, tag) { + if err != nil { continue } - if err := writeMetadata(m.paths, ref.Repository(), strings.TrimPrefix(meta.Digest, "sha256:"), meta); err != nil { + changed := false + for tag := range tagSet { + if cancelTagClaim(meta, ref, repository, tag) { + changed = true + } + } + if !changed { + continue + } + if err := writeMetadata(m.paths, ref.Repository(), meta.digestHex(), meta); err != nil { return err } } @@ -203,7 +232,7 @@ func cancelTagClaim(meta *imageMetadata, ref *NormalizedRef, repository, tag str func (m *manager) requestedTagImage(ref *NormalizedRef) *Image { m.createMu.Lock() - digestHex, ok := m.requestedTags[tagGenerationKey(ref.Repository(), ref.Tag())] + digestHex, ok := m.requestedTags[requestedTagKeyFor(ref.Repository(), ref.Tag())] m.createMu.Unlock() if !ok { return nil @@ -235,11 +264,7 @@ func (m *manager) claimMatchingTags(digestHex string) bool { if requestedDigest != digestHex { continue } - separator := strings.LastIndexByte(key, ':') - if separator <= 0 || separator == len(key)-1 { - continue - } - repository, tag := key[:separator], key[separator+1:] + repository, tag := key.repository, key.tag if err := createTagSymlink(m.paths, repository, tag, digestHex); err != nil { slog.Warn("failed to claim image tag", "repository", repository, "tag", tag, "digest", digestHex, "error", err) continue @@ -261,7 +286,7 @@ func (m *manager) claimPrimaryTag(ref *ResolvedRef, meta *imageMetadata, digestH if primaryTag == "" { return false } - primaryKey := tagGenerationKey(ref.Repository(), primaryTag) + primaryKey := requestedTagKeyFor(ref.Repository(), primaryTag) if _, hasRequest := m.requestedTags[primaryKey]; hasRequest || !m.tagClaimIsCurrent(ref.Repository(), primaryTag, digestHex, meta, meta.RequestedTag == "") { return false } diff --git a/lib/images/tag_test.go b/lib/images/tag_test.go index 4d30a5f8c..260824af6 100644 --- a/lib/images/tag_test.go +++ b/lib/images/tag_test.go @@ -54,7 +54,7 @@ func seedImage(t *testing.T, p *paths.Paths, repository, tag, digestHex string, func newTagTestCase(t *testing.T) (*paths.Paths, *manager, string) { t.Helper() p := paths.New(t.TempDir()) - return p, &manager{paths: p, tagGenerations: make(map[string]uint64)}, "docker.io/library/alpine" + return p, &manager{paths: p, tagGenerations: make(map[string]uint64), requestedTags: make(map[requestedTagKey]string)}, "docker.io/library/alpine" } func requireTagResolvesTo(t *testing.T, p *paths.Paths, repository, tag, digest string) { @@ -139,6 +139,25 @@ func TestTagImageCrossRepository(t *testing.T) { require.True(t, os.IsNotExist(err), "content must be removed once unreferenced") } +func TestReadyClaimDoesNotResurrectOlderPendingTag(t *testing.T) { + p, m, repository := newTagTestCase(t) + oldDigest := strings.Repeat("a1", 32) + newDigest := strings.Repeat("b2", 32) + seedContent(t, p, repository, "latest", newDigest) + + key := tagGenerationKey(repository, "latest") + m.tagGenerations[key] = 1 + m.requestedTags[requestedTagKeyFor(repository, "latest")] = oldDigest + meta := &imageMetadata{RequestedTag: "latest", TagGeneration: 1} + ref, err := ParseNormalizedRef(repository + ":latest") + require.NoError(t, err) + + require.NoError(t, m.claimReadyTag(repository, "latest", newDigest)) + require.NotContains(t, m.requestedTags, requestedTagKeyFor(repository, "latest")) + require.False(t, m.claimRequestedTags(NewResolvedRef(ref, "sha256:"+oldDigest), meta)) + requireTagResolvesTo(t, p, repository, "latest", newDigest) +} + func TestStaleTagClaimCollectsContent(t *testing.T) { p, m, repository := newTagTestCase(t) digest := "f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2" @@ -177,7 +196,7 @@ func TestDeletingPendingTagDoesNotReclaimIt(t *testing.T) { require.NoError(t, os.MkdirAll(p.ImageContentDir(newDigest), 0o755)) require.NoError(t, writeMetadataFile(p.ImageContentMetadata(newDigest), pending)) m.tagGenerations[tagGenerationKey(repository, "latest")] = 1 - m.requestedTags = map[string]string{tagGenerationKey(repository, "latest"): newDigest} + m.requestedTags = map[requestedTagKey]string{requestedTagKeyFor(repository, "latest"): newDigest} require.NoError(t, m.DeleteImage(context.Background(), repository+":latest")) pending, err := readContentMetadata(p, newDigest) @@ -199,8 +218,8 @@ func TestRestoreTagStateRestoresSecondaryClaims(t *testing.T) { TagClaims: []imageTagClaim{{Repository: repository, Tag: "stable", TagGeneration: 2}}, } m.restoreTagState([]*imageMetadata{meta}) - require.Equal(t, digest, m.requestedTags[tagGenerationKey(repository, "latest")]) - require.Equal(t, digest, m.requestedTags[tagGenerationKey(repository, "stable")]) + require.Equal(t, digest, m.requestedTags[requestedTagKeyFor(repository, "latest")]) + require.Equal(t, digest, m.requestedTags[requestedTagKeyFor(repository, "stable")]) require.Equal(t, uint64(2), m.tagGenerations[tagGenerationKey(repository, "stable")]) } @@ -230,7 +249,7 @@ func TestWaitForReadyUsesNewestPendingTag(t *testing.T) { p, m, repository := newTagTestCase(t) m.queue = queue.New(1) m.readySubscribers = make(map[string][]chan StatusEvent) - m.requestedTags = make(map[string]string) + m.requestedTags = make(map[requestedTagKey]string) oldDigest := strings.Repeat("a1", 32) newDigest := strings.Repeat("b2", 32) seedContent(t, p, repository, "latest", oldDigest) @@ -241,7 +260,7 @@ func TestWaitForReadyUsesNewestPendingTag(t *testing.T) { Status: StatusPending, CreatedAt: time.Now().UTC(), } require.NoError(t, writeMetadataFile(p.ImageContentMetadata(newDigest), pending)) - m.requestedTags[tagGenerationKey(repository, "latest")] = newDigest + m.requestedTags[requestedTagKeyFor(repository, "latest")] = newDigest result := make(chan error, 1) ctx, cancel := context.WithCancel(context.Background()) From 297c977e5d6b030aed02ffcd7a87965eac1ecfeb Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:24:02 +0000 Subject: [PATCH 12/13] Fix image tag claim recovery, cancellation, and promotion regressions --- lib/images/manager.go | 43 ++++++++-- lib/images/storage.go | 12 +++ lib/images/storage_refs.go | 13 ++- lib/images/storage_test.go | 44 ++++++++++ lib/images/tag_manager.go | 123 ++++++++++++++++++++++++-- lib/images/tag_test.go | 171 +++++++++++++++++++++++++++++++++++++ 6 files changed, 385 insertions(+), 21 deletions(-) diff --git a/lib/images/manager.go b/lib/images/manager.go index ca5203901..d55cc9277 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -688,6 +688,9 @@ func (m *manager) updateStatusByDigest(ref *ResolvedRef, status string, err erro if meta.RequestedTag != "" { m.releaseTagGeneration(ref.Repository(), meta.RequestedTag, meta.TagGeneration) } + for _, claim := range meta.TagClaims { + m.releaseTagGeneration(claim.Repository, claim.Tag, claim.TagGeneration) + } m.clearRequestedDigest(meta.digestHex()) } } @@ -774,20 +777,37 @@ func (m *manager) DeleteImage(ctx context.Context, name string) error { } func (m *manager) deleteDigestImage(repository, digestHex string) error { - if _, err := readMetadata(m.paths, repository, digestHex); err != nil { + meta, err := readMetadata(m.paths, repository, digestHex) + if err != nil { return err } digestTags, err := tagsForDigest(m.paths, repository, digestHex) if err != nil { return err } - if err := deleteTagsForDigest(m.paths, repository, digestTags); err != nil { + if err := deleteTags(m.paths, repository, digestTags); err != nil { return err } + allTags := make(map[string]struct{}, len(digestTags)+len(meta.TagClaims)+1) for _, tag := range digestTags { + allTags[tag] = struct{}{} + } + if meta.RequestedTag != "" { + allTags[meta.RequestedTag] = struct{}{} + } + for _, claim := range meta.TagClaims { + if claim.Repository == repository && claim.Tag != "" { + allTags[claim.Tag] = struct{}{} + } + } + tagsToClean := make([]string, 0, len(allTags)) + for tag := range allTags { + tagsToClean = append(tagsToClean, tag) + } + for _, tag := range tagsToClean { m.forgetTagState(repository, tag) } - if err := m.cancelPendingTags(repository, digestTags); err != nil { + if err := m.cancelPendingTags(repository, tagsToClean); err != nil { return fmt.Errorf("cancel pending image tags: %w", err) } if err := removeDigestIfUnreferenced(m.paths, repository, digestHex, false); err != nil { @@ -799,18 +819,25 @@ func (m *manager) deleteDigestImage(repository, digestHex string) error { } func (m *manager) deleteTaggedImage(repository, tag string) error { - digestHex, err := resolveTag(m.paths, repository, tag) - if err != nil { - return err + digestHex, resolveErr := resolveTag(m.paths, repository, tag) + if resolveErr != nil && !errors.Is(resolveErr, ErrNotFound) && !os.IsNotExist(resolveErr) { + return resolveErr } - if err := deleteTag(m.paths, repository, tag); err != nil { - return err + + if resolveErr == nil { + if err := deleteTag(m.paths, repository, tag); err != nil { + return err + } } m.forgetTagState(repository, tag) if err := m.cancelPendingTag(repository, tag); err != nil { return fmt.Errorf("cancel pending image tag: %w", err) } + if resolveErr != nil { + return resolveErr + } + count, err := countTagsForDigest(m.paths, repository, digestHex) if err != nil { return fmt.Errorf("count tags for digest %s: %w", digestHex, err) diff --git a/lib/images/storage.go b/lib/images/storage.go index 72c57e213..5b1d9e566 100644 --- a/lib/images/storage.go +++ b/lib/images/storage.go @@ -297,6 +297,15 @@ func promoteImageToContent(p *paths.Paths, sourceRepository, digestHex string, s }); err != nil { return fmt.Errorf("link source disk: %w", err) } + sourceManifestPath := filepath.Join(sourceLayout.dir, "manifest.json") + if _, err := os.Stat(sourceManifestPath); err == nil { + targetManifestPath := p.ImageContentManifestModel(digestHex) + if _, err := os.Stat(targetManifestPath); os.IsNotExist(err) { + _ = installAtomically(targetManifestPath, func(path string) error { + return os.Link(sourceManifestPath, path) + }) + } + } if err := writeMetadataFile(p.ImageContentMetadata(digestHex), sourceMeta); err != nil { return fmt.Errorf("finalize content metadata: %w", err) } @@ -342,6 +351,9 @@ func promoteLegacyTags(p *paths.Paths, repository, digestHex string) error { } return fmt.Errorf("promote legacy tag: %w", err) } + if err := removeStaleTagSymlink(p, &ref); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to remove stale legacy tag symlink: %v\n", err) + } } return nil } diff --git a/lib/images/storage_refs.go b/lib/images/storage_refs.go index 78eecef70..0cf341c3d 100644 --- a/lib/images/storage_refs.go +++ b/lib/images/storage_refs.go @@ -196,11 +196,12 @@ func (i *metadataIndex) appendMetadataForTag(p *paths.Paths, repository, tag, di if err != nil { return nil } - meta.Name = repository + ":" + tag + clone := *meta + clone.Name = repository + ":" + tag i.seen[tagKey] = struct{}{} i.taggedDigests[repository+"@"+digestHex] = struct{}{} i.taggedContentDigests[digestHex] = struct{}{} - i.metas = append(i.metas, meta) + i.metas = append(i.metas, &clone) return nil } @@ -248,7 +249,7 @@ func countTagsForDigest(p *paths.Paths, repository, digestHex string) (int, erro return len(tags), err } -func deleteTagsForDigest(p *paths.Paths, repository string, tags []string) error { +func deleteTags(p *paths.Paths, repository string, tags []string) error { for _, tag := range tags { if err := deleteTag(p, repository, tag); err != nil && !errors.Is(err, ErrNotFound) { return err @@ -257,6 +258,10 @@ func deleteTagsForDigest(p *paths.Paths, repository string, tags []string) error return nil } +func deleteTagsForDigest(p *paths.Paths, repository string, tags []string) error { + return deleteTags(p, repository, tags) +} + func contentTagCount(p *paths.Paths, digestHex string) (int, error) { root := p.ImageRepositoriesDir() count := 0 @@ -286,7 +291,7 @@ func contentTagCount(p *paths.Paths, digestHex string) (int, error) { func contentPullInProgress(p *paths.Paths, digestHex string) bool { status, ok := metadataStatus(p.ImageContentMetadata(digestHex)) - return ok && (status == StatusPending || status == StatusPulling || status == StatusConverting) + return ok && isPendingImageStatus(status) } func contentIsDigestOnly(p *paths.Paths, digestHex string) bool { diff --git a/lib/images/storage_test.go b/lib/images/storage_test.go index 003bdf26c..b6479d615 100644 --- a/lib/images/storage_test.go +++ b/lib/images/storage_test.go @@ -271,3 +271,47 @@ func TestPromoteImageToContentReplacesFailedContent(t *testing.T) { require.Equal(t, StatusReady, contentMeta.Status) require.FileExists(t, p.ImageContentPath(digest)) } + +func TestPromoteLegacyTagsRemovesStaleLegacySymlinks(t *testing.T) { + p := paths.New(t.TempDir()) + repository := "docker.io/library/alpine" + digest := "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + seedLegacy(t, p, repository, "latest", digest) + + legacySymlink := p.ImageTagSymlink(repository, "latest") + require.FileExists(t, legacySymlink) + + legacyMeta, err := readMetadata(p, repository, digest) + require.NoError(t, err) + + require.NoError(t, promoteImageToContent(p, repository, digest, legacyMeta)) + + // New repository tag symlink should exist + require.FileExists(t, p.ImageRepositoryTagSymlink(repository, "latest")) + // Old legacy symlink must be removed + _, err = os.Lstat(legacySymlink) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestPromoteImageToContentLinksManifestModel(t *testing.T) { + p := paths.New(t.TempDir()) + repository := "docker.io/library/alpine" + digest := "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + seedLegacy(t, p, repository, "latest", digest) + + // Create a legacy manifest.json + legacyManifestPath := filepath.Join(p.ImageDigestDir(repository, digest), "manifest.json") + require.NoError(t, os.WriteFile(legacyManifestPath, []byte(`{"schemaVersion":2}`), 0o644)) + + legacyMeta, err := readMetadata(p, repository, digest) + require.NoError(t, err) + + require.NoError(t, promoteImageToContent(p, repository, digest, legacyMeta)) + + // The promoted content directory must have manifest.json + contentManifest := p.ImageContentManifestModel(digest) + require.FileExists(t, contentManifest) + data, err := os.ReadFile(contentManifest) + require.NoError(t, err) + require.Equal(t, `{"schemaVersion":2}`, string(data)) +} diff --git a/lib/images/tag_manager.go b/lib/images/tag_manager.go index eafce2d3b..e6b1e2248 100644 --- a/lib/images/tag_manager.go +++ b/lib/images/tag_manager.go @@ -6,6 +6,7 @@ import ( "fmt" "log/slog" "strings" + "time" "github.com/kernel/hypeman/lib/paths" ) @@ -60,6 +61,8 @@ func (m *manager) forgetTagState(repository, tag string) { } func (m *manager) restoreTagState(metas []*imageMetadata) { + m.createMu.Lock() + defer m.createMu.Unlock() m.ensureTagState() for _, meta := range metas { if !isPendingImageStatus(meta.Status) { @@ -87,10 +90,10 @@ func (m *manager) restoreRequestedTag(repository, tag, digestHex string, generat return } key := tagGenerationKey(repository, tag) - if generation > m.tagGenerations[key] { + if currentGen, ok := m.tagGenerations[key]; !ok || generation >= currentGen { m.tagGenerations[key] = generation + m.requestedTags[requestedTagKeyFor(repository, tag)] = digestHex } - m.requestedTags[requestedTagKeyFor(repository, tag)] = digestHex } func (m *manager) claimTagForStatus(meta *imageMetadata, ref *ResolvedRef) error { @@ -129,7 +132,17 @@ func (m *manager) claimTagForStatus(meta *imageMetadata, ref *ResolvedRef) error PreviousTagDigest: previous, TagGeneration: generation, }) + if previous == "" { + if err := createTagSymlink(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()); err != nil { + m.revertTagGeneration(ref.Repository(), ref.Tag()) + meta.TagClaims = meta.TagClaims[:len(meta.TagClaims)-1] + return err + } + } if err := writeMetadata(m.paths, ref.Repository(), ref.DigestHex(), meta); err != nil { + if previous == "" { + _ = deleteTag(m.paths, ref.Repository(), ref.Tag()) + } m.revertTagGeneration(ref.Repository(), ref.Tag()) meta.TagClaims = meta.TagClaims[:len(meta.TagClaims)-1] return err @@ -164,10 +177,50 @@ func (m *manager) clearRequestedDigest(digestHex string) { for key, current := range m.requestedTags { if current == digestHex { delete(m.requestedTags, key) + if nextDigest := m.findPendingDigestForTag(key.repository, key.tag, digestHex); nextDigest != "" { + m.requestedTags[key] = nextDigest + } } } } +func (m *manager) findPendingDigestForTag(repository, tag, excludeDigestHex string) string { + metas, err := listAllMetadata(m.paths) + if err != nil { + return "" + } + var ( + bestDigest string + bestTime time.Time + bestGen uint64 + ) + for _, meta := range metas { + if !isPendingImageStatus(meta.Status) || meta.digestHex() == excludeDigestHex { + continue + } + if !meta.TagClaimCanceled && meta.RequestedTag == tag { + ref, err := ParseNormalizedRef(meta.Name) + if err == nil && ref.Repository() == repository { + if meta.TagGeneration >= bestGen || (meta.TagGeneration == bestGen && meta.CreatedAt.After(bestTime)) { + bestDigest = meta.digestHex() + bestTime = meta.CreatedAt + bestGen = meta.TagGeneration + } + } + } + for _, claim := range meta.TagClaims { + if claim.Repository == repository && claim.Tag == tag { + if claim.TagGeneration >= bestGen || (claim.TagGeneration == bestGen && meta.CreatedAt.After(bestTime)) { + bestDigest = meta.digestHex() + bestTime = meta.CreatedAt + bestGen = claim.TagGeneration + } + } + } + } + return bestDigest +} + func (m *manager) cancelPendingTag(repository, tag string) error { return m.cancelPendingTags(repository, []string{tag}) } @@ -184,24 +237,42 @@ func (m *manager) cancelPendingTags(repository string, tags []string) error { if err != nil { return err } + seenDigests := make(map[string]struct{}, len(metas)) for _, meta := range metas { if !isPendingImageStatus(meta.Status) { continue } - ref, err := ParseNormalizedRef(meta.Name) + digestHex := meta.digestHex() + if _, seen := seenDigests[digestHex]; seen { + continue + } + seenDigests[digestHex] = struct{}{} + + canonicalMeta, err := readMetadata(m.paths, repository, digestHex) if err != nil { + canonicalMeta = meta + } + + ref, err := ParseNormalizedRef(canonicalMeta.Name) + if canonicalMeta.Request != nil && canonicalMeta.Request.Name != "" { + if reqRef, err := ParseNormalizedRef(canonicalMeta.Request.Name); err == nil { + ref = reqRef + } + } + if ref == nil { continue } + changed := false for tag := range tagSet { - if cancelTagClaim(meta, ref, repository, tag) { + if cancelTagClaim(canonicalMeta, ref, repository, tag) { changed = true } } if !changed { continue } - if err := writeMetadata(m.paths, ref.Repository(), meta.digestHex(), meta); err != nil { + if err := writeMetadata(m.paths, ref.Repository(), digestHex, canonicalMeta); err != nil { return err } } @@ -215,8 +286,10 @@ func isPendingImageStatus(status string) bool { func cancelTagClaim(meta *imageMetadata, ref *NormalizedRef, repository, tag string) bool { changed := false if ref.Repository() == repository && meta.RequestedTag == tag { - meta.TagClaimCanceled = true - changed = true + if !meta.TagClaimCanceled { + meta.TagClaimCanceled = true + changed = true + } } claims := meta.TagClaims[:0] for _, claim := range meta.TagClaims { @@ -246,9 +319,41 @@ func (m *manager) requestedTagImage(ref *NormalizedRef) *Image { func (m *manager) claimRequestedTags(ref *ResolvedRef, meta *imageMetadata) bool { digestHex := ref.DigestHex() - claimed := m.claimMatchingTags(digestHex) - if m.claimPrimaryTag(ref, meta, digestHex) { + primaryTag := meta.RequestedTag + if primaryTag == "" { + primaryTag = ref.Tag() + } + primaryKey := requestedTagKeyFor(ref.Repository(), primaryTag) + + claimed := false + primaryClaimed := false + for key, requestedDigest := range m.requestedTags { + if requestedDigest != digestHex { + continue + } + repository, tag := key.repository, key.tag + if err := createTagSymlink(m.paths, repository, tag, digestHex); err != nil { + slog.Warn("failed to claim image tag", "repository", repository, "tag", tag, "digest", digestHex, "error", err) + continue + } + delete(m.requestedTags, key) claimed = true + if key == primaryKey { + primaryClaimed = true + } + } + if !primaryClaimed && m.claimPrimaryTag(ref, meta, digestHex) { + claimed = true + } + for _, claim := range meta.TagClaims { + claimKey := requestedTagKeyFor(claim.Repository, claim.Tag) + if m.tagClaimIsCurrent(claim.Repository, claim.Tag, digestHex, meta, false) { + if _, hasReq := m.requestedTags[claimKey]; !hasReq { + if err := createTagSymlink(m.paths, claim.Repository, claim.Tag, digestHex); err == nil { + claimed = true + } + } + } } if claimed { return true diff --git a/lib/images/tag_test.go b/lib/images/tag_test.go index 260824af6..fe411b8c9 100644 --- a/lib/images/tag_test.go +++ b/lib/images/tag_test.go @@ -398,3 +398,174 @@ func TestTagImageSelfHealsDanglingTargetSymlink(t *testing.T) { tagImage(t, m, repository+":latest", repository+":stable", digest) requireTagResolvesTo(t, p, repository, "stable", digest) } + +func TestOlderPendingBuildRestoredWhenNewerBuildFails(t *testing.T) { + p, m, repository := newTagTestCase(t) + digestA := strings.Repeat("11", 32) + digestB := strings.Repeat("22", 32) + digestC := strings.Repeat("33", 32) + seedContent(t, p, repository, "latest", digestA) + + // Build B is pending for latest (generation 1) + require.NoError(t, os.MkdirAll(p.ImageContentDir(digestB), 0o755)) + metaB := &imageMetadata{ + Name: repository + ":latest", Digest: "sha256:" + digestB, + Status: StatusPending, CreatedAt: time.Now().UTC(), + RequestedTag: "latest", TagGeneration: 1, BuildID: "b", + } + require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digestB), metaB)) + m.tagGenerations[tagGenerationKey(repository, "latest")] = 1 + m.requestedTags[requestedTagKeyFor(repository, "latest")] = digestB + + // Build C starts for latest (generation 2) + require.NoError(t, os.MkdirAll(p.ImageContentDir(digestC), 0o755)) + metaC := &imageMetadata{ + Name: repository + ":latest", Digest: "sha256:" + digestC, + Status: StatusPending, CreatedAt: time.Now().UTC().Add(time.Second), + RequestedTag: "latest", TagGeneration: 2, BuildID: "c", + } + require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digestC), metaC)) + m.tagGenerations[tagGenerationKey(repository, "latest")] = 2 + m.requestedTags[requestedTagKeyFor(repository, "latest")] = digestC + + // C fails + m.releaseTagGeneration(repository, metaC.RequestedTag, metaC.TagGeneration) + m.clearRequestedDigest(digestC) + + // requestedTags must restore to build B + require.Equal(t, digestB, m.requestedTags[requestedTagKeyFor(repository, "latest")]) + + ref, err := ParseNormalizedRef(repository + ":latest") + require.NoError(t, err) + img := m.requestedTagImage(ref) + require.NotNil(t, img) + require.Equal(t, StatusPending, img.Status) + require.Equal(t, "sha256:"+digestB, img.Digest) + + // When B completes, it claims latest + resolvedB := NewResolvedRef(ref, "sha256:"+digestB) + claimed := m.claimRequestedTags(resolvedB, metaB) + require.True(t, claimed) + requireTagResolvesTo(t, p, repository, "latest", digestB) +} + +func TestSecondaryClaimReclaimedAfterNewerBuildFails(t *testing.T) { + p, m, repository := newTagTestCase(t) + digestA := strings.Repeat("a1", 32) + digestB := strings.Repeat("b2", 32) + digestC := strings.Repeat("c3", 32) + seedContent(t, p, repository, "stable", digestA) + + // Build B is pending for other, and has secondary claim on stable (generation 1) + require.NoError(t, os.MkdirAll(p.ImageContentDir(digestB), 0o755)) + metaB := &imageMetadata{ + Name: repository + ":other", Digest: "sha256:" + digestB, + Status: StatusPending, CreatedAt: time.Now().UTC(), + RequestedTag: "other", TagGeneration: 1, BuildID: "b", + TagClaims: []imageTagClaim{ + {Repository: repository, Tag: "stable", PreviousTagDigest: digestA, TagGeneration: 1}, + }, + } + require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digestB), metaB)) + m.tagGenerations[tagGenerationKey(repository, "stable")] = 1 + m.requestedTags[requestedTagKeyFor(repository, "stable")] = digestB + + // Newer build C starts for stable (generation 2) + require.NoError(t, os.MkdirAll(p.ImageContentDir(digestC), 0o755)) + metaC := &imageMetadata{ + Name: repository + ":stable", Digest: "sha256:" + digestC, + Status: StatusPending, CreatedAt: time.Now().UTC().Add(time.Second), + RequestedTag: "stable", TagGeneration: 2, BuildID: "c", + } + require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digestC), metaC)) + m.tagGenerations[tagGenerationKey(repository, "stable")] = 2 + m.requestedTags[requestedTagKeyFor(repository, "stable")] = digestC + + // C fails + m.releaseTagGeneration(repository, metaC.RequestedTag, metaC.TagGeneration) + m.clearRequestedDigest(digestC) + + // Generation for stable rolled back to 1 and requested tag restored to B + require.Equal(t, uint64(1), m.tagGenerations[tagGenerationKey(repository, "stable")]) + require.Equal(t, digestB, m.requestedTags[requestedTagKeyFor(repository, "stable")]) + + // B completes + refB, err := ParseNormalizedRef(repository + ":other") + require.NoError(t, err) + resolvedB := NewResolvedRef(refB, "sha256:"+digestB) + claimed := m.claimRequestedTags(resolvedB, metaB) + require.True(t, claimed) + requireTagResolvesTo(t, p, repository, "stable", digestB) +} + +func TestClaimTagForStatusCreatesPendingSymlink(t *testing.T) { + p, m, repository := newTagTestCase(t) + digest := strings.Repeat("d4", 32) + require.NoError(t, os.MkdirAll(p.ImageContentDir(digest), 0o755)) + + meta := &imageMetadata{ + Name: repository + "@sha256:" + digest, Digest: "sha256:" + digest, + Status: StatusPending, CreatedAt: time.Now().UTC(), + } + require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digest), meta)) + + ref, err := ParseNormalizedRef(repository + ":newtag") + require.NoError(t, err) + resolved := NewResolvedRef(ref, "sha256:"+digest) + + require.NoError(t, m.claimTagForStatus(meta, resolved)) + + // The symlink for newtag should now exist and resolve to digest + requireTagResolvesTo(t, p, repository, "newtag", digest) +} + +func TestDeleteTaggedImageCleansPendingTagWithoutSymlink(t *testing.T) { + _, m, repository := newTagTestCase(t) + digest := strings.Repeat("e5", 32) + + m.tagGenerations[tagGenerationKey(repository, "orphan")] = 3 + m.requestedTags[requestedTagKeyFor(repository, "orphan")] = digest + + err := m.deleteTaggedImage(repository, "orphan") + require.Error(t, err) // Symlink was missing on disk + + // But memory tag state should be cleared + _, hasGen := m.tagGenerations[tagGenerationKey(repository, "orphan")] + require.False(t, hasGen) + _, hasReq := m.requestedTags[requestedTagKeyFor(repository, "orphan")] + require.False(t, hasReq) +} + +func TestDeleteDigestImageCleansPendingTagClaims(t *testing.T) { + p, m, repository := newTagTestCase(t) + digestA := strings.Repeat("a1", 32) + digestB := strings.Repeat("b2", 32) + seedContent(t, p, repository, "latest", digestA) + + // Build B is pending for latest, but latest symlink still points to A + require.NoError(t, os.MkdirAll(p.ImageContentDir(digestB), 0o755)) + metaB := &imageMetadata{ + Name: repository + ":latest", Digest: "sha256:" + digestB, + Status: StatusPending, CreatedAt: time.Now().UTC(), + RequestedTag: "latest", TagGeneration: 2, BuildID: "b", + } + require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digestB), metaB)) + m.tagGenerations[tagGenerationKey(repository, "latest")] = 2 + m.requestedTags[requestedTagKeyFor(repository, "latest")] = digestB + + require.NoError(t, m.deleteDigestImage(repository, digestB)) + + // In-memory state for latest should be cleared + _, hasGen := m.tagGenerations[tagGenerationKey(repository, "latest")] + require.False(t, hasGen) + _, hasReq := m.requestedTags[requestedTagKeyFor(repository, "latest")] + require.False(t, hasReq) + + // Pending metadata should have tag claim marked as canceled + meta, err := readContentMetadata(p, digestB) + require.NoError(t, err) + require.True(t, meta.TagClaimCanceled) + + // Image A and its tag should still exist + requireTagResolvesTo(t, p, repository, "latest", digestA) +} From 82e85a879b84fc618061a9cd2c6d40f78a866430 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:32:24 +0000 Subject: [PATCH 13/13] Consolidate image tagging and reduce line count --- lib/images/manager.go | 17 +-- lib/images/storage_refs.go | 4 - lib/images/tag.go | 122 ++++++++++++++++++++++ lib/images/tag_manager.go | 209 +++++-------------------------------- lib/images/tag_test.go | 60 +++++------ 5 files changed, 183 insertions(+), 229 deletions(-) create mode 100644 lib/images/tag.go diff --git a/lib/images/manager.go b/lib/images/manager.go index d55cc9277..8b905a417 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -77,7 +77,7 @@ type manager struct { createMu sync.Mutex diskUsageMu sync.RWMutex tagGenerations map[string]uint64 - requestedTags map[requestedTagKey]string // newest pull's digest per requested tag + requestedTags map[string]string // newest pull's digest per requested tag diskUsageLoaded bool readyImageBytes int64 ociCacheBytes int64 @@ -106,7 +106,7 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma borrowedCredentialsTimeout: DefaultBorrowedCredentialsTimeout, readySubscribers: make(map[string][]chan StatusEvent), tagGenerations: make(map[string]uint64), - requestedTags: make(map[requestedTagKey]string), + requestedTags: make(map[string]string), } // Initialize metrics if meter is provided @@ -788,22 +788,15 @@ func (m *manager) deleteDigestImage(repository, digestHex string) error { if err := deleteTags(m.paths, repository, digestTags); err != nil { return err } - allTags := make(map[string]struct{}, len(digestTags)+len(meta.TagClaims)+1) - for _, tag := range digestTags { - allTags[tag] = struct{}{} - } + tagsToClean := append([]string(nil), digestTags...) if meta.RequestedTag != "" { - allTags[meta.RequestedTag] = struct{}{} + tagsToClean = append(tagsToClean, meta.RequestedTag) } for _, claim := range meta.TagClaims { if claim.Repository == repository && claim.Tag != "" { - allTags[claim.Tag] = struct{}{} + tagsToClean = append(tagsToClean, claim.Tag) } } - tagsToClean := make([]string, 0, len(allTags)) - for tag := range allTags { - tagsToClean = append(tagsToClean, tag) - } for _, tag := range tagsToClean { m.forgetTagState(repository, tag) } diff --git a/lib/images/storage_refs.go b/lib/images/storage_refs.go index 0cf341c3d..be23c7290 100644 --- a/lib/images/storage_refs.go +++ b/lib/images/storage_refs.go @@ -258,10 +258,6 @@ func deleteTags(p *paths.Paths, repository string, tags []string) error { return nil } -func deleteTagsForDigest(p *paths.Paths, repository string, tags []string) error { - return deleteTags(p, repository, tags) -} - func contentTagCount(p *paths.Paths, digestHex string) (int, error) { root := p.ImageRepositoriesDir() count := 0 diff --git a/lib/images/tag.go b/lib/images/tag.go new file mode 100644 index 000000000..302263d7e --- /dev/null +++ b/lib/images/tag.go @@ -0,0 +1,122 @@ +package images + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/kernel/hypeman/lib/paths" +) + +// TagImage creates a ready-image tag without pulling or converting content. +// Cross-repository tags promote content into the shared layout. +func (m *manager) TagImage(ctx context.Context, source, target string) (*Image, error) { + sourceRef, targetRef, err := parseTagReferences(source, target) + if err != nil { + return nil, err + } + + m.createMu.Lock() + defer m.createMu.Unlock() + + previousDigest, err := existingTagDigest(m.paths, targetRef) + if err != nil { + return nil, err + } + digestHex, meta, err := m.readyTagImage(sourceRef) + if err != nil { + return nil, err + } + if err := m.cancelPendingTag(targetRef.Repository(), targetRef.Tag()); err != nil { + return nil, fmt.Errorf("cancel pending image tag: %w", err) + } + if err := m.installTag(sourceRef, targetRef, digestHex, meta); err != nil { + return nil, err + } + m.nextTagGeneration(targetRef.Repository(), targetRef.Tag()) + m.clearRequestedTag(targetRef.Repository(), targetRef.Tag(), "") + m.cleanupReplacedTag(targetRef, previousDigest, digestHex) + + return meta.toImageFor(targetRef.String()), nil +} + +func existingTagDigest(p *paths.Paths, ref *NormalizedRef) (string, error) { + digest, err := resolveTag(p, ref.Repository(), ref.Tag()) + if err == nil || errors.Is(err, ErrNotFound) || errors.Is(err, errInvalidSymlinkTarget) { + return digest, nil + } + return "", fmt.Errorf("resolve existing target tag: %w", err) +} + +func (m *manager) installTag(source, target *NormalizedRef, digest string, meta *imageMetadata) error { + if source.Repository() != target.Repository() { + if err := promoteImageToContent(m.paths, source.Repository(), digest, meta); err != nil { + return fmt.Errorf("promote image to content: %w", err) + } + } + installed, err := installTagSymlink(m.paths, target.Repository(), target.Tag(), digest) + if err != nil { + return fmt.Errorf("create image tag: %w", err) + } + if err := writeMetadata(m.paths, target.Repository(), digest, meta); err != nil { + if restoreErr := restoreTagSymlink(&installed); restoreErr != nil { + return errors.Join( + fmt.Errorf("write tagged image metadata: %w", err), + fmt.Errorf("restore image tag: %w", restoreErr), + ) + } + return fmt.Errorf("write tagged image metadata: %w", err) + } + return nil +} + +func (m *manager) cleanupUnclaimedImage(ref *ResolvedRef) { + if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), ref.DigestHex(), true); err != nil { + slog.Warn("failed to collect stale image", "repository", ref.Repository(), "digest", ref.DigestHex(), "error", err) + } +} + +func (m *manager) cleanupReplacedTag(ref *NormalizedRef, previousDigest, digestHex string) { + if previousDigest == "" || previousDigest == digestHex { + return + } + count, err := countTagsForDigest(m.paths, ref.Repository(), previousDigest) + if err != nil { + slog.Warn("failed to count tags for replaced image", "repository", ref.Repository(), "digest", previousDigest, "error", err) + return + } + if count > 0 { + return + } + if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), previousDigest, true); err != nil { + slog.Warn("failed to collect replaced image content", "repository", ref.Repository(), "digest", previousDigest, "error", err) + } + m.refreshDiskUsageTotals() +} + +func parseTagReferences(source, target string) (*NormalizedRef, *NormalizedRef, error) { + sourceRef, err := ParseNormalizedRef(source) + if err != nil { + return nil, nil, fmt.Errorf("%w: invalid source reference: %s", ErrInvalidName, err) + } + targetRef, err := ParseNormalizedRef(target) + if err != nil { + return nil, nil, fmt.Errorf("%w: invalid target reference: %s", ErrInvalidName, err) + } + if targetRef.IsDigest() { + return nil, nil, fmt.Errorf("%w: target must be a tag reference, not a digest", ErrInvalidName) + } + return sourceRef, targetRef, nil +} + +func (m *manager) readyTagImage(ref *NormalizedRef) (string, *imageMetadata, error) { + digestHex, meta, err := resolveRefMetadata(m.paths, ref) + if err != nil { + return "", nil, err + } + if meta.Status != StatusReady { + return "", nil, fmt.Errorf("%w: %s", ErrImageNotReady, meta.Status) + } + return digestHex, meta, nil +} diff --git a/lib/images/tag_manager.go b/lib/images/tag_manager.go index e6b1e2248..ea36e6fd4 100644 --- a/lib/images/tag_manager.go +++ b/lib/images/tag_manager.go @@ -1,47 +1,34 @@ package images import ( - "context" "errors" - "fmt" "log/slog" "strings" "time" - - "github.com/kernel/hypeman/lib/paths" ) -type requestedTagKey struct { - repository string - tag string -} - -func tagGenerationKey(repository, tag string) string { +func tagKey(repository, tag string) string { return repository + ":" + tag } -func requestedTagKeyFor(repository, tag string) requestedTagKey { - return requestedTagKey{repository: repository, tag: tag} -} - func (m *manager) ensureTagState() { if m.tagGenerations == nil { m.tagGenerations = make(map[string]uint64) } if m.requestedTags == nil { - m.requestedTags = make(map[requestedTagKey]string) + m.requestedTags = make(map[string]string) } } func (m *manager) nextTagGeneration(repository, tag string) uint64 { m.ensureTagState() - key := tagGenerationKey(repository, tag) + key := tagKey(repository, tag) m.tagGenerations[key]++ return m.tagGenerations[key] } func (m *manager) revertTagGeneration(repository, tag string) { - key := tagGenerationKey(repository, tag) + key := tagKey(repository, tag) if m.tagGenerations[key] <= 1 { delete(m.tagGenerations, key) return @@ -50,13 +37,13 @@ func (m *manager) revertTagGeneration(repository, tag string) { } func (m *manager) releaseTagGeneration(repository, tag string, generation uint64) { - if m.tagGenerations[tagGenerationKey(repository, tag)] == generation { + if m.tagGenerations[tagKey(repository, tag)] == generation { m.revertTagGeneration(repository, tag) } } func (m *manager) forgetTagState(repository, tag string) { - delete(m.tagGenerations, tagGenerationKey(repository, tag)) + delete(m.tagGenerations, tagKey(repository, tag)) m.clearRequestedTag(repository, tag, "") } @@ -65,10 +52,7 @@ func (m *manager) restoreTagState(metas []*imageMetadata) { defer m.createMu.Unlock() m.ensureTagState() for _, meta := range metas { - if !isPendingImageStatus(meta.Status) { - continue - } - if meta.Digest == "" { + if !isPendingImageStatus(meta.Status) || meta.Digest == "" { continue } digestHex := meta.digestHex() @@ -89,10 +73,10 @@ func (m *manager) restoreRequestedTag(repository, tag, digestHex string, generat if tag == "" { return } - key := tagGenerationKey(repository, tag) + key := tagKey(repository, tag) if currentGen, ok := m.tagGenerations[key]; !ok || generation >= currentGen { m.tagGenerations[key] = generation - m.requestedTags[requestedTagKeyFor(repository, tag)] = digestHex + m.requestedTags[key] = digestHex } } @@ -103,9 +87,6 @@ func (m *manager) claimTagForStatus(meta *imageMetadata, ref *ResolvedRef) error if meta.Status == StatusReady { return m.claimReadyTag(ref.Repository(), ref.Tag(), ref.DigestHex()) } - // Keep the existing tag visible until the build succeeds, but remember this - // digest as the newest request for the tag. Finalization will claim it along - // with every other tag waiting for the same digest. if meta.RequestedTag == ref.Tag() && strings.HasPrefix(meta.Name, ref.Repository()+":") { wasCanceled := meta.TagClaimCanceled meta.TagClaimCanceled = false @@ -163,11 +144,11 @@ func (m *manager) claimReadyTag(repository, tag, digestHex string) error { func (m *manager) trackRequestedTag(repository, tag, digestHex string) { m.ensureTagState() - m.requestedTags[requestedTagKeyFor(repository, tag)] = digestHex + m.requestedTags[tagKey(repository, tag)] = digestHex } func (m *manager) clearRequestedTag(repository, tag, digestHex string) { - key := requestedTagKeyFor(repository, tag) + key := tagKey(repository, tag) if current, ok := m.requestedTags[key]; ok && (digestHex == "" || current == digestHex) { delete(m.requestedTags, key) } @@ -177,8 +158,9 @@ func (m *manager) clearRequestedDigest(digestHex string) { for key, current := range m.requestedTags { if current == digestHex { delete(m.requestedTags, key) - if nextDigest := m.findPendingDigestForTag(key.repository, key.tag, digestHex); nextDigest != "" { - m.requestedTags[key] = nextDigest + repo, tag, _ := strings.Cut(key, ":") + if next := m.findPendingDigestForTag(repo, tag, digestHex); next != "" { + m.requestedTags[key] = next } } } @@ -202,18 +184,14 @@ func (m *manager) findPendingDigestForTag(repository, tag, excludeDigestHex stri ref, err := ParseNormalizedRef(meta.Name) if err == nil && ref.Repository() == repository { if meta.TagGeneration >= bestGen || (meta.TagGeneration == bestGen && meta.CreatedAt.After(bestTime)) { - bestDigest = meta.digestHex() - bestTime = meta.CreatedAt - bestGen = meta.TagGeneration + bestDigest, bestTime, bestGen = meta.digestHex(), meta.CreatedAt, meta.TagGeneration } } } for _, claim := range meta.TagClaims { if claim.Repository == repository && claim.Tag == tag { if claim.TagGeneration >= bestGen || (claim.TagGeneration == bestGen && meta.CreatedAt.After(bestTime)) { - bestDigest = meta.digestHex() - bestTime = meta.CreatedAt - bestGen = claim.TagGeneration + bestDigest, bestTime, bestGen = meta.digestHex(), meta.CreatedAt, claim.TagGeneration } } } @@ -305,7 +283,7 @@ func cancelTagClaim(meta *imageMetadata, ref *NormalizedRef, repository, tag str func (m *manager) requestedTagImage(ref *NormalizedRef) *Image { m.createMu.Lock() - digestHex, ok := m.requestedTags[requestedTagKeyFor(ref.Repository(), ref.Tag())] + digestHex, ok := m.requestedTags[tagKey(ref.Repository(), ref.Tag())] m.createMu.Unlock() if !ok { return nil @@ -323,17 +301,16 @@ func (m *manager) claimRequestedTags(ref *ResolvedRef, meta *imageMetadata) bool if primaryTag == "" { primaryTag = ref.Tag() } - primaryKey := requestedTagKeyFor(ref.Repository(), primaryTag) + primaryKey := tagKey(ref.Repository(), primaryTag) - claimed := false - primaryClaimed := false + claimed, primaryClaimed := false, false for key, requestedDigest := range m.requestedTags { if requestedDigest != digestHex { continue } - repository, tag := key.repository, key.tag - if err := createTagSymlink(m.paths, repository, tag, digestHex); err != nil { - slog.Warn("failed to claim image tag", "repository", repository, "tag", tag, "digest", digestHex, "error", err) + repo, tag, _ := strings.Cut(key, ":") + if err := createTagSymlink(m.paths, repo, tag, digestHex); err != nil { + slog.Warn("failed to claim image tag", "repository", repo, "tag", tag, "digest", digestHex, "error", err) continue } delete(m.requestedTags, key) @@ -346,7 +323,7 @@ func (m *manager) claimRequestedTags(ref *ResolvedRef, meta *imageMetadata) bool claimed = true } for _, claim := range meta.TagClaims { - claimKey := requestedTagKeyFor(claim.Repository, claim.Tag) + claimKey := tagKey(claim.Repository, claim.Tag) if m.tagClaimIsCurrent(claim.Repository, claim.Tag, digestHex, meta, false) { if _, hasReq := m.requestedTags[claimKey]; !hasReq { if err := createTagSymlink(m.paths, claim.Repository, claim.Tag, digestHex); err == nil { @@ -355,29 +332,7 @@ func (m *manager) claimRequestedTags(ref *ResolvedRef, meta *imageMetadata) bool } } } - if claimed { - return true - } - // Digest-only images have no tag claim but must remain addressable by their - // digest after finalization. - return meta.RequestedTag == "" && ref.Tag() == "" -} - -func (m *manager) claimMatchingTags(digestHex string) bool { - claimed := false - for key, requestedDigest := range m.requestedTags { - if requestedDigest != digestHex { - continue - } - repository, tag := key.repository, key.tag - if err := createTagSymlink(m.paths, repository, tag, digestHex); err != nil { - slog.Warn("failed to claim image tag", "repository", repository, "tag", tag, "digest", digestHex, "error", err) - continue - } - delete(m.requestedTags, key) - claimed = true - } - return claimed + return claimed || (meta.RequestedTag == "" && ref.Tag() == "") } func (m *manager) claimPrimaryTag(ref *ResolvedRef, meta *imageMetadata, digestHex string) bool { @@ -391,7 +346,7 @@ func (m *manager) claimPrimaryTag(ref *ResolvedRef, meta *imageMetadata, digestH if primaryTag == "" { return false } - primaryKey := requestedTagKeyFor(ref.Repository(), primaryTag) + primaryKey := tagKey(ref.Repository(), primaryTag) if _, hasRequest := m.requestedTags[primaryKey]; hasRequest || !m.tagClaimIsCurrent(ref.Repository(), primaryTag, digestHex, meta, meta.RequestedTag == "") { return false } @@ -403,7 +358,7 @@ func (m *manager) claimPrimaryTag(ref *ResolvedRef, meta *imageMetadata, digestH } func (m *manager) tagClaimIsCurrent(repository, tag, digest string, meta *imageMetadata, allowMissing bool) bool { - if tag == "" || m.tagGenerations[tagGenerationKey(repository, tag)] != meta.TagGeneration { + if tag == "" || m.tagGenerations[tagKey(repository, tag)] != meta.TagGeneration { return false } current, err := resolveTag(m.paths, repository, tag) @@ -412,115 +367,3 @@ func (m *manager) tagClaimIsCurrent(repository, tag, digest string, meta *imageM } return allowMissing && errors.Is(err, ErrNotFound) } - -// TagImage creates a ready-image tag without pulling or converting content. -// Cross-repository tags promote content into the shared layout. -func (m *manager) TagImage(ctx context.Context, source, target string) (*Image, error) { - sourceRef, targetRef, err := parseTagReferences(source, target) - if err != nil { - return nil, err - } - - m.createMu.Lock() - defer m.createMu.Unlock() - - previousDigest, err := existingTagDigest(m.paths, targetRef) - if err != nil { - return nil, err - } - digestHex, meta, err := m.readyTagImage(sourceRef) - if err != nil { - return nil, err - } - if err := m.cancelPendingTag(targetRef.Repository(), targetRef.Tag()); err != nil { - return nil, fmt.Errorf("cancel pending image tag: %w", err) - } - if err := m.installTag(sourceRef, targetRef, digestHex, meta); err != nil { - return nil, err - } - m.nextTagGeneration(targetRef.Repository(), targetRef.Tag()) - m.clearRequestedTag(targetRef.Repository(), targetRef.Tag(), "") - m.cleanupReplacedTag(targetRef, previousDigest, digestHex) - - return meta.toImageFor(targetRef.String()), nil -} - -func existingTagDigest(p *paths.Paths, ref *NormalizedRef) (string, error) { - digest, err := resolveTag(p, ref.Repository(), ref.Tag()) - if err == nil || errors.Is(err, ErrNotFound) || errors.Is(err, errInvalidSymlinkTarget) { - return digest, nil - } - return "", fmt.Errorf("resolve existing target tag: %w", err) -} - -func (m *manager) installTag(source, target *NormalizedRef, digest string, meta *imageMetadata) error { - if source.Repository() != target.Repository() { - if err := promoteImageToContent(m.paths, source.Repository(), digest, meta); err != nil { - return fmt.Errorf("promote image to content: %w", err) - } - } - installed, err := installTagSymlink(m.paths, target.Repository(), target.Tag(), digest) - if err != nil { - return fmt.Errorf("create image tag: %w", err) - } - if err := writeMetadata(m.paths, target.Repository(), digest, meta); err != nil { - if restoreErr := restoreTagSymlink(&installed); restoreErr != nil { - return errors.Join( - fmt.Errorf("write tagged image metadata: %w", err), - fmt.Errorf("restore image tag: %w", restoreErr), - ) - } - return fmt.Errorf("write tagged image metadata: %w", err) - } - return nil -} - -func (m *manager) cleanupUnclaimedImage(ref *ResolvedRef) { - if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), ref.DigestHex(), true); err != nil { - slog.Warn("failed to collect stale image", "repository", ref.Repository(), "digest", ref.DigestHex(), "error", err) - } -} - -func (m *manager) cleanupReplacedTag(ref *NormalizedRef, previousDigest, digestHex string) { - if previousDigest == "" || previousDigest == digestHex { - return - } - count, err := countTagsForDigest(m.paths, ref.Repository(), previousDigest) - if err != nil { - slog.Warn("failed to count tags for replaced image", "repository", ref.Repository(), "digest", previousDigest, "error", err) - return - } - if count > 0 { - return - } - if err := removeDigestIfUnreferenced(m.paths, ref.Repository(), previousDigest, true); err != nil { - slog.Warn("failed to collect replaced image content", "repository", ref.Repository(), "digest", previousDigest, "error", err) - } - m.refreshDiskUsageTotals() -} - -func parseTagReferences(source, target string) (*NormalizedRef, *NormalizedRef, error) { - sourceRef, err := ParseNormalizedRef(source) - if err != nil { - return nil, nil, fmt.Errorf("%w: invalid source reference: %s", ErrInvalidName, err) - } - targetRef, err := ParseNormalizedRef(target) - if err != nil { - return nil, nil, fmt.Errorf("%w: invalid target reference: %s", ErrInvalidName, err) - } - if targetRef.IsDigest() { - return nil, nil, fmt.Errorf("%w: target must be a tag reference, not a digest", ErrInvalidName) - } - return sourceRef, targetRef, nil -} - -func (m *manager) readyTagImage(ref *NormalizedRef) (string, *imageMetadata, error) { - digestHex, meta, err := resolveRefMetadata(m.paths, ref) - if err != nil { - return "", nil, err - } - if meta.Status != StatusReady { - return "", nil, fmt.Errorf("%w: %s", ErrImageNotReady, meta.Status) - } - return digestHex, meta, nil -} diff --git a/lib/images/tag_test.go b/lib/images/tag_test.go index fe411b8c9..e717d9eea 100644 --- a/lib/images/tag_test.go +++ b/lib/images/tag_test.go @@ -54,7 +54,7 @@ func seedImage(t *testing.T, p *paths.Paths, repository, tag, digestHex string, func newTagTestCase(t *testing.T) (*paths.Paths, *manager, string) { t.Helper() p := paths.New(t.TempDir()) - return p, &manager{paths: p, tagGenerations: make(map[string]uint64), requestedTags: make(map[requestedTagKey]string)}, "docker.io/library/alpine" + return p, &manager{paths: p, tagGenerations: make(map[string]uint64), requestedTags: make(map[string]string)}, "docker.io/library/alpine" } func requireTagResolvesTo(t *testing.T, p *paths.Paths, repository, tag, digest string) { @@ -145,15 +145,15 @@ func TestReadyClaimDoesNotResurrectOlderPendingTag(t *testing.T) { newDigest := strings.Repeat("b2", 32) seedContent(t, p, repository, "latest", newDigest) - key := tagGenerationKey(repository, "latest") + key := tagKey(repository, "latest") m.tagGenerations[key] = 1 - m.requestedTags[requestedTagKeyFor(repository, "latest")] = oldDigest + m.requestedTags[tagKey(repository, "latest")] = oldDigest meta := &imageMetadata{RequestedTag: "latest", TagGeneration: 1} ref, err := ParseNormalizedRef(repository + ":latest") require.NoError(t, err) require.NoError(t, m.claimReadyTag(repository, "latest", newDigest)) - require.NotContains(t, m.requestedTags, requestedTagKeyFor(repository, "latest")) + require.NotContains(t, m.requestedTags, tagKey(repository, "latest")) require.False(t, m.claimRequestedTags(NewResolvedRef(ref, "sha256:"+oldDigest), meta)) requireTagResolvesTo(t, p, repository, "latest", newDigest) } @@ -195,8 +195,8 @@ func TestDeletingPendingTagDoesNotReclaimIt(t *testing.T) { } require.NoError(t, os.MkdirAll(p.ImageContentDir(newDigest), 0o755)) require.NoError(t, writeMetadataFile(p.ImageContentMetadata(newDigest), pending)) - m.tagGenerations[tagGenerationKey(repository, "latest")] = 1 - m.requestedTags = map[requestedTagKey]string{requestedTagKeyFor(repository, "latest"): newDigest} + m.tagGenerations[tagKey(repository, "latest")] = 1 + m.requestedTags = map[string]string{tagKey(repository, "latest"): newDigest} require.NoError(t, m.DeleteImage(context.Background(), repository+":latest")) pending, err := readContentMetadata(p, newDigest) @@ -218,9 +218,9 @@ func TestRestoreTagStateRestoresSecondaryClaims(t *testing.T) { TagClaims: []imageTagClaim{{Repository: repository, Tag: "stable", TagGeneration: 2}}, } m.restoreTagState([]*imageMetadata{meta}) - require.Equal(t, digest, m.requestedTags[requestedTagKeyFor(repository, "latest")]) - require.Equal(t, digest, m.requestedTags[requestedTagKeyFor(repository, "stable")]) - require.Equal(t, uint64(2), m.tagGenerations[tagGenerationKey(repository, "stable")]) + require.Equal(t, digest, m.requestedTags[tagKey(repository, "latest")]) + require.Equal(t, digest, m.requestedTags[tagKey(repository, "stable")]) + require.Equal(t, uint64(2), m.tagGenerations[tagKey(repository, "stable")]) } func TestPendingDigestClaimsAllRequestedTags(t *testing.T) { @@ -249,7 +249,7 @@ func TestWaitForReadyUsesNewestPendingTag(t *testing.T) { p, m, repository := newTagTestCase(t) m.queue = queue.New(1) m.readySubscribers = make(map[string][]chan StatusEvent) - m.requestedTags = make(map[requestedTagKey]string) + m.requestedTags = make(map[string]string) oldDigest := strings.Repeat("a1", 32) newDigest := strings.Repeat("b2", 32) seedContent(t, p, repository, "latest", oldDigest) @@ -260,7 +260,7 @@ func TestWaitForReadyUsesNewestPendingTag(t *testing.T) { Status: StatusPending, CreatedAt: time.Now().UTC(), } require.NoError(t, writeMetadataFile(p.ImageContentMetadata(newDigest), pending)) - m.requestedTags[requestedTagKeyFor(repository, "latest")] = newDigest + m.requestedTags[tagKey(repository, "latest")] = newDigest result := make(chan error, 1) ctx, cancel := context.WithCancel(context.Background()) @@ -414,8 +414,8 @@ func TestOlderPendingBuildRestoredWhenNewerBuildFails(t *testing.T) { RequestedTag: "latest", TagGeneration: 1, BuildID: "b", } require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digestB), metaB)) - m.tagGenerations[tagGenerationKey(repository, "latest")] = 1 - m.requestedTags[requestedTagKeyFor(repository, "latest")] = digestB + m.tagGenerations[tagKey(repository, "latest")] = 1 + m.requestedTags[tagKey(repository, "latest")] = digestB // Build C starts for latest (generation 2) require.NoError(t, os.MkdirAll(p.ImageContentDir(digestC), 0o755)) @@ -425,15 +425,15 @@ func TestOlderPendingBuildRestoredWhenNewerBuildFails(t *testing.T) { RequestedTag: "latest", TagGeneration: 2, BuildID: "c", } require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digestC), metaC)) - m.tagGenerations[tagGenerationKey(repository, "latest")] = 2 - m.requestedTags[requestedTagKeyFor(repository, "latest")] = digestC + m.tagGenerations[tagKey(repository, "latest")] = 2 + m.requestedTags[tagKey(repository, "latest")] = digestC // C fails m.releaseTagGeneration(repository, metaC.RequestedTag, metaC.TagGeneration) m.clearRequestedDigest(digestC) // requestedTags must restore to build B - require.Equal(t, digestB, m.requestedTags[requestedTagKeyFor(repository, "latest")]) + require.Equal(t, digestB, m.requestedTags[tagKey(repository, "latest")]) ref, err := ParseNormalizedRef(repository + ":latest") require.NoError(t, err) @@ -467,8 +467,8 @@ func TestSecondaryClaimReclaimedAfterNewerBuildFails(t *testing.T) { }, } require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digestB), metaB)) - m.tagGenerations[tagGenerationKey(repository, "stable")] = 1 - m.requestedTags[requestedTagKeyFor(repository, "stable")] = digestB + m.tagGenerations[tagKey(repository, "stable")] = 1 + m.requestedTags[tagKey(repository, "stable")] = digestB // Newer build C starts for stable (generation 2) require.NoError(t, os.MkdirAll(p.ImageContentDir(digestC), 0o755)) @@ -478,16 +478,16 @@ func TestSecondaryClaimReclaimedAfterNewerBuildFails(t *testing.T) { RequestedTag: "stable", TagGeneration: 2, BuildID: "c", } require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digestC), metaC)) - m.tagGenerations[tagGenerationKey(repository, "stable")] = 2 - m.requestedTags[requestedTagKeyFor(repository, "stable")] = digestC + m.tagGenerations[tagKey(repository, "stable")] = 2 + m.requestedTags[tagKey(repository, "stable")] = digestC // C fails m.releaseTagGeneration(repository, metaC.RequestedTag, metaC.TagGeneration) m.clearRequestedDigest(digestC) // Generation for stable rolled back to 1 and requested tag restored to B - require.Equal(t, uint64(1), m.tagGenerations[tagGenerationKey(repository, "stable")]) - require.Equal(t, digestB, m.requestedTags[requestedTagKeyFor(repository, "stable")]) + require.Equal(t, uint64(1), m.tagGenerations[tagKey(repository, "stable")]) + require.Equal(t, digestB, m.requestedTags[tagKey(repository, "stable")]) // B completes refB, err := ParseNormalizedRef(repository + ":other") @@ -523,16 +523,16 @@ func TestDeleteTaggedImageCleansPendingTagWithoutSymlink(t *testing.T) { _, m, repository := newTagTestCase(t) digest := strings.Repeat("e5", 32) - m.tagGenerations[tagGenerationKey(repository, "orphan")] = 3 - m.requestedTags[requestedTagKeyFor(repository, "orphan")] = digest + m.tagGenerations[tagKey(repository, "orphan")] = 3 + m.requestedTags[tagKey(repository, "orphan")] = digest err := m.deleteTaggedImage(repository, "orphan") require.Error(t, err) // Symlink was missing on disk // But memory tag state should be cleared - _, hasGen := m.tagGenerations[tagGenerationKey(repository, "orphan")] + _, hasGen := m.tagGenerations[tagKey(repository, "orphan")] require.False(t, hasGen) - _, hasReq := m.requestedTags[requestedTagKeyFor(repository, "orphan")] + _, hasReq := m.requestedTags[tagKey(repository, "orphan")] require.False(t, hasReq) } @@ -550,15 +550,15 @@ func TestDeleteDigestImageCleansPendingTagClaims(t *testing.T) { RequestedTag: "latest", TagGeneration: 2, BuildID: "b", } require.NoError(t, writeMetadataFile(p.ImageContentMetadata(digestB), metaB)) - m.tagGenerations[tagGenerationKey(repository, "latest")] = 2 - m.requestedTags[requestedTagKeyFor(repository, "latest")] = digestB + m.tagGenerations[tagKey(repository, "latest")] = 2 + m.requestedTags[tagKey(repository, "latest")] = digestB require.NoError(t, m.deleteDigestImage(repository, digestB)) // In-memory state for latest should be cleared - _, hasGen := m.tagGenerations[tagGenerationKey(repository, "latest")] + _, hasGen := m.tagGenerations[tagKey(repository, "latest")] require.False(t, hasGen) - _, hasReq := m.requestedTags[requestedTagKeyFor(repository, "latest")] + _, hasReq := m.requestedTags[tagKey(repository, "latest")] require.False(t, hasReq) // Pending metadata should have tag claim marked as canceled