From 7c8ff36d78d01d16bddb52c483da90cac8e97489 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Mon, 4 Dec 2023 22:39:23 -0500 Subject: [PATCH] move around OCI logic, auto fallback/retry 1.1 -> 1.0 Signed-off-by: Milas Bowman --- cmd/compose/publish.go | 3 + docs/reference/compose_alpha_publish.md | 9 +- .../docker_compose_alpha_publish.yaml | 10 + internal/ocipush/push.go | 183 ++++++++++++++++++ pkg/api/api.go | 19 ++ pkg/compose/publish.go | 169 ++-------------- pkg/compose/publish_test.go | 56 ------ 7 files changed, 236 insertions(+), 213 deletions(-) create mode 100644 internal/ocipush/push.go delete mode 100644 pkg/compose/publish_test.go diff --git a/cmd/compose/publish.go b/cmd/compose/publish.go index 71f43ac03..40da5d944 100644 --- a/cmd/compose/publish.go +++ b/cmd/compose/publish.go @@ -28,6 +28,7 @@ import ( type publishOptions struct { *ProjectOptions resolveImageDigests bool + ociVersion string } func publishCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Service) *cobra.Command { @@ -44,6 +45,7 @@ func publishCommand(p *ProjectOptions, dockerCli command.Cli, backend api.Servic } flags := cmd.Flags() flags.BoolVar(&opts.resolveImageDigests, "resolve-image-digests", false, "Pin image tags to digests.") + flags.StringVar(&opts.ociVersion, "oci-version", "", "OCI Image specification version (automatically determined by default)") return cmd } @@ -55,5 +57,6 @@ func runPublish(ctx context.Context, dockerCli command.Cli, backend api.Service, return backend.Publish(ctx, project, repository, api.PublishOptions{ ResolveImageDigests: opts.resolveImageDigests, + OCIVersion: api.OCIVersion(opts.ociVersion), }) } diff --git a/docs/reference/compose_alpha_publish.md b/docs/reference/compose_alpha_publish.md index ba58d73c8..43bb3472c 100644 --- a/docs/reference/compose_alpha_publish.md +++ b/docs/reference/compose_alpha_publish.md @@ -5,10 +5,11 @@ Publish compose application ### Options -| Name | Type | Default | Description | -|:--------------------------|:-----|:--------|:--------------------------------| -| `--dry-run` | | | Execute command in dry run mode | -| `--resolve-image-digests` | | | Pin image tags to digests. | +| Name | Type | Default | Description | +|:--------------------------|:---------|:--------|:----------------------------------------------------------------------| +| `--dry-run` | | | Execute command in dry run mode | +| `--oci-version` | `string` | | OCI Image specification version (automatically determined by default) | +| `--resolve-image-digests` | | | Pin image tags to digests. | diff --git a/docs/reference/docker_compose_alpha_publish.yaml b/docs/reference/docker_compose_alpha_publish.yaml index 266e894c4..bc68468e7 100644 --- a/docs/reference/docker_compose_alpha_publish.yaml +++ b/docs/reference/docker_compose_alpha_publish.yaml @@ -5,6 +5,16 @@ usage: docker compose alpha publish [OPTIONS] [REPOSITORY] pname: docker compose alpha plink: docker_compose_alpha.yaml options: + - option: oci-version + value_type: string + description: | + OCI Image specification version (automatically determined by default) + deprecated: false + hidden: false + experimental: false + experimentalcli: false + kubernetes: false + swarm: false - option: resolve-image-digests value_type: bool default_value: "false" diff --git a/internal/ocipush/push.go b/internal/ocipush/push.go new file mode 100644 index 000000000..72c774d85 --- /dev/null +++ b/internal/ocipush/push.go @@ -0,0 +1,183 @@ +/* + Copyright 2023 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ocipush + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "path/filepath" + "time" + + pusherrors "github.com/containerd/containerd/remotes/errors" + "github.com/distribution/reference" + "github.com/docker/buildx/util/imagetools" + "github.com/docker/compose/v2/pkg/api" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +// clientAuthStatusCodes are client (4xx) errors that are authentication +// related. +var clientAuthStatusCodes = []int{ + http.StatusUnauthorized, + http.StatusForbidden, + http.StatusProxyAuthRequired, +} + +type Pushable struct { + Descriptor v1.Descriptor + Data []byte +} + +func DescriptorForComposeFile(path string, content []byte) v1.Descriptor { + return v1.Descriptor{ + MediaType: "application/vnd.docker.compose.file+yaml", + Digest: digest.FromString(string(content)), + Size: int64(len(content)), + Annotations: map[string]string{ + "com.docker.compose.version": api.ComposeVersion, + "com.docker.compose.file": filepath.Base(path), + }, + } +} + +func PushManifest( + ctx context.Context, + resolver *imagetools.Resolver, + named reference.Named, + layers []Pushable, + ociVersion api.OCIVersion, +) error { + // prepare to push the manifest by pushing the layers + layerDescriptors := make([]v1.Descriptor, len(layers)) + for i := range layers { + layerDescriptors[i] = layers[i].Descriptor + if err := resolver.Push(ctx, named, layers[i].Descriptor, layers[i].Data); err != nil { + return err + } + } + + if ociVersion != "" { + // if a version was explicitly specified, use it + return createAndPushManifest(ctx, resolver, named, layerDescriptors, ociVersion) + } + + // try to push in the OCI 1.1 format but fallback to OCI 1.0 on 4xx errors + // (other than auth) since it's most likely the result of the registry not + // having support + err := createAndPushManifest(ctx, resolver, named, layerDescriptors, api.OCIVersion1_1) + var pushErr pusherrors.ErrUnexpectedStatus + if errors.As(err, &pushErr) && isNonAuthClientError(pushErr.StatusCode) { + // TODO(milas): show a warning here (won't work with logrus) + return createAndPushManifest(ctx, resolver, named, layerDescriptors, api.OCIVersion1_0) + } + return err +} + +func createAndPushManifest( + ctx context.Context, + resolver *imagetools.Resolver, + named reference.Named, + layers []v1.Descriptor, + ociVersion api.OCIVersion, +) error { + toPush, err := generateManifest(layers, ociVersion) + if err != nil { + return err + } + for _, p := range toPush { + err = resolver.Push(ctx, named, p.Descriptor, p.Data) + if err != nil { + return err + } + } + return nil +} + +func isNonAuthClientError(statusCode int) bool { + if statusCode < 400 || statusCode >= 500 { + // not a client error + return false + } + for _, v := range clientAuthStatusCodes { + if statusCode == v { + // client auth error + return false + } + } + // any other 4xx client error + return true +} + +func generateManifest(layers []v1.Descriptor, ociCompat api.OCIVersion) ([]Pushable, error) { + var toPush []Pushable + var config v1.Descriptor + var artifactType string + switch ociCompat { + case api.OCIVersion1_0: + configData, err := json.Marshal(v1.ImageConfig{}) + if err != nil { + return nil, err + } + config = v1.Descriptor{ + MediaType: v1.MediaTypeImageConfig, + Digest: digest.FromBytes(configData), + Size: int64(len(configData)), + } + // N.B. OCI 1.0 does NOT support specifying the artifact type, so it's + // left as an empty string to omit it from the marshaled JSON + artifactType = "" + toPush = append(toPush, Pushable{Descriptor: config, Data: configData}) + case api.OCIVersion1_1: + config = v1.DescriptorEmptyJSON + artifactType = "application/vnd.docker.compose.project" + // N.B. the descriptor has the data embedded in it + toPush = append(toPush, Pushable{Descriptor: config, Data: nil}) + default: + return nil, fmt.Errorf("unsupported OCI version: %s", ociCompat) + } + + manifest, err := json.Marshal(v1.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: v1.MediaTypeImageManifest, + ArtifactType: artifactType, + Config: config, + Layers: layers, + Annotations: map[string]string{ + "org.opencontainers.image.created": time.Now().Format(time.RFC3339), + }, + }) + if err != nil { + return nil, err + } + + manifestDescriptor := v1.Descriptor{ + MediaType: v1.MediaTypeImageManifest, + Digest: digest.FromString(string(manifest)), + Size: int64(len(manifest)), + Annotations: map[string]string{ + "com.docker.compose.version": api.ComposeVersion, + }, + ArtifactType: artifactType, + } + toPush = append(toPush, Pushable{Descriptor: manifestDescriptor, Data: manifest}) + return toPush, nil +} diff --git a/pkg/api/api.go b/pkg/api/api.go index 926462efe..d07587401 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -361,9 +361,28 @@ type PortOptions struct { Index int } +// OCIVersion controls manifest generation to ensure compatibility +// with different registries. +// +// Currently, this is not exposed as an option to the user – Compose uses +// OCI 1.0 mode automatically for ECR registries based on domain and OCI 1.1 +// for all other registries. +// +// There are likely other popular registries that do not support the OCI 1.1 +// format, so it might make sense to expose this as a CLI flag or see if +// there's a way to generically probe the registry for support level. +type OCIVersion string + +const ( + OCIVersion1_0 OCIVersion = "1.0" + OCIVersion1_1 OCIVersion = "1.1" +) + // PublishOptions group options of the Publish API type PublishOptions struct { ResolveImageDigests bool + + OCIVersion OCIVersion } func (e Event) String() string { diff --git a/pkg/compose/publish.go b/pkg/compose/publish.go index 3c8dd2b3e..4a4a10d6d 100644 --- a/pkg/compose/publish.go +++ b/pkg/compose/publish.go @@ -18,38 +18,15 @@ package compose import ( "context" - "encoding/json" - "fmt" "os" - "path/filepath" - "strings" - "time" "github.com/compose-spec/compose-go/types" "github.com/distribution/reference" "github.com/docker/buildx/util/imagetools" + "github.com/docker/compose/v2/internal/ocipush" "github.com/docker/compose/v2/pkg/api" "github.com/docker/compose/v2/pkg/progress" "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/specs-go" - v1 "github.com/opencontainers/image-spec/specs-go/v1" -) - -// ociCompatibilityMode controls manifest generation to ensure compatibility -// with different registries. -// -// Currently, this is not exposed as an option to the user – Compose uses -// OCI 1.0 mode automatically for ECR registries based on domain and OCI 1.1 -// for all other registries. -// -// There are likely other popular registries that do not support the OCI 1.1 -// format, so it might make sense to expose this as a CLI flag or see if -// there's a way to generically probe the registry for support level. -type ociCompatibilityMode string - -const ( - ociCompatibility1_0 ociCompatibilityMode = "1.0" - ociCompatibility1_1 ociCompatibilityMode = "1.1" ) func (s *composeService) Publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error { @@ -73,18 +50,18 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re Auth: s.configFile(), }) - var layers []v1.Descriptor + var layers []ocipush.Pushable for _, file := range project.ComposeFiles { f, err := os.ReadFile(file) if err != nil { return err } - layer, err := s.pushComposeFile(ctx, file, f, resolver, named) - if err != nil { - return err - } - layers = append(layers, layer) + layerDescriptor := ocipush.DescriptorForComposeFile(file, f) + layers = append(layers, ocipush.Pushable{ + Descriptor: layerDescriptor, + Data: f, + }) } if options.ResolveImageDigests { @@ -93,17 +70,11 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re return err } - layer, err := s.pushComposeFile(ctx, "image-digests.yaml", yaml, resolver, named) - if err != nil { - return err - } - layers = append(layers, layer) - } - - ociCompat := inferOCIVersion(named) - toPush, err := s.generateManifest(layers, ociCompat) - if err != nil { - return err + layerDescriptor := ocipush.DescriptorForComposeFile("image-diegests.yaml", yaml) + layers = append(layers, ocipush.Pushable{ + Descriptor: layerDescriptor, + Data: yaml, + }) } w := progress.ContextWriter(ctx) @@ -113,12 +84,11 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re Status: progress.Working, }) if !s.dryRun { - for _, p := range toPush { - err = resolver.Push(ctx, named, p.Descriptor, p.Data) - if err != nil { - return err - } + err = ocipush.PushManifest(ctx, resolver, named, layers, options.OCIVersion) + if err != nil { + return err } + if err != nil { w.Event(progress.Event{ ID: repository, @@ -136,66 +106,6 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re return nil } -type push struct { - Descriptor v1.Descriptor - Data []byte -} - -func (s *composeService) generateManifest(layers []v1.Descriptor, ociCompat ociCompatibilityMode) ([]push, error) { - var toPush []push - var config v1.Descriptor - var artifactType string - switch ociCompat { - case ociCompatibility1_0: - configData, err := json.Marshal(v1.ImageConfig{}) - if err != nil { - return nil, err - } - config = v1.Descriptor{ - MediaType: v1.MediaTypeImageConfig, - Digest: digest.FromBytes(configData), - Size: int64(len(configData)), - } - // N.B. OCI 1.0 does NOT support specifying the artifact type, so it's - // left as an empty string to omit it from the marshaled JSON - artifactType = "" - toPush = append(toPush, push{Descriptor: config, Data: configData}) - case ociCompatibility1_1: - config = v1.DescriptorEmptyJSON - artifactType = "application/vnd.docker.compose.project" - // N.B. the descriptor has the data embedded in it - toPush = append(toPush, push{Descriptor: config, Data: nil}) - default: - return nil, fmt.Errorf("unsupported OCI version: %s", ociCompat) - } - - manifest, err := json.Marshal(v1.Manifest{ - Versioned: specs.Versioned{SchemaVersion: 2}, - MediaType: v1.MediaTypeImageManifest, - ArtifactType: artifactType, - Config: config, - Layers: layers, - Annotations: map[string]string{ - "org.opencontainers.image.created": time.Now().Format(time.RFC3339), - }, - }) - if err != nil { - return nil, err - } - - manifestDescriptor := v1.Descriptor{ - MediaType: v1.MediaTypeImageManifest, - Digest: digest.FromString(string(manifest)), - Size: int64(len(manifest)), - Annotations: map[string]string{ - "com.docker.compose.version": api.ComposeVersion, - }, - ArtifactType: artifactType, - } - toPush = append(toPush, push{Descriptor: manifestDescriptor, Data: manifest}) - return toPush, nil -} - func (s *composeService) generateImageDigestsOverride(ctx context.Context, project *types.Project) ([]byte, error) { project.ApplyProfiles([]string{"*"}) err := project.ResolveImages(func(named reference.Named) (digest.Digest, error) { @@ -221,50 +131,3 @@ func (s *composeService) generateImageDigestsOverride(ctx context.Context, proje } return override.MarshalYAML() } - -func (s *composeService) pushComposeFile(ctx context.Context, file string, content []byte, resolver *imagetools.Resolver, named reference.Named) (v1.Descriptor, error) { - w := progress.ContextWriter(ctx) - w.Event(progress.Event{ - ID: file, - Text: "publishing", - Status: progress.Working, - }) - layer := v1.Descriptor{ - MediaType: "application/vnd.docker.compose.file+yaml", - Digest: digest.FromString(string(content)), - Size: int64(len(content)), - Annotations: map[string]string{ - "com.docker.compose.version": api.ComposeVersion, - "com.docker.compose.file": filepath.Base(file), - }, - } - err := resolver.Push(ctx, named, layer, content) - w.Event(progress.Event{ - ID: file, - Text: "published", - Status: statusFor(err), - }) - return layer, err -} - -func statusFor(err error) progress.EventStatus { - if err != nil { - return progress.Error - } - return progress.Done -} - -// inferOCIVersion uses OCI 1.1 by default but falls back to OCI 1.0 if the -// registry domain is known to require it. -// -// This is not ideal - with private registries, there isn't a bounded set of -// domains. As it stands, it's primarily intended for compatibility with AWS -// Elastic Container Registry (ECR) due to its ubiquity. -func inferOCIVersion(named reference.Named) ociCompatibilityMode { - domain := reference.Domain(named) - if strings.HasSuffix(domain, "amazonaws.com") { - return ociCompatibility1_0 - } else { - return ociCompatibility1_1 - } -} diff --git a/pkg/compose/publish_test.go b/pkg/compose/publish_test.go deleted file mode 100644 index 2bb326f03..000000000 --- a/pkg/compose/publish_test.go +++ /dev/null @@ -1,56 +0,0 @@ -/* - Copyright 2023 Docker Compose CLI authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package compose - -import ( - "testing" - - "github.com/distribution/reference" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestInferOCIVersion(t *testing.T) { - tests := []struct { - ref string - want ociCompatibilityMode - }{ - { - ref: "175142243308.dkr.ecr.us-east-1.amazonaws.com/compose:test", - want: ociCompatibility1_0, - }, - { - ref: "my-image:latest", - want: ociCompatibility1_1, - }, - { - ref: "docker.io/docker/compose:test", - want: ociCompatibility1_1, - }, - { - ref: "ghcr.io/docker/compose:test", - want: ociCompatibility1_1, - }, - } - for _, tt := range tests { - t.Run(tt.ref, func(t *testing.T) { - named, err := reference.ParseDockerRef(tt.ref) - require.NoErrorf(t, err, "Test issue - invalid ref: %s", tt.ref) - assert.Equalf(t, tt.want, inferOCIVersion(named), "inferOCIVersion(%s)", tt.ref) - }) - } -}