diff --git a/go.mod b/go.mod index 359e3c2d6..d094b57b7 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.23.6 require ( github.com/AlecAivazis/survey/v2 v2.3.7 + github.com/DefangLabs/secret-detector v0.0.0-20250108223530-c2b44d4c1f8f github.com/Microsoft/go-winio v0.6.2 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/buger/goterm v1.0.4 @@ -107,6 +108,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.0.0 // indirect github.com/gofrs/flock v0.12.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -120,6 +122,7 @@ require ( github.com/imdario/mergo v0.3.16 // indirect github.com/in-toto/in-toto-golang v0.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect @@ -190,6 +193,7 @@ require ( google.golang.org/protobuf v1.36.4 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/api v0.31.2 // indirect k8s.io/apimachinery v0.31.2 // indirect diff --git a/go.sum b/go.sum index f382eacc9..d763d277b 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DefangLabs/secret-detector v0.0.0-20250108223530-c2b44d4c1f8f h1:RTbUqLhPxejgK92ifVdMTIW9H23QLlscy8QXPDTfaL4= +github.com/DefangLabs/secret-detector v0.0.0-20250108223530-c2b44d4c1f8f/go.mod h1:2UjtD/G/Sy2FxoHpxKnzHTXMpRURecwYal8HgbxcvkY= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -195,6 +197,8 @@ github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -249,6 +253,8 @@ github.com/in-toto/in-toto-golang v0.5.0/go.mod h1:/Rq0IZHLV7Ku5gielPT4wPHJfH1Gd github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s= +github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4= github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8 h1:CZkYfurY6KGhVtlalI4QwQ6T0Cu6iuY3e0x5RLu96WE= github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d h1:jRQLvyVGL+iVtDElaEIDdKwpPqUIZJfzkNLV34htpEc= @@ -642,6 +648,8 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1 h1:d4KQkxAaAiRY2h5Zqis161Pv91A37uZyJOx73duwUwM= gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1/go.mod h1:WbjuEoo1oadwzQ4apSDU+JTvmllEHtsNHS6y7vFc7iw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/pkg/compose/publish.go b/pkg/compose/publish.go index 7131a3772..97200613e 100644 --- a/pkg/compose/publish.go +++ b/pkg/compose/publish.go @@ -17,12 +17,17 @@ package compose import ( + "bytes" "context" "crypto/sha256" "errors" "fmt" + "io" "os" + "github.com/DefangLabs/secret-detector/pkg/scanner" + "github.com/DefangLabs/secret-detector/pkg/secrets" + "github.com/compose-spec/compose-go/v2/loader" "github.com/compose-spec/compose-go/v2/types" "github.com/distribution/reference" @@ -226,15 +231,37 @@ func (s *composeService) generateImageDigestsOverride(ctx context.Context, proje return override.MarshalYAML() } +//nolint:gocyclo func (s *composeService) preChecks(project *types.Project, options api.PublishOptions) (bool, error) { - if ok, err := s.checkOnlyBuildSection(project); !ok { + if ok, err := s.checkOnlyBuildSection(project); !ok || err != nil { return false, err } + if ok, err := s.checkForBindMount(project); !ok || err != nil { + return false, err + } + if options.AssumeYes { + return true, nil + } + detectedSecrets, err := s.checkForSensitiveData(project) + if err != nil { + return false, err + } + if len(detectedSecrets) > 0 { + fmt.Println("you are about to publish sensitive data within your OCI artifact.\n" + + "please double check that you are not leaking sensitive data") + for _, val := range detectedSecrets { + _, _ = fmt.Fprintln(s.dockerCli.Out(), val.Type) + _, _ = fmt.Fprintf(s.dockerCli.Out(), "%q: %s\n", val.Key, val.Value) + } + if ok, err := acceptPublishSensitiveData(s.dockerCli); err != nil || !ok { + return false, err + } + } envVariables, err := s.checkEnvironmentVariables(project, options) if err != nil { return false, err } - if !options.AssumeYes && len(envVariables) > 0 { + if len(envVariables) > 0 { fmt.Println("you are about to publish environment variables within your OCI artifact.\n" + "please double check that you are not leaking sensitive data") for key, val := range envVariables { @@ -243,17 +270,10 @@ func (s *composeService) preChecks(project *types.Project, options api.PublishOp _, _ = fmt.Fprintf(s.dockerCli.Out(), "%s=%v\n", k, *v) } } - return acceptPublishEnvVariables(s.dockerCli) - } - - for name, config := range project.Services { - for _, volume := range config.Volumes { - if volume.Type == types.VolumeTypeBind { - return false, fmt.Errorf("cannot publish compose file: service %q relies on bind-mount. You should use volumes", name) - } + if ok, err := acceptPublishEnvVariables(s.dockerCli); err != nil || !ok { + return false, err } } - return true, nil } @@ -299,6 +319,12 @@ func acceptPublishEnvVariables(cli command.Cli) (bool, error) { return confirm, err } +func acceptPublishSensitiveData(cli command.Cli) (bool, error) { + msg := "Are you ok to publish these sensitive data? [y/N]: " + confirm, err := prompt.NewPrompt(cli.In(), cli.Out()).Confirm(msg, false) + return confirm, err +} + func envFileLayers(project *types.Project) []ocipush.Pushable { var layers []ocipush.Pushable for _, service := range project.Services { @@ -334,3 +360,99 @@ func (s *composeService) checkOnlyBuildSection(project *types.Project) (bool, er } return true, nil } + +func (s *composeService) checkForBindMount(project *types.Project) (bool, error) { + for name, config := range project.Services { + for _, volume := range config.Volumes { + if volume.Type == types.VolumeTypeBind { + return false, fmt.Errorf("cannot publish compose file: service %q relies on bind-mount. You should use volumes", name) + } + } + } + return true, nil +} + +func (s *composeService) checkForSensitiveData(project *types.Project) ([]secrets.DetectedSecret, error) { + var allFindings []secrets.DetectedSecret + scan := scanner.NewDefaultScanner() + // Check all compose files + for _, file := range project.ComposeFiles { + in, err := composeFileAsByteReader(file, project) + if err != nil { + return nil, err + } + + findings, err := scan.ScanReader(in) + if err != nil { + return nil, fmt.Errorf("failed to scan compose file %s: %w", file, err) + } + allFindings = append(allFindings, findings...) + } + for _, service := range project.Services { + // Check env files + for _, envFile := range service.EnvFiles { + findings, err := scan.ScanFile(envFile.Path) + if err != nil { + return nil, fmt.Errorf("failed to scan env file %s: %w", envFile.Path, err) + } + allFindings = append(allFindings, findings...) + } + } + + // Check configs defined by files + for _, config := range project.Configs { + if config.File != "" { + findings, err := scan.ScanFile(config.File) + if err != nil { + return nil, fmt.Errorf("failed to scan config file %s: %w", config.File, err) + } + allFindings = append(allFindings, findings...) + } + } + + // Check secrets defined by files + for _, secret := range project.Secrets { + if secret.File != "" { + findings, err := scan.ScanFile(secret.File) + if err != nil { + return nil, fmt.Errorf("failed to scan secret file %s: %w", secret.File, err) + } + allFindings = append(allFindings, findings...) + } + } + + return allFindings, nil +} + +func composeFileAsByteReader(filePath string, project *types.Project) (io.Reader, error) { + composeFile, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open compose file %s: %w", filePath, err) + } + base, err := loader.LoadWithContext(context.TODO(), types.ConfigDetails{ + WorkingDir: project.WorkingDir, + Environment: project.Environment, + ConfigFiles: []types.ConfigFile{ + { + Filename: filePath, + Content: composeFile, + }, + }, + }, func(options *loader.Options) { + options.SkipValidation = true + options.SkipExtends = true + options.SkipConsistencyCheck = true + options.ResolvePaths = true + options.SkipInterpolation = true + options.SkipResolveEnvironment = true + }) + if err != nil { + return nil, err + } + + in, err := base.MarshalYAML() + if err != nil { + return nil, err + } + return bytes.NewBuffer(in), nil +} diff --git a/pkg/e2e/fixtures/publish/compose-sensitive.yml b/pkg/e2e/fixtures/publish/compose-sensitive.yml new file mode 100644 index 000000000..68dd59b83 --- /dev/null +++ b/pkg/e2e/fixtures/publish/compose-sensitive.yml @@ -0,0 +1,20 @@ +services: + serviceA: + image: "alpine:3.12" + environment: + - AWS_ACCESS_KEY_ID=A3TX1234567890ABCDEF + - AWS_SECRET_ACCESS_KEY=aws"12345+67890/abcdefghijklm+NOPQRSTUVWXYZ+" + configs: + - myconfig + serviceB: + image: "alpine:3.12" + env_file: + - publish-sensitive.env + secrets: + - mysecret +configs: + myconfig: + file: config.txt +secrets: + mysecret: + file: secret.txt \ No newline at end of file diff --git a/pkg/e2e/fixtures/publish/config.txt b/pkg/e2e/fixtures/publish/config.txt new file mode 100644 index 000000000..32501ce3e --- /dev/null +++ b/pkg/e2e/fixtures/publish/config.txt @@ -0,0 +1 @@ +eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw \ No newline at end of file diff --git a/pkg/e2e/fixtures/publish/publish-sensitive.env b/pkg/e2e/fixtures/publish/publish-sensitive.env new file mode 100644 index 000000000..6ced33510 --- /dev/null +++ b/pkg/e2e/fixtures/publish/publish-sensitive.env @@ -0,0 +1 @@ +GITHUB_TOKEN=ghp_1234567890abcdefghijklmnopqrstuvwxyz diff --git a/pkg/e2e/fixtures/publish/publish.env b/pkg/e2e/fixtures/publish/publish.env index 7a41ee448..62eddb614 100644 --- a/pkg/e2e/fixtures/publish/publish.env +++ b/pkg/e2e/fixtures/publish/publish.env @@ -1,2 +1,2 @@ FOO=bar -QUIX= +QUIX= \ No newline at end of file diff --git a/pkg/e2e/fixtures/publish/secret.txt b/pkg/e2e/fixtures/publish/secret.txt new file mode 100644 index 000000000..5df0a6eea --- /dev/null +++ b/pkg/e2e/fixtures/publish/secret.txt @@ -0,0 +1,3 @@ +-----BEGIN DSA PRIVATE KEY----- +wxyz+ABC= +-----END DSA PRIVATE KEY----- \ No newline at end of file diff --git a/pkg/e2e/publish_test.go b/pkg/e2e/publish_test.go index d457455c2..92196daeb 100644 --- a/pkg/e2e/publish_test.go +++ b/pkg/e2e/publish_test.go @@ -134,4 +134,22 @@ FOO=bar`), res.Combined()) "-p", projectName, "alpha", "publish", "test/test", "--dry-run") res.Assert(t, icmd.Expected{ExitCode: 1, Err: "cannot publish compose file with local includes"}) }) + + t.Run("detect sensitive data", func(t *testing.T) { + cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/publish/compose-sensitive.yml", + "-p", projectName, "alpha", "publish", "test/test", "--with-env", "--dry-run") + cmd.Stdin = strings.NewReader("n\n") + res := icmd.RunCmd(cmd) + res.Assert(t, icmd.Expected{ExitCode: 0}) + + output := res.Combined() + assert.Assert(t, strings.Contains(output, "you are about to publish sensitive data within your OCI artifact.\n"), output) + assert.Assert(t, strings.Contains(output, "please double check that you are not leaking sensitive data"), output) + assert.Assert(t, strings.Contains(output, "AWS Client ID\n\"services.serviceA.environment.AWS_ACCESS_KEY_ID\": A3TX1234567890ABCDEF"), output) + assert.Assert(t, strings.Contains(output, "AWS Secret Key\n\"services.serviceA.environment.AWS_SECRET_ACCESS_KEY\": aws\"12345+67890/abcdefghijklm+NOPQRSTUVWXYZ+\""), output) + assert.Assert(t, strings.Contains(output, "Github authentication\n\"GITHUB_TOKEN\": ghp_1234567890abcdefghijklmnopqrstuvwxyz"), output) + assert.Assert(t, strings.Contains(output, "JSON Web Token\n\"\": eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9."+ + "eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw"), output) + assert.Assert(t, strings.Contains(output, "Private Key\n\"\": -----BEGIN DSA PRIVATE KEY-----\nwxyz+ABC=\n-----END DSA PRIVATE KEY-----"), output) + }) }