diff --git a/go.mod b/go.mod
index 067ff46db..37099dfce 100644
--- a/go.mod
+++ b/go.mod
@@ -46,6 +46,7 @@ require (
 	go.uber.org/goleak v1.3.0
 	golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1
 	golang.org/x/sync v0.4.0
+	golang.org/x/sys v0.11.0
 	google.golang.org/grpc v1.59.0
 	gotest.tools/v3 v3.5.1
 )
@@ -154,7 +155,6 @@ require (
 	golang.org/x/mod v0.11.0 // indirect
 	golang.org/x/net v0.14.0 // indirect
 	golang.org/x/oauth2 v0.11.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
 	golang.org/x/term v0.11.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
diff --git a/pkg/remote/cache.go b/pkg/remote/cache.go
new file mode 100644
index 000000000..a0a6d0319
--- /dev/null
+++ b/pkg/remote/cache.go
@@ -0,0 +1,36 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package remote
+
+import (
+	"os"
+	"path/filepath"
+)
+
+func cacheDir() (string, error) {
+	cache, ok := os.LookupEnv("XDG_CACHE_HOME")
+	if ok {
+		return filepath.Join(cache, "docker-compose"), nil
+	}
+
+	path, err := osDependentCacheDir()
+	if err != nil {
+		return "", err
+	}
+	err = os.MkdirAll(path, 0o700)
+	return path, err
+}
diff --git a/pkg/remote/cache_darwin.go b/pkg/remote/cache_darwin.go
new file mode 100644
index 000000000..59f5589ae
--- /dev/null
+++ b/pkg/remote/cache_darwin.go
@@ -0,0 +1,30 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package remote
+
+import (
+	"os"
+	"path/filepath"
+)
+
+func osDependentCacheDir() (string, error) {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(home, "Library", "Caches", "docker-compose"), nil
+}
diff --git a/pkg/remote/cache_unix.go b/pkg/remote/cache_unix.go
new file mode 100644
index 000000000..bf9a95479
--- /dev/null
+++ b/pkg/remote/cache_unix.go
@@ -0,0 +1,32 @@
+//go:build linux
+
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package remote
+
+import (
+	"os"
+	"path/filepath"
+)
+
+func osDependentCacheDir() (string, error) {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(home, ".cache", "docker-compose"), nil
+}
diff --git a/pkg/remote/cache_windows.go b/pkg/remote/cache_windows.go
new file mode 100644
index 000000000..767d83a5c
--- /dev/null
+++ b/pkg/remote/cache_windows.go
@@ -0,0 +1,45 @@
+/*
+   Copyright 2020 Docker Compose CLI authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package remote
+
+import (
+	"os"
+	"path/filepath"
+
+	"golang.org/x/sys/windows"
+)
+
+func osDependentCacheDir() (string, error) {
+	flags := []uint32{windows.KF_FLAG_DEFAULT, windows.KF_FLAG_DEFAULT_PATH}
+	for _, flag := range flags {
+		p, _ := windows.KnownFolderPath(windows.FOLDERID_LocalAppData, flag|windows.KF_FLAG_DONT_VERIFY)
+		if p != "" {
+			return filepath.Join(p, "cache", "docker-compose"), nil
+		}
+	}
+
+	appData, ok := os.LookupEnv("LOCALAPPDATA")
+	if ok {
+		return filepath.Join(appData, "cache", "docker-compose"), nil
+	}
+
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(home, "AppData", "Local", "cache", "docker-compose"), nil
+}
diff --git a/pkg/remote/git.go b/pkg/remote/git.go
index b6228286b..4f3d233de 100644
--- a/pkg/remote/git.go
+++ b/pkg/remote/git.go
@@ -25,7 +25,6 @@ import (
 	"regexp"
 	"strconv"
 
-	"github.com/adrg/xdg"
 	"github.com/compose-spec/compose-go/cli"
 	"github.com/compose-spec/compose-go/loader"
 	"github.com/compose-spec/compose-go/types"
@@ -47,15 +46,10 @@ func gitRemoteLoaderEnabled() (bool, error) {
 }
 
 func NewGitRemoteLoader(offline bool) (loader.ResourceLoader, error) {
-	// xdg.CacheFile creates the parent directories for the target file path
-	// and returns the fully qualified path, so use "git" as a filename and
-	// then chop it off after, i.e. no ~/.cache/docker-compose/git file will
-	// ever be created
-	cache, err := xdg.CacheFile(filepath.Join("docker-compose", "git"))
+	cache, err := cacheDir()
 	if err != nil {
-		return nil, fmt.Errorf("initializing git cache: %w", err)
+		return nil, fmt.Errorf("initializing remote resource cache: %w", err)
 	}
-	cache = filepath.Dir(cache)
 	return gitRemoteLoader{
 		cache:   cache,
 		offline: offline,
diff --git a/pkg/remote/oci.go b/pkg/remote/oci.go
index 9e3a1b731..a67bfb0a1 100644
--- a/pkg/remote/oci.go
+++ b/pkg/remote/oci.go
@@ -25,7 +25,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/adrg/xdg"
 	"github.com/compose-spec/compose-go/loader"
 	"github.com/distribution/reference"
 	"github.com/docker/buildx/store/storeutil"
@@ -48,15 +47,11 @@ func ociRemoteLoaderEnabled() (bool, error) {
 }
 
 func NewOCIRemoteLoader(dockerCli command.Cli, offline bool) (loader.ResourceLoader, error) {
-	// xdg.CacheFile creates the parent directories for the target file path
-	// and returns the fully qualified path, so use "git" as a filename and
-	// then chop it off after, i.e. no ~/.cache/docker-compose/git file will
-	// ever be created
-	cache, err := xdg.CacheFile(filepath.Join("docker-compose", "oci"))
+	cache, err := cacheDir()
 	if err != nil {
-		return nil, fmt.Errorf("initializing git cache: %w", err)
+		return nil, fmt.Errorf("initializing remote resource cache: %w", err)
 	}
-	cache = filepath.Dir(cache)
+
 	return ociRemoteLoader{
 		cache:     cache,
 		dockerCli: dockerCli,