diff --git a/cli/internal/cache/cache_fs.go b/cli/internal/cache/cache_fs.go
index 1256934fb870a..70991014efb95 100644
--- a/cli/internal/cache/cache_fs.go
+++ b/cli/internal/cache/cache_fs.go
@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"runtime"
 
@@ -77,12 +78,16 @@ func (f *fsCache) Put(target, hash string, duration int, files []string) error {
 	for i := 0; i < numDigesters; i++ {
 		g.Go(func() error {
 			for file := range fileQueue {
-				if !fs.IsDirectory(file) {
+				fromInfo, err := os.Lstat(file)
+				if err != nil {
+					return fmt.Errorf("error stat'ing cache source %v: %v", file, err)
+				}
+				if !fromInfo.IsDir() {
 					if err := fs.EnsureDir(filepath.Join(f.cacheDirectory, hash, file)); err != nil {
 						return fmt.Errorf("error ensuring directory file from cache: %w", err)
 					}
 
-					if err := fs.CopyOrLinkFile(file, filepath.Join(f.cacheDirectory, hash, file), fs.DirPermissions, fs.DirPermissions, true, true); err != nil {
+					if err := fs.CopyOrLinkFile(file, filepath.Join(f.cacheDirectory, hash, file), fromInfo.Mode(), fs.DirPermissions, true, true); err != nil {
 						return fmt.Errorf("error copying file from cache: %w", err)
 					}
 				}
diff --git a/cli/internal/fs/copy_file.go b/cli/internal/fs/copy_file.go
index 6e797499bbd84..e12292ae091cd 100644
--- a/cli/internal/fs/copy_file.go
+++ b/cli/internal/fs/copy_file.go
@@ -22,6 +22,10 @@ func CopyOrLinkFile(from, to string, fromMode, toMode os.FileMode, link, fallbac
 			if err != nil {
 				return err
 			}
+			// Make sure the link we're about to create doesn't already exist
+			if err := os.Remove(to); err != nil && !errors.Is(err, os.ErrNotExist) {
+				return err
+			}
 			return os.Symlink(dest, to)
 		}
 		if err := os.Link(from, to); err == nil || !fallback {