这是indexloc提供的服务，不要输入任何密码
Skip to content

Log to temp files then rename #1351

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 9, 2022
Merged

Log to temp files then rename #1351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3e273fc
Log to temp file then rename
Jun 8, 2022
8f150a5
Add some comments about our log file renaming
Jun 8, 2022
5ac1862
Merge branch 'main' into gsoltis/log_to_tmp
kodiakhq[bot] Jun 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions cli/internal/fs/path.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,11 @@ func (ap AbsolutePath) Remove() error {
return os.Remove(ap.asString())
}

// RenameTo implements os.Rename for an absolute path
func (ap AbsolutePath) RenameTo(target AbsolutePath) error {
return os.Rename(ap.asString(), target.asString())
}

// GetVolumeRoot returns the root directory given an absolute path.
func GetVolumeRoot(absolutePath string) string {
return filepath.VolumeName(absolutePath) + string(os.PathSeparator)
Expand Down
50 changes: 37 additions & 13 deletions cli/internal/runcache/runcache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,8 @@ type TaskCache struct {
hash string
pt *nodes.PackageTask
cachingDisabled bool
LogFileName fs.AbsolutePath
targetLogFileName fs.AbsolutePath
tempLogFileName fs.AbsolutePath
}

// RestoreOutputs attempts to restore output for the corresponding task from the cache. Returns true
Expand All @@ -182,11 +183,11 @@ func (tc TaskCache) RestoreOutputs(terminal *cli.PrefixedUi, logger hclog.Logger
case HashLogs:
terminal.Output(fmt.Sprintf("cache hit, suppressing output %s", ui.Dim(tc.hash)))
case FullLogs:
logger.Debug("log file", "path", tc.LogFileName)
if tc.LogFileName.FileExists() {
logger.Debug("log file", "path", tc.targetLogFileName)
if tc.targetLogFileName.FileExists() {
// The task label is baked into the log file, so we need to grab the underlying Ui
// instance in order to not duplicate it
tc.rc.logReplayer(logger, terminal.Ui, tc.LogFileName)
tc.rc.logReplayer(logger, terminal.Ui, tc.targetLogFileName)
}
default:
// NoLogs, do not output anything
Expand All @@ -201,17 +202,35 @@ type nopWriteCloser struct {

func (nopWriteCloser) Close() error { return nil }

type fileWriterCloser struct {
// renamingCloser flushes, closes, and renames the given file
// to the target log file in the TaskCache.
//
// Log files are hard-linked into the cache, so when re-running a task
// we need to ensure we don't pollute a previous run. We do this by logging
// to a temp file and then performing this rename to the actual log file.
// Note that:
// - renames will succeed even if the target already exists, as long as the target isn't a directory.
// - All of our targets are log files
// - renames can fail if the rename is across filesystem boundaries.
// - Our renames are within the same directory.
type renamingCloser struct {
io.Writer
file *os.File
tc *TaskCache
bufio *bufio.Writer
}

func (fwc *fileWriterCloser) Close() error {
func (fwc *renamingCloser) Close() error {
if err := fwc.bufio.Flush(); err != nil {
return err
}
return fwc.file.Close()
if err := fwc.file.Close(); err != nil {
return err
}
if err := fwc.tc.tempLogFileName.RenameTo(fwc.tc.targetLogFileName); err != nil {
return err
}
return nil
}

// OutputWriter creates a sink suitable for handling the output of the command associated
Expand All @@ -220,11 +239,11 @@ func (tc TaskCache) OutputWriter() (io.WriteCloser, error) {
if tc.cachingDisabled || tc.rc.writesDisabled {
return nopWriteCloser{os.Stdout}, nil
}
// Setup log file
if err := tc.LogFileName.EnsureDir(); err != nil {
// Setup the temp log file
if err := tc.tempLogFileName.EnsureDir(); err != nil {
return nil, err
}
output, err := tc.LogFileName.Create()
output, err := tc.tempLogFileName.Create()
if err != nil {
return nil, err
}
Expand All @@ -237,9 +256,10 @@ func (tc TaskCache) OutputWriter() (io.WriteCloser, error) {
_ = output.Close()
return nil, err
}
fwc := &fileWriterCloser{
fwc := &renamingCloser{
file: output,
bufio: bufWriter,
tc: &tc,
}
if tc.rc.cacheMissLogsMode == NoLogs || tc.rc.cacheMissLogsMode == HashLogs {
// only write to log file, not to stdout
Expand Down Expand Up @@ -283,7 +303,10 @@ func (tc TaskCache) SaveOutputs(logger hclog.Logger, terminal cli.Ui, duration i
// TaskCache returns a TaskCache instance, providing an interface to the underlying cache specific
// to this run and the given PackageTask
func (rc *RunCache) TaskCache(pt *nodes.PackageTask, hash string) TaskCache {
logFileName := rc.repoRoot.Join(pt.RepoRelativeLogFile())
repoRelativeLogFile := pt.RepoRelativeLogFile()
repoRelativeTempLogFile := repoRelativeLogFile + ".tmp"
logFileName := rc.repoRoot.Join(repoRelativeLogFile)
tempLogFileName := rc.repoRoot.Join(repoRelativeTempLogFile)
hashableOutputs := pt.HashableOutputs()
repoRelativeGlobs := make([]string, len(hashableOutputs))
for index, output := range hashableOutputs {
Expand All @@ -295,7 +318,8 @@ func (rc *RunCache) TaskCache(pt *nodes.PackageTask, hash string) TaskCache {
hash: hash,
pt: pt,
cachingDisabled: !pt.TaskDefinition.ShouldCache,
LogFileName: logFileName,
targetLogFileName: logFileName,
tempLogFileName: tempLogFileName,
}
}

Expand Down