这是indexloc提供的服务，不要输入任何密码
Skip to content

Add two log output modes, stdout-full and stdout-new-only #3439

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 9 commits into from
Closed

Add two log output modes, stdout-full and stdout-new-only #3439

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
77331ad
Add output modes for stdout-full and stdout-new-only
rafaeltab Jan 21, 2023
17f62fe
Stop adding prefixes when using stdout mode
rafaeltab Jan 21, 2023
15f4c70
Fixed two problems
rafaeltab Jan 22, 2023
a702f5b
Wrong way around in logstreamer
rafaeltab Jan 22, 2023
f20f8a8
Update integration tests to reflect help changes
rafaeltab Jan 22, 2023
289d664
Integration tests for output logs, TODO for stdout > is interpreted a…
rafaeltab Jan 22, 2023
df712e1
Fix the integration tests
rafaeltab Jan 23, 2023
a17b189
Add integration tests for no force for new only, and stdout-full
rafaeltab Jan 23, 2023
b48e362
Merge branch 'main' into feat/stdout-log-output
rafaeltab Jan 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 137 additions & 0 deletions cli/integration_tests/basic_monorepo/output_logs.t
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
Setup
$ . ${TESTDIR}/../setup.sh
$ . ${TESTDIR}/setup.sh $(pwd)

Output-logs full
$ ${TURBO} run build --output-logs=full --filter=util --force
\xe2\x80\xa2 Packages in scope: util (esc)
\xe2\x80\xa2 Running build in 1 packages (esc)
\xe2\x80\xa2 Remote caching disabled (esc)
util:build: cache bypass, force executing 6dec18f9f767112f
util:build:
util:build: \x3e build (esc)
util:build: \x3e echo 'building' (esc)
util:build:
util:build: building

Tasks: 1 successful, 1 total
Cached: 0 cached, 1 total
Time:\s*[\.0-9]+m?s (re)

Output-logs none
$ ${TURBO} run build --output-logs=none --filter=util --force
\xe2\x80\xa2 Packages in scope: util (esc)
\xe2\x80\xa2 Running build in 1 packages (esc)
\xe2\x80\xa2 Remote caching disabled (esc)

Tasks: 1 successful, 1 total
Cached: 0 cached, 1 total
Time:\s*[\.0-9]+m?s (re)

Output-logs hash-only
$ ${TURBO} run build --output-logs=hash-only --filter=util --force
\xe2\x80\xa2 Packages in scope: util (esc)
\xe2\x80\xa2 Running build in 1 packages (esc)
\xe2\x80\xa2 Remote caching disabled (esc)
util:build: cache bypass, force executing 6dec18f9f767112f

Tasks: 1 successful, 1 total
Cached: 0 cached, 1 total
Time:\s*[\.0-9]+m?s (re)

Output-logs new-only
$ ${TURBO} run build --output-logs=new-only --filter=util --force
\xe2\x80\xa2 Packages in scope: util (esc)
\xe2\x80\xa2 Running build in 1 packages (esc)
\xe2\x80\xa2 Remote caching disabled (esc)
util:build: cache bypass, force executing 6dec18f9f767112f
util:build:
util:build: \x3e build (esc)
util:build: \x3e echo 'building' (esc)
util:build:
util:build: building

Tasks: 1 successful, 1 total
Cached: 0 cached, 1 total
Time:\s*[\.0-9]+m?s (re)

Output-logs new-only no force
$ ${TURBO} run build --output-logs=new-only --filter=util
\xe2\x80\xa2 Packages in scope: util (esc)
\xe2\x80\xa2 Running build in 1 packages (esc)
\xe2\x80\xa2 Remote caching disabled (esc)
util:build: cache hit, suppressing output 6dec18f9f767112f

Tasks: 1 successful, 1 total
Cached: 1 cached, 1 total
Time:\s*[\.0-9]+m?s >>> FULL TURBO (re)

Output-logs error-only
$ ${TURBO} run build --output-logs=errors-only --filter=util --force
\xe2\x80\xa2 Packages in scope: util (esc)
\xe2\x80\xa2 Running build in 1 packages (esc)
\xe2\x80\xa2 Remote caching disabled (esc)

Tasks: 1 successful, 1 total
Cached: 0 cached, 1 total
Time:\s*[\.0-9]+m?s (re)

Output-logs stdout-full
$ ${TURBO} run build --output-logs=stdout-full --filter=util --force
\xe2\x80\xa2 Packages in scope: util (esc)
\xe2\x80\xa2 Running build in 1 packages (esc)
\xe2\x80\xa2 Remote caching disabled (esc)
util:build: cache bypass, force executing 6dec18f9f767112f

\x3e build (esc)
\x3e echo 'building' (esc)

building

Tasks: 1 successful, 1 total
Cached: 0 cached, 1 total
Time:\s*[\.0-9]+m?s (re)

Output-logs stdout-new-only
$ ${TURBO} run build --output-logs=stdout-new-only --filter=util --force
\xe2\x80\xa2 Packages in scope: util (esc)
\xe2\x80\xa2 Running build in 1 packages (esc)
\xe2\x80\xa2 Remote caching disabled (esc)
util:build: cache bypass, force executing 6dec18f9f767112f

\x3e build (esc)
\x3e echo 'building' (esc)

building

Tasks: 1 successful, 1 total
Cached: 0 cached, 1 total
Time:\s*[\.0-9]+m?s (re)

Output-logs stdout-full no force
$ ${TURBO} run build --output-logs=stdout-full --filter=util
\xe2\x80\xa2 Packages in scope: util (esc)
\xe2\x80\xa2 Running build in 1 packages (esc)
\xe2\x80\xa2 Remote caching disabled (esc)
util:build: cache hit, replaying output 6dec18f9f767112f

\x3e build (esc)
\x3e echo 'building' (esc)

building

Tasks: 1 successful, 1 total
Cached: 1 cached, 1 total
Time:\s*[\.0-9]+m?s >>> FULL TURBO (re)

Output-logs stdout-new-only no force
$ ${TURBO} run build --output-logs=stdout-new-only --filter=util
\xe2\x80\xa2 Packages in scope: util (esc)
\xe2\x80\xa2 Running build in 1 packages (esc)
\xe2\x80\xa2 Remote caching disabled (esc)
util:build: cache hit, suppressing output 6dec18f9f767112f

Tasks: 1 successful, 1 total
Cached: 1 cached, 1 total
Time:\s*[\.0-9]+m?s >>> FULL TURBO (re)

2 changes: 1 addition & 1 deletion cli/integration_tests/no_args.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ Make sure exit code is 2 when no args are passed
--no-cache Avoid saving task results to the cache. Useful for development/watch tasks
--no-daemon Run without using turbo's daemon process
--no-deps Exclude dependent task consumers from execution
--output-logs <OUTPUT_LOGS> Set type of process output logging. Use "full" to show all output. Use "hash-only" to show only turbo-computed task hashes. Use "new-only" to show only new output with only hashes for cached tasks. Use "none" to hide process output. (default full) [possible values: full, none, hash-only, new-only, errors-only]
--output-logs <OUTPUT_LOGS> Set type of process output logging. Use "full" to show all output. Use "hash-only" to show only turbo-computed task hashes. Use "new-only" to show only new output with only hashes for cached tasks. Use "none" to hide process output. (default full) [possible values: full, none, hash-only, new-only, errors-only, stdout-full, stdout-new-only]
--parallel Execute all tasks in parallel
--profile <PROFILE> File to write turbo's performance profile output into. You can load the file up in chrome://tracing to see which parts of your build were slow
--remote-only Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache
Expand Down
4 changes: 2 additions & 2 deletions cli/integration_tests/turbo_help.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ Test help flag
--no-cache Avoid saving task results to the cache. Useful for development/watch tasks
--no-daemon Run without using turbo's daemon process
--no-deps Exclude dependent task consumers from execution
--output-logs <OUTPUT_LOGS> Set type of process output logging. Use "full" to show all output. Use "hash-only" to show only turbo-computed task hashes. Use "new-only" to show only new output with only hashes for cached tasks. Use "none" to hide process output. (default full) [possible values: full, none, hash-only, new-only, errors-only]
--output-logs <OUTPUT_LOGS> Set type of process output logging. Use "full" to show all output. Use "hash-only" to show only turbo-computed task hashes. Use "new-only" to show only new output with only hashes for cached tasks. Use "none" to hide process output. (default full) [possible values: full, none, hash-only, new-only, errors-only, stdout-full, stdout-new-only]
--parallel Execute all tasks in parallel
--profile <PROFILE> File to write turbo's performance profile output into. You can load the file up in chrome://tracing to see which parts of your build were slow
--remote-only Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache
Expand Down Expand Up @@ -112,7 +112,7 @@ Test help flag
--no-cache Avoid saving task results to the cache. Useful for development/watch tasks
--no-daemon Run without using turbo's daemon process
--no-deps Exclude dependent task consumers from execution
--output-logs <OUTPUT_LOGS> Set type of process output logging. Use "full" to show all output. Use "hash-only" to show only turbo-computed task hashes. Use "new-only" to show only new output with only hashes for cached tasks. Use "none" to hide process output. (default full) [possible values: full, none, hash-only, new-only, errors-only]
--output-logs <OUTPUT_LOGS> Set type of process output logging. Use "full" to show all output. Use "hash-only" to show only turbo-computed task hashes. Use "new-only" to show only new output with only hashes for cached tasks. Use "none" to hide process output. (default full) [possible values: full, none, hash-only, new-only, errors-only, stdout-full, stdout-new-only]
--parallel Execute all tasks in parallel
--profile <PROFILE> File to write turbo's performance profile output into. You can load the file up in chrome://tracing to see which parts of your build were slow
--remote-only Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache
Expand Down
54 changes: 40 additions & 14 deletions cli/internal/logstreamer/logstreamer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,12 @@ type Logstreamer struct {
colorOkay string
colorFail string
colorReset string

// If true, output is not altered, and written straight to stdout
stdout bool
}

func NewLogstreamer(logger *log.Logger, prefix string, record bool) *Logstreamer {
func NewLogstreamer(logger *log.Logger, prefix string, record bool, stdout bool) *Logstreamer {
streamer := &Logstreamer{
Logger: logger,
buf: bytes.NewBuffer([]byte("")),
Expand All @@ -39,6 +42,7 @@ func NewLogstreamer(logger *log.Logger, prefix string, record bool) *Logstreamer
colorOkay: "",
colorFail: "",
colorReset: "",
stdout: stdout,
}

if strings.HasPrefix(os.Getenv("TERM"), "xterm") {
Expand Down Expand Up @@ -121,39 +125,61 @@ func (l *Logstreamer) out(str string) {
l.persist = l.persist + str
}

if l.prefix == "stdout" {
str = l.colorOkay + l.prefix + l.colorReset + " " + str
} else if l.prefix == "stderr" {
str = l.colorFail + l.prefix + l.colorReset + " " + str
if !l.stdout {
if l.prefix == "stdout" {
str = l.colorOkay + l.prefix + l.colorReset + " " + str
} else if l.prefix == "stderr" {
str = l.colorFail + l.prefix + l.colorReset + " " + str
}
}

l.Logger.Print(str)
}

// PrettyStdoutWriter wraps an ioWriter so it can add string
// StdoutWriter wraps an ioWriter so it can add string
// prefixes to every message it writes to stdout.
type PrettyStdoutWriter struct {
type StdoutWriter struct {
w io.Writer
Prefix string
// If true, output is not altered, and written straight to stdout
Stdout bool
}

var _ io.Writer = (*PrettyStdoutWriter)(nil)
var _ io.Writer = (*StdoutWriter)(nil)

// NewPrettyStdoutWriter returns an instance of PrettyStdoutWriter
func NewPrettyStdoutWriter(prefix string) *PrettyStdoutWriter {
return &PrettyStdoutWriter{
// NewPrettyStdoutWriter returns an instance of StdoutWriter in pretty mode
func NewPrettyStdoutWriter(prefix string) *StdoutWriter {
return &StdoutWriter{
w: os.Stdout,
Prefix: prefix,
Stdout: false,
}
}

func (psw *PrettyStdoutWriter) Write(p []byte) (int, error) {
str := psw.Prefix + string(p)
n, err := psw.w.Write([]byte(str))
func (psw *StdoutWriter) Write(p []byte) (int, error) {
var data []byte

if !psw.Stdout {
str := psw.Prefix + string(p)
data = []byte(str)
} else {
data = p
}

n, err := psw.w.Write(data)

if err != nil {
return n, err
}

return len(p), nil
}

// NewStdoutWriter returns an instance of StdoutWriter in stdout mode
func NewStdoutWriter() *StdoutWriter {
return &StdoutWriter{
w: os.Stdout,
Prefix: "",
Stdout: true,
}
}
10 changes: 5 additions & 5 deletions cli/internal/logstreamer/logstreamer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@ func TestLogstreamerOk(t *testing.T) {
logger := log.New(os.Stdout, "--> ", log.Ldate|log.Ltime)

// Setup a streamer that we'll pipe cmd.Stdout to
logStreamerOut := NewLogstreamer(logger, "stdout", false)
logStreamerOut := NewLogstreamer(logger, "stdout", false, false)
defer logStreamerOut.Close()
// Setup a streamer that we'll pipe cmd.Stderr to.
// We want to record/buffer anything that's written to this (3rd argument true)
logStreamerErr := NewLogstreamer(logger, "stderr", true)
logStreamerErr := NewLogstreamer(logger, "stderr", true, false)
defer logStreamerErr.Close()

// Execute something that succeeds
Expand Down Expand Up @@ -57,11 +57,11 @@ func TestLogstreamerErr(t *testing.T) {
logger := log.New(os.Stdout, "--> ", log.Ldate|log.Ltime)

// Setup a streamer that we'll pipe cmd.Stdout to
logStreamerOut := NewLogstreamer(logger, "stdout", false)
logStreamerOut := NewLogstreamer(logger, "stdout", false, false)
defer logStreamerOut.Close()
// Setup a streamer that we'll pipe cmd.Stderr to.
// We want to record/buffer anything that's written to this (3rd argument true)
logStreamerErr := NewLogstreamer(logger, "stderr", true)
logStreamerErr := NewLogstreamer(logger, "stderr", true, false)
defer logStreamerErr.Close()

// Execute something that succeeds
Expand Down Expand Up @@ -99,7 +99,7 @@ func TestLogstreamerFlush(t *testing.T) {
byteWriter := bufio.NewWriter(&buffer)

logger := log.New(byteWriter, "", 0)
logStreamerOut := NewLogstreamer(logger, "", false)
logStreamerOut := NewLogstreamer(logger, "", false, false)
defer logStreamerOut.Close()

logStreamerOut.Write([]byte(text))
Expand Down
12 changes: 8 additions & 4 deletions cli/internal/run/real_run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/vercel/turbo/cli/internal/taskhash"
"github.com/vercel/turbo/cli/internal/turbopath"
"github.com/vercel/turbo/cli/internal/ui"
"github.com/vercel/turbo/cli/internal/util"
)

// RealRun executes a set of tasks
Expand Down Expand Up @@ -157,6 +158,9 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas
progressLogger := ec.logger.Named("")
progressLogger.Debug("start")

// TODO(rafaeltab) this doesn't work
isStdout := ec.rs.Opts.runcacheOpts.TaskOutputModeOverride != nil && (*ec.rs.Opts.runcacheOpts.TaskOutputModeOverride == util.StdoutFullTaskOutput || *ec.rs.Opts.runcacheOpts.TaskOutputModeOverride == util.StdoutNewTaskOutput)

// Setup tracer
tracer := ec.runState.Run(packageTask.TaskID)

Expand Down Expand Up @@ -211,7 +215,7 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas
// Setup stdout/stderr
// If we are not caching anything, then we don't need to write logs to disk
// be careful about this conditional given the default of cache = true
writer, err := taskCache.OutputWriter(prettyPrefix)
writer, err := taskCache.OutputWriter(prettyPrefix, isStdout)
if err != nil {
tracer(TargetBuildFailed, err)
ec.logError(progressLogger, prettyPrefix, err)
Expand All @@ -223,9 +227,9 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas
// Create a logger
logger := log.New(writer, "", 0)
// Setup a streamer that we'll pipe cmd.Stdout to
logStreamerOut := logstreamer.NewLogstreamer(logger, prettyPrefix, false)
logStreamerOut := logstreamer.NewLogstreamer(logger, prettyPrefix, false, isStdout)
// Setup a streamer that we'll pipe cmd.Stderr to.
logStreamerErr := logstreamer.NewLogstreamer(logger, prettyPrefix, false)
logStreamerErr := logstreamer.NewLogstreamer(logger, prettyPrefix, false, isStdout)
cmd.Stderr = logStreamerErr
cmd.Stdout = logStreamerOut
// Flush/Reset any error we recorded
Expand Down Expand Up @@ -274,7 +278,7 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas
}

// If there was an error, flush the buffered output
taskCache.OnError(prefixedUI, progressLogger)
taskCache.OnError(prefixedUI, progressLogger, isStdout)

return err
}
Expand Down
Loading