diff --git a/.vscode/launch.json b/.vscode/launch.json index 55f314e399e2c..2c7030df187ba 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,7 +5,7 @@ "version": "0.2.0", "configurations": [ { - "name": "Basic Turbo Build", + "name": "Build Basic", "type": "go", "request": "launch", "mode": "debug", @@ -21,6 +21,15 @@ "program": "${workspaceRoot}/cli/cmd/turbo", "cwd": "${workspaceRoot}/examples/basic", "args": ["--version"] + }, + { + "name": "Build All (Force)", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${workspaceRoot}/cli/cmd/turbo", + "cwd": "${workspaceRoot}", + "args": ["run", "build", "--force"] } ] } diff --git a/cli/internal/client/client.go b/cli/internal/client/client.go index e7f4aca6710c9..30e5b0cd6f884 100644 --- a/cli/internal/client/client.go +++ b/cli/internal/client/client.go @@ -2,7 +2,9 @@ package client import ( "context" + "crypto/md5" "crypto/x509" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -132,7 +134,7 @@ func (c *ApiClient) UserAgent() string { return fmt.Sprintf("turbo %v %v %v (%v)", c.turboVersion, runtime.Version(), runtime.GOOS, runtime.GOARCH) } -func (c *ApiClient) PutArtifact(hash string, duration int, rawBody interface{}) error { +func (c *ApiClient) PutArtifact(hash string, duration int, artifactReader io.Reader) error { if err := c.okToRequest(); err != nil { return err } @@ -143,11 +145,28 @@ func (c *ApiClient) PutArtifact(hash string, duration int, rawBody interface{}) if encoded != "" { encoded = "?" + encoded } - req, err := retryablehttp.NewRequest(http.MethodPut, c.makeUrl("/v8/artifacts/"+hash+encoded), rawBody) + // Read the entire artifactReader into memory so we can easily compute the Content-MD5. + // Note: retryablehttp.NewRequest reads the artifactReader into memory so there's no + // additional overhead by doing the ioutil.ReadAll here instead. + artifactBody, err := ioutil.ReadAll(artifactReader) + if err != nil { + return fmt.Errorf("failed to store files in HTTP cache: %w", err) + } + md5Sum := md5.Sum(artifactBody) + contentMd5 := base64.StdEncoding.EncodeToString(md5Sum[:]) + + req, err := retryablehttp.NewRequest(http.MethodPut, c.makeUrl("/v8/artifacts/"+hash+encoded), artifactBody) + // We need to initialize the trailer since it's not auto-initialized by the http module + req.Trailer = make(http.Header) + req.Trailer.Add("Content-MD5", contentMd5) + // ContentLenth -1 is required to set trailers on the http request. + req.ContentLength = -1 req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("x-artifact-duration", fmt.Sprintf("%v", duration)) req.Header.Set("Authorization", "Bearer "+c.Token) req.Header.Set("User-Agent", c.UserAgent()) + req.Header.Set("Trailer", "Content-MD5") + if err != nil { return fmt.Errorf("[WARNING] Invalid cache URL: %w", err) } diff --git a/cli/internal/client/client_test.go b/cli/internal/client/client_test.go index e534f43f436f0..3f9b61d2067c7 100644 --- a/cli/internal/client/client_test.go +++ b/cli/internal/client/client_test.go @@ -1,10 +1,14 @@ package client import ( + "crypto/md5" + "encoding/base64" "encoding/json" "io/ioutil" "net/http" + "net/http/httptest" "reflect" + "strings" "testing" "github.com/google/uuid" @@ -12,22 +16,21 @@ import ( ) func Test_sendToServer(t *testing.T) { - handler := http.NewServeMux() ch := make(chan []byte, 1) - handler.HandleFunc("/v8/artifacts/events", func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - b, err := ioutil.ReadAll(req.Body) - if err != nil { - t.Errorf("failed to read request %v", err) - } - ch <- b - w.WriteHeader(200) - w.Write([]byte{}) - }) - server := &http.Server{Addr: "localhost:8888", Handler: handler} - go server.ListenAndServe() + ts := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + defer req.Body.Close() + b, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Errorf("failed to read request %v", err) + } + ch <- b + w.WriteHeader(200) + w.Write([]byte{}) + })) + defer ts.Close() - apiClient := NewClient("http://localhost:8888", hclog.Default(), "v1", "", "my-team-slug", 1) + apiClient := NewClient(ts.URL, hclog.Default(), "v1", "", "my-team-slug", 1) apiClient.SetToken("my-token") myUUID, err := uuid.NewUUID() @@ -61,6 +64,42 @@ func Test_sendToServer(t *testing.T) { if !reflect.DeepEqual(events, result) { t.Errorf("roundtrip got %v, want %v", result, events) } +} + +func Test_PutArtifact(t *testing.T) { + ch := make(chan string, 2) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + defer req.Body.Close() + b, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Errorf("failed to read request %v", err) + } + ch <- string(b) + trailerMd5 := req.Trailer.Get("Content-MD5") + ch <- trailerMd5 + w.WriteHeader(200) + w.Write([]byte{}) + })) + defer ts.Close() + + // Set up test expected values + apiClient := NewClient(ts.URL+"/hash", hclog.Default(), "v1", "", "my-team-slug", 1) + apiClient.SetToken("my-token") + expectedArtifactBody := "My string artifact" + artifactReader := strings.NewReader(expectedArtifactBody) + md5Sum := md5.Sum([]byte(expectedArtifactBody)) + expectedMd5 := base64.StdEncoding.EncodeToString(md5Sum[:]) + + // Test Put Artifact + apiClient.PutArtifact("hash", 500, artifactReader) + testBody := <-ch + if expectedArtifactBody != testBody { + t.Errorf("Handler read '%v', wants '%v'", testBody, expectedArtifactBody) + } + + testMd5 := <-ch + if expectedMd5 != testMd5 { + t.Errorf("Handler read trailer '%v', wants '%v'", testMd5, expectedMd5) + } - server.Close() }