diff --git a/api/src/main/java/com/theokanning/openai/fine_tuning/FineTuningJob.java b/api/src/main/java/com/theokanning/openai/fine_tuning/FineTuningJob.java index 685d751..abd6b23 100644 --- a/api/src/main/java/com/theokanning/openai/fine_tuning/FineTuningJob.java +++ b/api/src/main/java/com/theokanning/openai/fine_tuning/FineTuningJob.java @@ -1,6 +1,7 @@ package com.theokanning.openai.fine_tuning; import com.fasterxml.jackson.annotation.JsonProperty; +import com.theokanning.openai.OpenAiError; import lombok.Data; import java.util.List; @@ -27,6 +28,8 @@ public class FineTuningJob { @JsonProperty("created_at") Long createdAt; + OpenAiError.OpenAiErrorDetails error; + /** * The unix timestamp for when the fine-tuning job was finished. */ @@ -87,4 +90,20 @@ public class FineTuningJob { */ @JsonProperty("trained_tokens") Integer trainedTokens; -} \ No newline at end of file + + /** + * A list of integrations to enable for this fine-tuning job. + */ + List integrations; + + /** + * The seed controls the reproducibility of the job. Passing in the same seed and job parameters should produce the same results, but may differ in rare cases. If a seed is not specified, one will be generated for you. + */ + Integer seed; + + /** + * The Unix timestamp (in seconds) for when the fine-tuning job is estimated to finish. The value will be null if the fine-tuning job is not running. + */ + @JsonProperty("estimated_finish") + Long estimatedFinish; +} diff --git a/api/src/main/java/com/theokanning/openai/fine_tuning/FineTuningJobCheckpoint.java b/api/src/main/java/com/theokanning/openai/fine_tuning/FineTuningJobCheckpoint.java new file mode 100644 index 0000000..d23c118 --- /dev/null +++ b/api/src/main/java/com/theokanning/openai/fine_tuning/FineTuningJobCheckpoint.java @@ -0,0 +1,90 @@ +package com.theokanning.openai.fine_tuning; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +/** + * @author LiangTao + * @date 2024年05月14 10:23 + **/ +@Data +public class FineTuningJobCheckpoint { + /** + * id + * string + * + * The checkpoint identifier, which can be referenced in the API endpoints. + * + * created_at + * integer + * + * The Unix timestamp (in seconds) for when the checkpoint was created. + * + * fine_tuned_model_checkpoint + * string + * + * The name of the fine-tuned checkpoint model that is created. + * + * step_number + * integer + * + * The step number that the checkpoint was created at. + * + * metrics + * object + * + * Metrics at the step number during the fine-tuning job. + * + * + * Show properties + * fine_tuning_job_id + * string + * + * The name of the fine-tuning job that this checkpoint was created from. + * + * object + * string + * + * The object type, which is always "fine_tuning.job.checkpoint". + */ + + /** + * The checkpoint identifier, which can be referenced in the API endpoints. + */ + String id; + + /** + * The Unix timestamp (in seconds) for when the checkpoint was created. + */ + @JsonProperty("created_at") + Long createdAt; + + /** + * The name of the fine-tuned checkpoint model that is created. + */ + @JsonProperty("fine_tuned_model_checkpoint") + String fineTunedModelCheckpoint; + + /** + * The step number that the checkpoint was created at. + */ + @JsonProperty("step_number") + Integer stepNumber; + + /** + * Metrics at the step number during the fine-tuning job. + */ + Metrics metrics; + + /** + * The name of the fine-tuning job that this checkpoint was created from. + */ + @JsonProperty("fine_tuning_job_id") + String fineTuningJobId; + + /** + * The object type, which is always "fine_tuning.job.checkpoint". + */ + String object; + +} diff --git a/api/src/main/java/com/theokanning/openai/fine_tuning/FineTuningJobRequest.java b/api/src/main/java/com/theokanning/openai/fine_tuning/FineTuningJobRequest.java index 44a501c..c1d6182 100644 --- a/api/src/main/java/com/theokanning/openai/fine_tuning/FineTuningJobRequest.java +++ b/api/src/main/java/com/theokanning/openai/fine_tuning/FineTuningJobRequest.java @@ -3,6 +3,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import lombok.*; +import java.util.List; + /** * Request to create a fine tuning job @@ -13,13 +15,11 @@ @AllArgsConstructor @Data public class FineTuningJobRequest { - /** - * The ID of an uploaded file that contains training data. + * The name of the model to fine-tune. */ @NonNull - @JsonProperty("training_file") - String trainingFile; + String model; /** * The ID of an uploaded file that contains validation data. @@ -29,10 +29,11 @@ public class FineTuningJobRequest { String validationFile; /** - * The name of the model to fine-tune. + * The ID of an uploaded file that contains training data. */ @NonNull - String model; + @JsonProperty("training_file") + String trainingFile; /** * The hyperparameters used for the fine-tuning job. @@ -43,4 +44,12 @@ public class FineTuningJobRequest { * A string of up to 40 characters that will be added to your fine-tuned model name. */ String suffix; + + + List integrations; + + /** + * The seed controls the reproducibility of the job. Passing in the same seed and job parameters should produce the same results, but may differ in rare cases. If a seed is not specified, one will be generated for you. + */ + Integer seed; } diff --git a/api/src/main/java/com/theokanning/openai/fine_tuning/Hyperparameters.java b/api/src/main/java/com/theokanning/openai/fine_tuning/Hyperparameters.java index 3d59913..e685c07 100644 --- a/api/src/main/java/com/theokanning/openai/fine_tuning/Hyperparameters.java +++ b/api/src/main/java/com/theokanning/openai/fine_tuning/Hyperparameters.java @@ -16,7 +16,6 @@ @AllArgsConstructor @Data public class Hyperparameters { - /** * The number of epochs to train the model for. * An epoch refers to one full cycle through the training dataset. @@ -25,4 +24,16 @@ public class Hyperparameters { */ @JsonProperty("n_epochs") Integer nEpochs; + + /** + * Number of examples in each batch. A larger batch size means that model parameters are updated less frequently, but with lower variance. + */ + @JsonProperty("batch_size") + Integer batchSize; + + /** + * Scaling factor for the learning rate. A smaller learning rate may be useful to avoid overfitting. + */ + @JsonProperty("learning_rate_multiplier") + Double learningRateMultiplier; } diff --git a/api/src/main/java/com/theokanning/openai/fine_tuning/Integrations.java b/api/src/main/java/com/theokanning/openai/fine_tuning/Integrations.java new file mode 100644 index 0000000..c00bfdd --- /dev/null +++ b/api/src/main/java/com/theokanning/openai/fine_tuning/Integrations.java @@ -0,0 +1,27 @@ +package com.theokanning.openai.fine_tuning; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author LiangTao + * @date 2024年05月14 10:11 + **/ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Integrations { + /** + * The type of integration to enable. Currently, only "wandb" (Weights and Biases) is supported. + */ + private String type; + + /** + * The settings for your integration with Weights and Biases. This payload specifies the project that metrics will be sent to.
+ * Optionally, you can set an explicit display name for your run, add tags to your run, and set a default entity (team, username, etc) to be associated with your run. + */ + private Wandb wandb; +} diff --git a/api/src/main/java/com/theokanning/openai/fine_tuning/Metrics.java b/api/src/main/java/com/theokanning/openai/fine_tuning/Metrics.java new file mode 100644 index 0000000..fb770c0 --- /dev/null +++ b/api/src/main/java/com/theokanning/openai/fine_tuning/Metrics.java @@ -0,0 +1,31 @@ +package com.theokanning.openai.fine_tuning; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +/** + * @author LiangTao + * @date 2024年05月14 10:24 + **/ +@Data +public class Metrics { + Integer step; + + @JsonProperty("train_loss") + Double trainLoss; + + @JsonProperty("train_mean_token_accuracy") + Double trainMeanTokenAccuracy; + + @JsonProperty("valid_loss") + Double validLoss; + + @JsonProperty("valid_mean_token_accuracy") + Double validMeanTokenAccuracy; + + @JsonProperty("full_valid_loss") + Double fullValidLoss; + + @JsonProperty("full_valid_mean_token_accuracy") + Double fullValidMeanTokenAccuracy; +} diff --git a/api/src/main/java/com/theokanning/openai/fine_tuning/Wandb.java b/api/src/main/java/com/theokanning/openai/fine_tuning/Wandb.java new file mode 100644 index 0000000..ba7c8e5 --- /dev/null +++ b/api/src/main/java/com/theokanning/openai/fine_tuning/Wandb.java @@ -0,0 +1,49 @@ +package com.theokanning.openai.fine_tuning; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author LiangTao + * @date 2024年05月14 10:12 + **/ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class Wandb { + + /** + * The name of the project that the new run will be created under. + */ + private String project; + + + /** + * A display name to set for the run. If not set, we will use the Job ID as the name. + */ + private String name; + + /** + * run_id + */ + @JsonProperty("run_id") + private String runId; + + /** + * The entity to use for the run. This allows you to set the team or username of the WandB user that you would like associated with the run.
+ * If not set, the default entity for the registered WandB API key is used. + */ + private String entity; + + /** + * A list of tags to be attached to the newly created run.
+ * These tags are passed through directly to WandB. Some default tags are generated by OpenAI: "openai/finetune", "openai/{base-model}", "openai/{ftjob-abcdef}". + */ + private List tags; +} diff --git a/api/src/test/java/com/theokanning/openai/JsonTest.java b/api/src/test/java/com/theokanning/openai/JsonTest.java index 5f75728..f087444 100644 --- a/api/src/test/java/com/theokanning/openai/JsonTest.java +++ b/api/src/test/java/com/theokanning/openai/JsonTest.java @@ -37,6 +37,7 @@ import com.theokanning.openai.file.File; import com.theokanning.openai.fine_tuning.FineTuningEvent; import com.theokanning.openai.fine_tuning.FineTuningJob; +import com.theokanning.openai.fine_tuning.FineTuningJobCheckpoint; import com.theokanning.openai.fine_tuning.FineTuningJobRequest; import com.theokanning.openai.image.ImageResult; import com.theokanning.openai.model.Model; @@ -66,6 +67,7 @@ public class JsonTest { FineTuningEvent.class, FineTuningJob.class, FineTuningJobRequest.class, + FineTuningJobCheckpoint.class, ImageResult.class, TranscriptionResult.class, TranslationResult.class, diff --git a/api/src/test/resources/fixtures/FineTuningJob.json b/api/src/test/resources/fixtures/FineTuningJob.json index 51eb64c..910a950 100644 --- a/api/src/test/resources/fixtures/FineTuningJob.json +++ b/api/src/test/resources/fixtures/FineTuningJob.json @@ -15,5 +15,14 @@ "file-abc123" ], "validation_file": "validation-file", - "trained_tokens": 5768 + "trained_tokens": 5768, + "integrations": [ + { + "type": "wandb", + "wandb": { + "project": "my-wandb-project", + "run_id": "ftjob-abc123" + } + } + ] } diff --git a/api/src/test/resources/fixtures/FineTuningJobCheckpoint.json b/api/src/test/resources/fixtures/FineTuningJobCheckpoint.json new file mode 100644 index 0000000..9ddf293 --- /dev/null +++ b/api/src/test/resources/fixtures/FineTuningJobCheckpoint.json @@ -0,0 +1,17 @@ +{ + "object": "fine_tuning.job.checkpoint", + "id": "ftckpt_qtZ5Gyk4BLq1SfLFWp3RtO3P", + "created_at": 1712211699, + "fine_tuned_model_checkpoint": "ft:gpt-3.5-turbo-0125:my-org:custom_suffix:9ABel2dg:ckpt-step-88", + "fine_tuning_job_id": "ftjob-fpbNQ3H1GrMehXRf8cO97xTN", + "metrics": { + "step": 88, + "train_loss": 0.478, + "train_mean_token_accuracy": 0.924, + "valid_loss": 10.112, + "valid_mean_token_accuracy": 0.145, + "full_valid_loss": 0.567, + "full_valid_mean_token_accuracy": 0.944 + }, + "step_number": 88 +} diff --git a/api/src/test/resources/fixtures/FineTuningJobRequest.json b/api/src/test/resources/fixtures/FineTuningJobRequest.json index 45e79f7..5405141 100644 --- a/api/src/test/resources/fixtures/FineTuningJobRequest.json +++ b/api/src/test/resources/fixtures/FineTuningJobRequest.json @@ -3,7 +3,22 @@ "validation_file": "file-abc123", "training_file": "file-abc123", "hyperparameters": { - "n_epochs": 4 + "n_epochs": 4, + "batch_size": 1, + "learning_rate_multiplier": 1.0 }, - "suffix": "test" + "suffix": "test", + "integrations": [ + { + "type": "wandb", + "wandb": { + "project": "my-wandb-project", + "name": "ft-run-display-name", + "tags": [ + "first-experiment", + "v2" + ] + } + } + ] } diff --git a/client/src/main/java/com/theokanning/openai/client/OpenAiApi.java b/client/src/main/java/com/theokanning/openai/client/OpenAiApi.java index 593fd8f..e920c40 100644 --- a/client/src/main/java/com/theokanning/openai/client/OpenAiApi.java +++ b/client/src/main/java/com/theokanning/openai/client/OpenAiApi.java @@ -37,6 +37,7 @@ import com.theokanning.openai.file.File; import com.theokanning.openai.fine_tuning.FineTuningEvent; import com.theokanning.openai.fine_tuning.FineTuningJob; +import com.theokanning.openai.fine_tuning.FineTuningJobCheckpoint; import com.theokanning.openai.fine_tuning.FineTuningJobRequest; import com.theokanning.openai.image.CreateImageRequest; import com.theokanning.openai.image.ImageResult; @@ -118,6 +119,10 @@ public interface OpenAiApi { @GET("fine_tuning/jobs/{fine_tuning_job_id}/events") Single> listFineTuningJobEvents(@Path("fine_tuning_job_id") String fineTuningJobId); + //List fine-tuning checkpoints + @GET("fine_tuning/jobs/{fine_tuning_job_id}/checkpoints") + Single> listFineTuningCheckpoints(@Path("fine_tuning_job_id") String fineTuningJobId); + @DELETE("models/{fine_tune_id}") Single deleteFineTune(@Path("fine_tune_id") String fineTuneId); diff --git a/service/src/main/java/com/theokanning/openai/service/OpenAiService.java b/service/src/main/java/com/theokanning/openai/service/OpenAiService.java index 5b7814c..6f2aef7 100644 --- a/service/src/main/java/com/theokanning/openai/service/OpenAiService.java +++ b/service/src/main/java/com/theokanning/openai/service/OpenAiService.java @@ -42,6 +42,7 @@ import com.theokanning.openai.file.File; import com.theokanning.openai.fine_tuning.FineTuningEvent; import com.theokanning.openai.fine_tuning.FineTuningJob; +import com.theokanning.openai.fine_tuning.FineTuningJobCheckpoint; import com.theokanning.openai.fine_tuning.FineTuningJobRequest; import com.theokanning.openai.image.CreateImageEditRequest; import com.theokanning.openai.image.CreateImageRequest; @@ -242,6 +243,10 @@ public List listFineTuningJobEvents(String fineTuningJobId) { return execute(api.listFineTuningJobEvents(fineTuningJobId)).data; } + public List listFineTuningCheckpoints(String fineTuningJobId) { + return execute(api.listFineTuningCheckpoints(fineTuningJobId)).data; + } + @Deprecated public Flowable streamCompletion(CompletionRequest request) { request.setStream(true);