这是indexloc提供的服务,不要输入任何密码
Skip to content

feat: add Storage.BlobListOption#includeTrailingDelimiter #3038

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 24, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2677,6 +2677,17 @@ public static BlobListOption includeFolders(boolean includeFolders) {
return new BlobListOption(UnifiedOpts.includeFoldersAsPrefixes(includeFolders));
}

/**
* Returns an option which will cause blobs that end in exactly one instance of `delimiter` will
* have their metadata included rather than being synthetic objects.
*
* @since 2.52.0
*/
@TransportCompatibility({Transport.HTTP, Transport.GRPC})
public static BlobListOption includeTrailingDelimiter() {
return new BlobListOption(UnifiedOpts.includeTrailingDelimiter());
}

/**
* Returns an option to define the billing user project. This option is required by buckets with
* `requester_pays` flag enabled to assign operation costs.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,10 @@ static IncludeFoldersAsPrefixes includeFoldersAsPrefixes(boolean includeFoldersA
return new IncludeFoldersAsPrefixes(includeFoldersAsPrefixes);
}

static IncludeTrailingDelimiter includeTrailingDelimiter() {
return new IncludeTrailingDelimiter(true);
}

@Deprecated
static DetectContentType detectContentType() {
return DetectContentType.INSTANCE;
Expand Down Expand Up @@ -706,6 +710,20 @@ public Mapper<ListObjectsRequest.Builder> listObjects() {
}
}

static final class IncludeTrailingDelimiter extends RpcOptVal<Boolean> implements ObjectListOpt {

private static final long serialVersionUID = 321916692864878282L;

private IncludeTrailingDelimiter(boolean val) {
super(StorageRpc.Option.INCLUDE_TRAILING_DELIMITER, val);
}

@Override
public Mapper<ListObjectsRequest.Builder> listObjects() {
return b -> b.setIncludeTrailingDelimiter(val);
}
}

static final class Delimiter extends RpcOptVal<String> implements ObjectListOpt {
private static final long serialVersionUID = -3789556789947615714L;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -496,7 +496,8 @@ public Tuple<String, Iterable<StorageObject>> list(final String bucket, Map<Opti
.setFields(Option.FIELDS.getString(options))
.setUserProject(Option.USER_PROJECT.getString(options))
.setSoftDeleted(Option.SOFT_DELETED.getBoolean(options))
.setIncludeFoldersAsPrefixes(Option.INCLUDE_FOLDERS_AS_PREFIXES.getBoolean(options));
.setIncludeFoldersAsPrefixes(Option.INCLUDE_FOLDERS_AS_PREFIXES.getBoolean(options))
.setIncludeTrailingDelimiter(Option.INCLUDE_TRAILING_DELIMITER.getBoolean(options));
setExtraHeaders(list, options);
Objects objects = list.execute();
Iterable<StorageObject> storageObjects =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ enum Option {
COPY_SOURCE_ACL("copySourceAcl"),
GENERATION("generation"),
INCLUDE_FOLDERS_AS_PREFIXES("includeFoldersAsPrefixes"),
INCLUDE_TRAILING_DELIMITER("includeTrailingDelimiter"),
X_UPLOAD_CONTENT_LENGTH("x-upload-content-length"),
/**
* An {@link com.google.common.collect.ImmutableMap ImmutableMap&lt;String, String>} of values
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1607,4 +1607,86 @@ public void blob_update() throws Exception {
() -> assertThat(gen1_2.getMetadata()).isEqualTo(meta3),
() -> assertThat(gen1_2.getGeneration()).isEqualTo(gen1.getGeneration()));
}

@Test
public void listBlob_includeTrailingDelimiter() throws Exception {
final byte[] A = new byte[] {(byte) 'A'};

String basePath = generator.randomObjectName();
// create a series of objects under a stable test specific path
BlobId a = BlobId.of(bucket.getName(), String.format("%s/a", basePath));
BlobId b = BlobId.of(bucket.getName(), String.format("%s/b", basePath));
BlobId c = BlobId.of(bucket.getName(), String.format("%s/c", basePath));
BlobId a_ = BlobId.of(bucket.getName(), String.format("%s/a/", basePath));
BlobId b_ = BlobId.of(bucket.getName(), String.format("%s/b/", basePath));
BlobId c_ = BlobId.of(bucket.getName(), String.format("%s/c/", basePath));
BlobId d_ = BlobId.of(bucket.getName(), String.format("%s/d/", basePath));
BlobId a_A1 = BlobId.of(bucket.getName(), String.format("%s/a/A1", basePath));
BlobId a_A2 = BlobId.of(bucket.getName(), String.format("%s/a/A2", basePath));
BlobId b_B1 = BlobId.of(bucket.getName(), String.format("%s/b/B1", basePath));
BlobId c_C2 = BlobId.of(bucket.getName(), String.format("%s/c/C2", basePath));

storage.create(BlobInfo.newBuilder(a).build(), A, BlobTargetOption.doesNotExist());
storage.create(BlobInfo.newBuilder(b).build(), A, BlobTargetOption.doesNotExist());
storage.create(BlobInfo.newBuilder(c).build(), A, BlobTargetOption.doesNotExist());
storage.create(BlobInfo.newBuilder(a_).build(), A, BlobTargetOption.doesNotExist());
storage.create(BlobInfo.newBuilder(b_).build(), A, BlobTargetOption.doesNotExist());
storage.create(BlobInfo.newBuilder(c_).build(), A, BlobTargetOption.doesNotExist());
storage.create(BlobInfo.newBuilder(d_).build(), A, BlobTargetOption.doesNotExist());
storage.create(BlobInfo.newBuilder(a_A1).build(), A, BlobTargetOption.doesNotExist());
storage.create(BlobInfo.newBuilder(a_A2).build(), A, BlobTargetOption.doesNotExist());
storage.create(BlobInfo.newBuilder(b_B1).build(), A, BlobTargetOption.doesNotExist());
storage.create(BlobInfo.newBuilder(c_C2).build(), A, BlobTargetOption.doesNotExist());

// define all our options
BlobListOption[] blobListOptions =
new BlobListOption[] {
BlobListOption.currentDirectory(),
BlobListOption.includeTrailingDelimiter(),
BlobListOption.fields(BlobField.NAME, BlobField.GENERATION, BlobField.SIZE),
BlobListOption.prefix(basePath + "/")
};
// list and collect all the object names
List<Blob> blobs =
storage.list(bucket.getName(), blobListOptions).streamAll().collect(Collectors.toList());

// figure out what the base prefix of the objects is, so we can trim it down to make assertions
// more terse.
int trimLen = String.format(Locale.US, "gs://%s/%s", bucket.getName(), basePath).length();
List<String> names =
blobs.stream()
.map(
bi -> {
String uri = bi.getBlobId().toGsUtilUriWithGeneration();
int genIdx = uri.indexOf("#");
String substring;
if (genIdx > -1) {
// trim the string representation of the generation to make assertions easier.
// We really only need to know that a generation is present, not it's exact
// value.
substring = uri.substring(trimLen, genIdx + 1);
} else {
substring = uri.substring(trimLen);
}
return "..." + substring;
})
.collect(Collectors.toList());

assertThat(names)
.containsExactly(
// items
".../a#",
".../b#",
".../c#",
// items included due to includeTrailingDelimiter
".../a/#",
".../b/#",
".../c/#",
".../d/#",
// prefixes
".../a/",
".../b/",
".../c/",
".../d/");
}
}