Skip to content

fix: give user provided checksum precondition priority for Storage#create methods that accept byte[] #3182

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
BenWhitehead wants to merge 2 commits into from
Closed

fix: give user provided checksum precondition priority for Storage#create methods that accept byte[] #3182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
503a373
fix: give user provided checksum precondition priority for Storage#cr…
BenWhitehead Jun 30, 2025
bd27b4b
test: remove obsolete mockito tests that don't test anything unique
BenWhitehead Jun 30, 2025
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -804,7 +804,7 @@ public BlobInfo internalDirectUpload(
GrpcCallContext grpcCallContext =
optsWithDefaults.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
WriteObjectRequest req = getWriteObjectRequest(blobInfo, optsWithDefaults);
Hasher hasher = Hasher.enabled();
Hasher hasher = opts.getHasher();
GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
RewindableContent content = RewindableContent.of(buf);
return retrier.run(
Expand Down
82 changes: 15 additions & 67 deletions google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import com.google.cloud.storage.BlobReadChannelV2.BlobReadChannelContext;
import com.google.cloud.storage.BlobWriteSessionConfig.WriterFactory;
import com.google.cloud.storage.Conversions.Decoder;
import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown;
import com.google.cloud.storage.HmacKey.HmacKeyMetadata;
import com.google.cloud.storage.PostPolicyV4.ConditionV4Type;
import com.google.cloud.storage.PostPolicyV4.PostConditionsV4;
Expand All @@ -55,18 +56,15 @@
import com.google.cloud.storage.spi.v1.StorageRpc;
import com.google.cloud.storage.spi.v1.StorageRpc.RewriteRequest;
import com.google.common.base.CharMatcher;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;
import com.google.common.io.CountingOutputStream;
import com.google.common.primitives.Ints;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
Expand Down Expand Up @@ -97,7 +95,6 @@
import java.util.concurrent.TimeoutException;
import java.util.function.Supplier;
import java.util.stream.Stream;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;

final class StorageImpl extends BaseService<StorageOptions> implements Storage, StorageInternal {
Expand Down Expand Up @@ -152,46 +149,23 @@ public Bucket create(BucketInfo bucketInfo, BucketTargetOption... options) {

@Override
public Blob create(BlobInfo blobInfo, BlobTargetOption... options) {
BlobInfo updatedInfo =
blobInfo.toBuilder()
.setMd5(EMPTY_BYTE_ARRAY_MD5)
.setCrc32c(EMPTY_BYTE_ARRAY_CRC32C)
.build();
final Opts<ObjectTargetOpt> objectTargetOptOpts = Opts.unwrap(options).resolveFrom(updatedInfo);
return internalCreate(updatedInfo, EMPTY_BYTE_ARRAY, 0, 0, objectTargetOptOpts);
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blobInfo);
return internalDirectUpload(blobInfo, opts, Buffers.allocate(0)).asBlob(this);
}

@Override
public Blob create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options) {
content = firstNonNull(content, EMPTY_BYTE_ARRAY);
BlobInfo updatedInfo =
blobInfo.toBuilder()
.setMd5(BaseEncoding.base64().encode(Hashing.md5().hashBytes(content).asBytes()))
.setCrc32c(
BaseEncoding.base64()
.encode(Ints.toByteArray(Hashing.crc32c().hashBytes(content).asInt())))
.build();
final Opts<ObjectTargetOpt> objectTargetOptOpts = Opts.unwrap(options).resolveFrom(updatedInfo);
return internalCreate(updatedInfo, content, 0, content.length, objectTargetOptOpts);
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blobInfo);
return internalDirectUpload(blobInfo, opts, ByteBuffer.wrap(content)).asBlob(this);
}

@Override
public Blob create(
BlobInfo blobInfo, byte[] content, int offset, int length, BlobTargetOption... options) {
content = firstNonNull(content, EMPTY_BYTE_ARRAY);
BlobInfo updatedInfo =
blobInfo.toBuilder()
.setMd5(
BaseEncoding.base64()
.encode(Hashing.md5().hashBytes(content, offset, length).asBytes()))
.setCrc32c(
BaseEncoding.base64()
.encode(
Ints.toByteArray(
Hashing.crc32c().hashBytes(content, offset, length).asInt())))
.build();
final Opts<ObjectTargetOpt> objectTargetOptOpts = Opts.unwrap(options).resolveFrom(updatedInfo);
return internalCreate(updatedInfo, content, offset, length, objectTargetOptOpts);
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blobInfo);
return internalDirectUpload(blobInfo, opts, ByteBuffer.wrap(content, offset, length))
.asBlob(this);
}

@Override
Expand All @@ -212,30 +186,6 @@ public Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... op
return info.asBlob(this);
}

private Blob internalCreate(
BlobInfo info,
final byte @NonNull [] content,
final int offset,
final int length,
Opts<ObjectTargetOpt> opts) {
Preconditions.checkNotNull(content);
final Map<StorageRpc.Option, ?> optionsMap = opts.getRpcOptions();

BlobInfo updated = opts.blobInfoMapper().apply(info.toBuilder()).build();
final StorageObject blobPb = codecs.blobInfo().encode(updated);
ResultRetryAlgorithm<?> algorithm =
retryAlgorithmManager.getForObjectsCreate(blobPb, optionsMap);
return run(
algorithm,
() ->
storageRpc.create(
blobPb, new ByteArrayInputStream(content, offset, length), optionsMap),
(x) -> {
BlobInfo info1 = Conversions.json().blobInfo().decode(x);
return info1.asBlob(this);
});
}

@Override
public Blob createFrom(BlobInfo blobInfo, Path path, BlobWriteOption... options)
throws IOException {
Expand Down Expand Up @@ -1731,7 +1681,7 @@ public BlobInfo internalCreateFrom(Path path, BlobInfo info, Opts<ObjectTargetOp
}
long size = Files.size(path);
if (size == 0L) {
return internalCreate(info, EMPTY_BYTE_ARRAY, 0, 0, opts);
return internalDirectUpload(info, opts, ByteBuffer.allocate(0));
}
final Map<StorageRpc.Option, ?> optionsMap = opts.getRpcOptions();
BlobInfo.Builder builder = info.toBuilder().setMd5(null).setCrc32c(null);
Expand Down Expand Up @@ -1767,17 +1717,15 @@ public BlobInfo internalCreateFrom(Path path, BlobInfo info, Opts<ObjectTargetOp

@Override
public BlobInfo internalDirectUpload(BlobInfo info, Opts<ObjectTargetOpt> opts, ByteBuffer buf) {

BlobInfo.Builder builder =
info.toBuilder()
.setMd5(
BaseEncoding.base64().encode(Hashing.md5().hashBytes(buf.duplicate()).asBytes()))
.setCrc32c(
BaseEncoding.base64()
.encode(Ints.toByteArray(Hashing.crc32c().hashBytes(buf.duplicate()).asInt())));
opts.blobInfoMapper().apply(info.toBuilder().clearMd5().clearCrc32c());
@Nullable Crc32cLengthKnown hash = opts.getHasher().hash(buf.duplicate());
Comment thread
BrandonY marked this conversation as resolved.
if (hash != null) {
builder.setCrc32c(Utils.crc32cCodec.encode(hash.getValue()));
}
final Map<StorageRpc.Option, ?> optionsMap = opts.getRpcOptions();

BlobInfo updated = opts.blobInfoMapper().apply(builder).build();
BlobInfo updated = builder.build();
final StorageObject encoded = codecs.blobInfo().encode(updated);
ResultRetryAlgorithm<?> algorithm =
retryAlgorithmManager.getForObjectsCreate(encoded, optionsMap);
Expand Down
Loading