From 129455b4290dfc75349f5529d167809df40be783 Mon Sep 17 00:00:00 2001 From: Pulkit Aggarwal <54775856+Pulkit0110@users.noreply.github.com> Date: Mon, 23 Jun 2025 15:55:50 +0530 Subject: [PATCH 1/8] chore: add owlbot for gapic generation (#1492) * chore: add owlbot for gapic generation * use only v2 for copying * use complete path for storage-v2-py --- .github/.OwlBot.yaml | 7 +++++++ .repo-metadata.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml index 27096f674..837bf4fd6 100644 --- a/.github/.OwlBot.yaml +++ b/.github/.OwlBot.yaml @@ -15,5 +15,12 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest +deep-remove-regex: + - /owl-bot-staging + +deep-copy-regex: + - source: /google/storage/v2/storage-v2-py/(.*) + dest: /owl-bot-staging/$1/$2 + begin-after-commit-hash: 6acf4a0a797f1082027985c55c4b14b60f673dd7 diff --git a/.repo-metadata.json b/.repo-metadata.json index 5d5e49c84..f644429bc 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -11,7 +11,7 @@ "distribution_name": "google-cloud-storage", "api_id": "storage.googleapis.com", "requires_billing": true, - "default_version": "", + "default_version": "v2", "codeowner_team": "@googleapis/gcs-sdk-team", "api_shortname": "storage", "api_description": "is a durable and highly available object storage service. Google Cloud Storage is almost infinitely scalable and guarantees consistency: when a write succeeds, the latest copy of the object will be returned to any GET, globally." From 5821134d6cb86cdd73f4dcf4cdb4f3c0a9e5ddd2 Mon Sep 17 00:00:00 2001 From: Pulkit Aggarwal <54775856+Pulkit0110@users.noreply.github.com> Date: Fri, 27 Jun 2025 11:23:16 +0530 Subject: [PATCH 2/8] chore: add configurations in owlbot.py to copy gapic code (#1494) --- .gitignore | 3 +++ owlbot.py | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/.gitignore b/.gitignore index d083ea1dd..6f2ab1aff 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,6 @@ system_tests/local_test_setup # Make sure a generated file isn't accidentally committed. pylintrc pylintrc.test + +# Docker files +get-docker.sh diff --git a/owlbot.py b/owlbot.py index 08ddbb8fc..ba3f32df3 100644 --- a/owlbot.py +++ b/owlbot.py @@ -14,10 +14,33 @@ """This script is used to synthesize generated parts of this library.""" +import json + import synthtool as s from synthtool import gcp from synthtool.languages import python +# ---------------------------------------------------------------------------- +# Copy the generated client from the owl-bot staging directory +# ---------------------------------------------------------------------------- + +# Load the default version defined in .repo-metadata.json. +default_version = json.load(open(".repo-metadata.json", "rt")).get("default_version") + +for library in s.get_staging_dirs(default_version): + s.move( + [library], + excludes=[ + "**/gapic_version.py", + "docs/**/*", + "scripts/fixup*.py", + "setup.py", + "noxfile.py", + "README.rst", + ], + ) +s.remove_staging_dirs() + common = gcp.CommonTemplates() # ---------------------------------------------------------------------------- From 61c5d5f62c88506f200bc6d86b399a2c28715bc4 Mon Sep 17 00:00:00 2001 From: shubham-up-47 Date: Tue, 1 Jul 2025 04:26:52 +0000 Subject: [PATCH 3/8] feat: Adding support of single shot download (#1493) --- google/cloud/storage/_media/_download.py | 7 +- .../cloud/storage/_media/requests/download.py | 47 +++++++---- google/cloud/storage/blob.py | 61 ++++++++++++++ google/cloud/storage/client.py | 5 ++ google/cloud/storage/fileio.py | 8 +- .../system/requests/test_download.py | 40 ++++++++++ .../unit/requests/test_download.py | 39 +++++++++ tests/system/test_blob.py | 19 +++++ tests/unit/test_blob.py | 79 +++++++++++++++++-- tests/unit/test_client.py | 3 + 10 files changed, 285 insertions(+), 23 deletions(-) diff --git a/google/cloud/storage/_media/_download.py b/google/cloud/storage/_media/_download.py index 349ddf30c..422b98041 100644 --- a/google/cloud/storage/_media/_download.py +++ b/google/cloud/storage/_media/_download.py @@ -140,7 +140,7 @@ class Download(DownloadBase): ``start`` to the end of the media. headers (Optional[Mapping[str, str]]): Extra headers that should be sent with the request, e.g. headers for encrypted data. - checksum Optional([str]): The type of checksum to compute to verify + checksum (Optional[str]): The type of checksum to compute to verify the integrity of the object. The response headers must contain a checksum of the requested type. If the headers lack an appropriate checksum (for instance in the case of transcoded or @@ -157,6 +157,9 @@ class Download(DownloadBase): See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for information on retry types and how to configure them. + single_shot_download (Optional[bool]): If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your use case. """ @@ -169,6 +172,7 @@ def __init__( headers=None, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ): super(Download, self).__init__( media_url, stream=stream, start=start, end=end, headers=headers, retry=retry @@ -178,6 +182,7 @@ def __init__( self.checksum = ( "crc32c" if _helpers._is_crc32c_available_and_fast() else "md5" ) + self.single_shot_download = single_shot_download self._bytes_downloaded = 0 self._expected_checksum = None self._checksum_object = None diff --git a/google/cloud/storage/_media/requests/download.py b/google/cloud/storage/_media/requests/download.py index 67535f923..b8e2758e1 100644 --- a/google/cloud/storage/_media/requests/download.py +++ b/google/cloud/storage/_media/requests/download.py @@ -132,13 +132,24 @@ def _write_to_stream(self, response): # the stream is indeed compressed, this will delegate the checksum # object to the decoder and return a _DoNothingHash here. local_checksum_object = _add_decoder(response.raw, checksum_object) - body_iter = response.iter_content( - chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False - ) - for chunk in body_iter: - self._stream.write(chunk) - self._bytes_downloaded += len(chunk) - local_checksum_object.update(chunk) + + # This is useful for smaller files, or when the user wants to + # download the entire file in one go. + if self.single_shot_download: + content = response.raw.read(decode_content=True) + self._stream.write(content) + self._bytes_downloaded += len(content) + local_checksum_object.update(content) + response._content_consumed = True + else: + body_iter = response.iter_content( + chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, + decode_unicode=False, + ) + for chunk in body_iter: + self._stream.write(chunk) + self._bytes_downloaded += len(chunk) + local_checksum_object.update(chunk) # Don't validate the checksum for partial responses. if ( @@ -345,13 +356,21 @@ def _write_to_stream(self, response): checksum_object = self._checksum_object with response: - body_iter = response.raw.stream( - _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False - ) - for chunk in body_iter: - self._stream.write(chunk) - self._bytes_downloaded += len(chunk) - checksum_object.update(chunk) + # This is useful for smaller files, or when the user wants to + # download the entire file in one go. + if self.single_shot_download: + content = response.raw.read() + self._stream.write(content) + self._bytes_downloaded += len(content) + checksum_object.update(content) + else: + body_iter = response.raw.stream( + _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False + ) + for chunk in body_iter: + self._stream.write(chunk) + self._bytes_downloaded += len(chunk) + checksum_object.update(chunk) response._content_consumed = True # Don't validate the checksum for partial responses. diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 0d0e8ee80..0eb94fd47 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -987,6 +987,7 @@ def _do_download( timeout=_DEFAULT_TIMEOUT, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ): """Perform a download without any error handling. @@ -1047,6 +1048,12 @@ def _do_download( See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for information on retry types and how to configure them. + + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your use case. """ extra_attributes = { @@ -1054,6 +1061,7 @@ def _do_download( "download.chunk_size": f"{self.chunk_size}", "download.raw_download": raw_download, "upload.checksum": f"{checksum}", + "download.single_shot_download": single_shot_download, } args = {"timeout": timeout} @@ -1073,6 +1081,10 @@ def _do_download( end=end, checksum=checksum, retry=retry, + # NOTE: single_shot_download is only supported in Download and RawDownload + # classes, i.e., when chunk_size is set to None (the default value). It is + # not supported for chunked downloads. + single_shot_download=single_shot_download, ) with create_trace_span( name=f"Storage.{download_class}/consume", @@ -1127,6 +1139,7 @@ def download_to_file( timeout=_DEFAULT_TIMEOUT, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ): """Download the contents of this blob into a file-like object. @@ -1222,6 +1235,12 @@ def download_to_file( (google.cloud.storage.retry) for information on retry types and how to configure them. + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your use case. + :raises: :class:`google.cloud.exceptions.NotFound` """ with create_trace_span(name="Storage.Blob.downloadToFile"): @@ -1240,6 +1259,7 @@ def download_to_file( timeout=timeout, checksum=checksum, retry=retry, + single_shot_download=single_shot_download, ) def _handle_filename_and_download(self, filename, *args, **kwargs): @@ -1285,6 +1305,7 @@ def download_to_filename( timeout=_DEFAULT_TIMEOUT, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ): """Download the contents of this blob into a named file. @@ -1370,6 +1391,12 @@ def download_to_filename( (google.cloud.storage.retry) for information on retry types and how to configure them. + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your use case. + :raises: :class:`google.cloud.exceptions.NotFound` """ with create_trace_span(name="Storage.Blob.downloadToFilename"): @@ -1388,6 +1415,7 @@ def download_to_filename( timeout=timeout, checksum=checksum, retry=retry, + single_shot_download=single_shot_download, ) def download_as_bytes( @@ -1405,6 +1433,7 @@ def download_as_bytes( timeout=_DEFAULT_TIMEOUT, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ): """Download the contents of this blob as a bytes object. @@ -1484,6 +1513,12 @@ def download_as_bytes( (google.cloud.storage.retry) for information on retry types and how to configure them. + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your use case. + :rtype: bytes :returns: The data stored in this blob. @@ -1507,6 +1542,7 @@ def download_as_bytes( timeout=timeout, checksum=checksum, retry=retry, + single_shot_download=single_shot_download, ) return string_buffer.getvalue() @@ -1524,6 +1560,7 @@ def download_as_string( if_metageneration_not_match=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY, + single_shot_download=False, ): """(Deprecated) Download the contents of this blob as a bytes object. @@ -1594,6 +1631,12 @@ def download_as_string( (google.cloud.storage.retry) for information on retry types and how to configure them. + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your use case. + :rtype: bytes :returns: The data stored in this blob. @@ -1616,6 +1659,7 @@ def download_as_string( if_metageneration_not_match=if_metageneration_not_match, timeout=timeout, retry=retry, + single_shot_download=single_shot_download, ) def download_as_text( @@ -1633,6 +1677,7 @@ def download_as_text( if_metageneration_not_match=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY, + single_shot_download=False, ): """Download the contents of this blob as text (*not* bytes). @@ -1705,6 +1750,12 @@ def download_as_text( (google.cloud.storage.retry) for information on retry types and how to configure them. + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your use case. + :rtype: text :returns: The data stored in this blob, decoded to text. """ @@ -1722,6 +1773,7 @@ def download_as_text( if_metageneration_not_match=if_metageneration_not_match, timeout=timeout, retry=retry, + single_shot_download=single_shot_download, ) if encoding is not None: @@ -4019,6 +4071,7 @@ def open( For downloads only, the following additional arguments are supported: - ``raw_download`` + - ``single_shot_download`` For uploads only, the following additional arguments are supported: @@ -4209,6 +4262,7 @@ def _prep_and_do_download( timeout=_DEFAULT_TIMEOUT, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, command=None, ): """Download the contents of a blob object into a file-like object. @@ -4294,6 +4348,12 @@ def _prep_and_do_download( (google.cloud.storage.retry) for information on retry types and how to configure them. + :type single_shot_download: bool + :param single_shot_download: + (Optional) If true, download the object in a single request. + Caution: Enabling this will increase the memory overload for your application. + Please enable this as per your use case. + :type command: str :param command: (Optional) Information about which interface for download was used, @@ -4349,6 +4409,7 @@ def _prep_and_do_download( timeout=timeout, checksum=checksum, retry=retry, + single_shot_download=single_shot_download, ) except InvalidResponse as exc: _raise_from_invalid_response(exc) diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index ba94b26fc..2f56d8719 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -1143,6 +1143,7 @@ def download_blob_to_file( timeout=_DEFAULT_TIMEOUT, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ): """Download the contents of a blob object or blob URI into a file-like object. @@ -1216,6 +1217,9 @@ def download_blob_to_file( See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for information on retry types and how to configure them. + + single_shot_download (bool): + (Optional) If true, download the object in a single request. """ with create_trace_span(name="Storage.Client.downloadBlobToFile"): if not isinstance(blob_or_uri, Blob): @@ -1236,6 +1240,7 @@ def download_blob_to_file( timeout=timeout, checksum=checksum, retry=retry, + single_shot_download=single_shot_download, ) def list_blobs( diff --git a/google/cloud/storage/fileio.py b/google/cloud/storage/fileio.py index 289a09cee..7c30f39be 100644 --- a/google/cloud/storage/fileio.py +++ b/google/cloud/storage/fileio.py @@ -35,6 +35,7 @@ "timeout", "retry", "raw_download", + "single_shot_download", } # Valid keyword arguments for upload methods. @@ -99,8 +100,9 @@ class BlobReader(io.BufferedIOBase): - ``if_metageneration_not_match`` - ``timeout`` - ``raw_download`` + - ``single_shot_download`` - Note that download_kwargs (excluding ``raw_download``) are also applied to blob.reload(), + Note that download_kwargs (excluding ``raw_download`` and ``single_shot_download``) are also applied to blob.reload(), if a reload is needed during seek(). """ @@ -177,7 +179,9 @@ def seek(self, pos, whence=0): if self._blob.size is None: reload_kwargs = { - k: v for k, v in self._download_kwargs.items() if k != "raw_download" + k: v + for k, v in self._download_kwargs.items() + if (k != "raw_download" and k != "single_shot_download") } self._blob.reload(**reload_kwargs) diff --git a/tests/resumable_media/system/requests/test_download.py b/tests/resumable_media/system/requests/test_download.py index 04c7246f6..84c44c94c 100644 --- a/tests/resumable_media/system/requests/test_download.py +++ b/tests/resumable_media/system/requests/test_download.py @@ -286,6 +286,23 @@ def test_download_full(self, add_files, authorized_transport, checksum): assert self._read_response_content(response) == actual_contents check_tombstoned(download, authorized_transport) + @pytest.mark.parametrize("checksum", ["auto", "md5", "crc32c", None]) + def test_single_shot_download_full(self, add_files, authorized_transport, checksum): + for info in ALL_FILES: + actual_contents = self._get_contents(info) + blob_name = get_blob_name(info) + + # Create the actual download object. + media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) + download = self._make_one( + media_url, checksum=checksum, single_shot_download=True + ) + # Consume the resource with single_shot_download enabled. + response = download.consume(authorized_transport) + assert response.status_code == http.client.OK + assert self._read_response_content(response) == actual_contents + check_tombstoned(download, authorized_transport) + def test_download_to_stream(self, add_files, authorized_transport): for info in ALL_FILES: actual_contents = self._get_contents(info) @@ -306,6 +323,29 @@ def test_download_to_stream(self, add_files, authorized_transport): assert stream.getvalue() == actual_contents check_tombstoned(download, authorized_transport) + @pytest.mark.parametrize("checksum", ["auto", "md5", "crc32c", None]) + def test_single_shot_download_to_stream(self, add_files, authorized_transport, checksum): + for info in ALL_FILES: + actual_contents = self._get_contents(info) + blob_name = get_blob_name(info) + + # Create the actual download object. + media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name) + stream = io.BytesIO() + download = self._make_one( + media_url, checksum=checksum, stream=stream, single_shot_download=True + ) + # Consume the resource with single_shot_download enabled. + response = download.consume(authorized_transport) + assert response.status_code == http.client.OK + with pytest.raises(RuntimeError) as exc_info: + getattr(response, "content") + assert exc_info.value.args == (NO_BODY_ERR,) + assert response._content is False + assert response._content_consumed is True + assert stream.getvalue() == actual_contents + check_tombstoned(download, authorized_transport) + def test_download_gzip_w_stored_content_headers( self, add_files, authorized_transport ): diff --git a/tests/resumable_media/unit/requests/test_download.py b/tests/resumable_media/unit/requests/test_download.py index 568d3238c..b17fbb905 100644 --- a/tests/resumable_media/unit/requests/test_download.py +++ b/tests/resumable_media/unit/requests/test_download.py @@ -213,6 +213,25 @@ def test__write_to_stream_incomplete_read(self, checksum): in error.args[0] ) + @pytest.mark.parametrize("checksum", ["auto", "md5", "crc32c", None]) + def test__write_to_stream_single_shot_download(self, checksum): + stream = io.BytesIO() + download = download_mod.Download( + EXAMPLE_URL, stream=stream, checksum=checksum, single_shot_download=True + ) + + chunk1 = b"all at once!" + response = _mock_response(chunks=[chunk1], headers={}) + ret_val = download._write_to_stream(response) + + assert ret_val is None + assert stream.getvalue() == chunk1 + assert download._bytes_downloaded == len(chunk1) + + response.__enter__.assert_called_once_with() + response.__exit__.assert_called_once_with(None, None, None) + response.raw.read.assert_called_once_with(decode_content=True) + def _consume_helper( self, stream=None, @@ -692,6 +711,24 @@ def test__write_to_stream_incomplete_read(self, checksum): in error.args[0] ) + def test__write_to_stream_single_shot_download(self): + stream = io.BytesIO() + download = download_mod.RawDownload( + EXAMPLE_URL, stream=stream, single_shot_download=True + ) + + chunk1 = b"all at once, raw!" + response = _mock_raw_response(chunks=[chunk1], headers={}) + ret_val = download._write_to_stream(response) + + assert ret_val is None + assert stream.getvalue() == chunk1 + assert download._bytes_downloaded == len(chunk1) + + response.__enter__.assert_called_once_with() + response.__exit__.assert_called_once_with(None, None, None) + response.raw.read.assert_called_once_with() + def _consume_helper( self, stream=None, @@ -1333,6 +1370,7 @@ def _mock_response(status_code=http.client.OK, chunks=None, headers=None): response.__enter__.return_value = response response.__exit__.return_value = None response.iter_content.return_value = iter(chunks) + response.raw.read = mock.Mock(side_effect=lambda *args, **kwargs: b"".join(chunks)) return response else: return mock.Mock( @@ -1348,6 +1386,7 @@ def _mock_raw_response(status_code=http.client.OK, chunks=(), headers=None): mock_raw = mock.Mock(headers=headers, spec=["stream"]) mock_raw.stream.return_value = iter(chunks) + mock_raw.read = mock.Mock(return_value=b"".join(chunks)) response = mock.MagicMock( headers=headers, status_code=int(status_code), diff --git a/tests/system/test_blob.py b/tests/system/test_blob.py index 00f218534..8b50322ba 100644 --- a/tests/system/test_blob.py +++ b/tests/system/test_blob.py @@ -1149,3 +1149,22 @@ def test_object_retention_lock(storage_client, buckets_to_delete, blobs_to_delet blob.retention.retain_until_time = None blob.patch(override_unlocked_retention=True) assert blob.retention.mode is None + + +def test_blob_download_as_bytes_single_shot_download( + shared_bucket, blobs_to_delete, file_data, service_account +): + blob_name = f"download-single-shot-{uuid.uuid4().hex}" + info = file_data["simple"] + with open(info["path"], "rb") as f: + payload = f.read() + + blob = shared_bucket.blob(blob_name) + blob.upload_from_string(payload) + blobs_to_delete.append(blob) + + result_regular_download = blob.download_as_bytes(single_shot_download=False) + assert result_regular_download == payload + + result_single_shot_download = blob.download_as_bytes(single_shot_download=True) + assert result_single_shot_download == payload diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 06ba62220..937bebaf5 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -1282,6 +1282,7 @@ def _do_download_helper_wo_chunks( end=3, checksum="auto", retry=retry, + single_shot_download=False, ) else: patched.assert_called_once_with( @@ -1292,6 +1293,7 @@ def _do_download_helper_wo_chunks( end=None, checksum="auto", retry=retry, + single_shot_download=False, ) patched.return_value.consume.assert_called_once_with( @@ -1499,6 +1501,7 @@ def test_download_to_file_with_failure(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_to_file_wo_media_link(self): @@ -1530,6 +1533,7 @@ def test_download_to_file_wo_media_link(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_to_file_w_etag_match(self): @@ -1557,6 +1561,7 @@ def test_download_to_file_w_etag_match(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_to_file_w_generation_match(self): @@ -1584,10 +1589,16 @@ def test_download_to_file_w_generation_match(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) def _download_to_file_helper( - self, use_chunks, raw_download, timeout=None, **extra_kwargs + self, + use_chunks, + raw_download, + timeout=None, + single_shot_download=False, + **extra_kwargs, ): blob_name = "blob-name" client = self._make_client() @@ -1612,9 +1623,16 @@ def _download_to_file_helper( with mock.patch.object(blob, "_prep_and_do_download"): if raw_download: - blob.download_to_file(file_obj, raw_download=True, **extra_kwargs) + blob.download_to_file( + file_obj, + raw_download=True, + single_shot_download=single_shot_download, + **extra_kwargs, + ) else: - blob.download_to_file(file_obj, **extra_kwargs) + blob.download_to_file( + file_obj, single_shot_download=single_shot_download, **extra_kwargs + ) expected_retry = extra_kwargs.get("retry", DEFAULT_RETRY) blob._prep_and_do_download.assert_called_once_with( @@ -1632,6 +1650,7 @@ def _download_to_file_helper( timeout=expected_timeout, checksum="auto", retry=expected_retry, + single_shot_download=single_shot_download, ) def test_download_to_file_wo_chunks_wo_raw(self): @@ -1643,6 +1662,26 @@ def test_download_to_file_wo_chunks_no_retry(self): def test_download_to_file_w_chunks_wo_raw(self): self._download_to_file_helper(use_chunks=True, raw_download=False) + def test_download_to_file_wo_single_shot_download_wo_raw(self): + self._download_to_file_helper( + use_chunks=False, raw_download=False, single_shot_download=False + ) + + def test_download_to_file_w_single_shot_download_wo_raw(self): + self._download_to_file_helper( + use_chunks=False, raw_download=False, single_shot_download=True + ) + + def test_download_to_file_wo_single_shot_download_w_raw(self): + self._download_to_file_helper( + use_chunks=False, raw_download=True, single_shot_download=False + ) + + def test_download_to_file_w_single_shot_download_w_raw(self): + self._download_to_file_helper( + use_chunks=False, raw_download=True, single_shot_download=True + ) + def test_download_to_file_wo_chunks_w_raw(self): self._download_to_file_helper(use_chunks=False, raw_download=True) @@ -1711,6 +1750,7 @@ def _download_to_filename_helper( timeout=expected_timeout, checksum="auto", retry=expected_retry, + single_shot_download=False, ) stream = blob._prep_and_do_download.mock_calls[0].args[0] self.assertEqual(stream.name, temp.name) @@ -1767,6 +1807,7 @@ def test_download_to_filename_w_etag_match(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) stream = blob._prep_and_do_download.mock_calls[0].args[0] self.assertEqual(stream.name, temp.name) @@ -1800,6 +1841,7 @@ def test_download_to_filename_w_generation_match(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) stream = blob._prep_and_do_download.mock_calls[0].args[0] self.assertEqual(stream.name, temp.name) @@ -1842,6 +1884,7 @@ def test_download_to_filename_corrupted(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) stream = blob._prep_and_do_download.mock_calls[0].args[0] self.assertEqual(stream.name, filename) @@ -1884,11 +1927,14 @@ def test_download_to_filename_notfound(self): timeout=expected_timeout, checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) stream = blob._prep_and_do_download.mock_calls[0].args[0] self.assertEqual(stream.name, filename) - def _download_as_bytes_helper(self, raw_download, timeout=None, **extra_kwargs): + def _download_as_bytes_helper( + self, raw_download, timeout=None, single_shot_download=False, **extra_kwargs + ): blob_name = "blob-name" client = self._make_client() bucket = _Bucket(client) @@ -1898,12 +1944,17 @@ def _download_as_bytes_helper(self, raw_download, timeout=None, **extra_kwargs): if timeout is None: expected_timeout = self._get_default_timeout() fetched = blob.download_as_bytes( - raw_download=raw_download, **extra_kwargs + raw_download=raw_download, + single_shot_download=single_shot_download, + **extra_kwargs, ) else: expected_timeout = timeout fetched = blob.download_as_bytes( - raw_download=raw_download, timeout=timeout, **extra_kwargs + raw_download=raw_download, + timeout=timeout, + single_shot_download=single_shot_download, + **extra_kwargs, ) self.assertEqual(fetched, b"") @@ -1924,6 +1975,7 @@ def _download_as_bytes_helper(self, raw_download, timeout=None, **extra_kwargs): timeout=expected_timeout, checksum="auto", retry=expected_retry, + single_shot_download=single_shot_download, ) stream = blob._prep_and_do_download.mock_calls[0].args[0] self.assertIsInstance(stream, io.BytesIO) @@ -1959,6 +2011,7 @@ def test_download_as_bytes_w_etag_match(self): timeout=self._get_default_timeout(), checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_as_bytes_w_generation_match(self): @@ -1989,6 +2042,7 @@ def test_download_as_bytes_w_generation_match(self): timeout=self._get_default_timeout(), checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_as_bytes_wo_raw(self): @@ -2003,6 +2057,16 @@ def test_download_as_bytes_w_raw(self): def test_download_as_byte_w_custom_timeout(self): self._download_as_bytes_helper(raw_download=False, timeout=9.58) + def test_download_as_bytes_wo_single_shot_download(self): + self._download_as_bytes_helper( + raw_download=False, retry=None, single_shot_download=False + ) + + def test_download_as_bytes_w_single_shot_download(self): + self._download_as_bytes_helper( + raw_download=False, retry=None, single_shot_download=True + ) + def _download_as_text_helper( self, raw_download, @@ -2100,6 +2164,7 @@ def _download_as_text_helper( if_metageneration_match=if_metageneration_match, if_metageneration_not_match=if_metageneration_not_match, retry=expected_retry, + single_shot_download=False, ) def test_download_as_text_wo_raw(self): @@ -2226,6 +2291,7 @@ def test_download_as_string(self, mock_warn): timeout=self._get_default_timeout(), checksum="auto", retry=DEFAULT_RETRY, + single_shot_download=False, ) mock_warn.assert_any_call( @@ -2264,6 +2330,7 @@ def test_download_as_string_no_retry(self, mock_warn): timeout=self._get_default_timeout(), checksum="auto", retry=None, + single_shot_download=False, ) mock_warn.assert_any_call( diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index b671cc092..db8094a95 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1866,6 +1866,7 @@ def test_download_blob_to_file_with_failure(self): checksum="auto", timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_blob_to_file_with_uri(self): @@ -1905,6 +1906,7 @@ def test_download_blob_to_file_with_uri(self): checksum="auto", timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY, + single_shot_download=False, ) def test_download_blob_to_file_with_invalid_uri(self): @@ -2032,6 +2034,7 @@ def _download_blob_to_file_helper( checksum="auto", timeout=_DEFAULT_TIMEOUT, retry=expected_retry, + single_shot_download=False, ) def test_download_blob_to_file_wo_chunks_wo_raw(self): From f02739f5f4f21083382f75169de719c6d5926a4e Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 3 Jul 2025 06:36:00 -0400 Subject: [PATCH 4/8] tests: update default runtime used for tests (#1498) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * tests: update default runtime used for tests * update system presubmits * update conformance presubmit * install setuptools for lint_setup_py * lint * use venv instead of virtualenv for nox backend * Revert "use venv instead of virtualenv for nox backend" This reverts commit 7fc0fe8ace1f85b33b316b70fa1bb3955365f1a6. * remove unit 3.7/3.8 in default nox session * update comment * ignore lint false positive * Use python 3.10 for blacken nox session, which is the latest version available in the python post processor * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- .github/sync-repo-settings.yaml | 2 +- .gitignore | 3 --- .../{system-3.8.cfg => system-3.12.cfg} | 2 +- google/cloud/storage/notification.py | 2 +- noxfile.py | 20 +++++++++++++------ owlbot.py | 4 +++- .../system/requests/test_download.py | 4 +++- .../unit/requests/test_download.py | 4 +++- tests/unit/test_blob.py | 2 +- tests/unit/test_bucket.py | 2 +- 10 files changed, 28 insertions(+), 17 deletions(-) rename .kokoro/presubmit/{system-3.8.cfg => system-3.12.cfg} (91%) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index cc1eb10e1..0d304cfe2 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -9,7 +9,7 @@ branchProtectionRules: requiredStatusCheckContexts: - 'Kokoro' - 'cla/google' - - 'Kokoro system-3.8' + - 'Kokoro system-3.12' - 'OwlBot Post Processor' - pattern: python2 requiresCodeOwnerReviews: true diff --git a/.gitignore b/.gitignore index 6f2ab1aff..d083ea1dd 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,3 @@ system_tests/local_test_setup # Make sure a generated file isn't accidentally committed. pylintrc pylintrc.test - -# Docker files -get-docker.sh diff --git a/.kokoro/presubmit/system-3.8.cfg b/.kokoro/presubmit/system-3.12.cfg similarity index 91% rename from .kokoro/presubmit/system-3.8.cfg rename to .kokoro/presubmit/system-3.12.cfg index 6d3603eed..d4cca031b 100644 --- a/.kokoro/presubmit/system-3.8.cfg +++ b/.kokoro/presubmit/system-3.12.cfg @@ -3,7 +3,7 @@ # Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "system-3.8" + value: "system-3.12" } # Credentials needed to test universe domain. diff --git a/google/cloud/storage/notification.py b/google/cloud/storage/notification.py index d13b80fc4..2dddbcee4 100644 --- a/google/cloud/storage/notification.py +++ b/google/cloud/storage/notification.py @@ -257,7 +257,7 @@ def create(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=None): with create_trace_span(name="Storage.BucketNotification.create"): if self.notification_id is not None: raise ValueError( - f"notification_id already set to {self.notification_id}; must be None to create a Notification." + f"notification_id already set to {self.notification_id}; must be None to create a Notification." # noqa: E702 ) client = self._require_client(client) diff --git a/noxfile.py b/noxfile.py index 2a7614331..b62092e97 100644 --- a/noxfile.py +++ b/noxfile.py @@ -26,10 +26,10 @@ BLACK_VERSION = "black==23.7.0" BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] -DEFAULT_PYTHON_VERSION = "3.8" -SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"] +DEFAULT_PYTHON_VERSION = "3.12" +SYSTEM_TEST_PYTHON_VERSIONS = ["3.12"] UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] -CONFORMANCE_TEST_PYTHON_VERSIONS = ["3.8"] +CONFORMANCE_TEST_PYTHON_VERSIONS = ["3.12"] CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() @@ -44,7 +44,13 @@ "lint", "lint_setup_py", "system", - "unit", + # TODO(https://github.com/googleapis/python-storage/issues/1499): + # Remove or restore testing for Python 3.7/3.8 + "unit-3.9", + "unit-3.10", + "unit-3.11", + "unit-3.12", + "unit-3.13", # cover must be last to avoid error `No data to report` "cover", ] @@ -68,7 +74,9 @@ def lint(session): session.run("flake8", "google", "tests") -@nox.session(python=DEFAULT_PYTHON_VERSION) +# Use a python runtime which is available in the owlbot post processor here +# https://github.com/googleapis/synthtool/blob/master/docker/owlbot/python/Dockerfile +@nox.session(python=["3.10", DEFAULT_PYTHON_VERSION]) def blacken(session): """Run black. @@ -84,7 +92,7 @@ def blacken(session): @nox.session(python=DEFAULT_PYTHON_VERSION) def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" - session.install("docutils", "pygments") + session.install("docutils", "pygments", "setuptools>=79.0.1") session.run("python", "setup.py", "check", "--restructuredtext", "--strict") diff --git a/owlbot.py b/owlbot.py index ba3f32df3..5292db5fd 100644 --- a/owlbot.py +++ b/owlbot.py @@ -104,4 +104,6 @@ python.py_samples(skip_readmes=True) -s.shell.run(["nox", "-s", "blacken"], hide_output=False) +# Use a python runtime which is available in the owlbot post processor here +# https://github.com/googleapis/synthtool/blob/master/docker/owlbot/python/Dockerfile +s.shell.run(["nox", "-s", "blacken-3.10"], hide_output=False) diff --git a/tests/resumable_media/system/requests/test_download.py b/tests/resumable_media/system/requests/test_download.py index 84c44c94c..5417dd7bc 100644 --- a/tests/resumable_media/system/requests/test_download.py +++ b/tests/resumable_media/system/requests/test_download.py @@ -324,7 +324,9 @@ def test_download_to_stream(self, add_files, authorized_transport): check_tombstoned(download, authorized_transport) @pytest.mark.parametrize("checksum", ["auto", "md5", "crc32c", None]) - def test_single_shot_download_to_stream(self, add_files, authorized_transport, checksum): + def test_single_shot_download_to_stream( + self, add_files, authorized_transport, checksum + ): for info in ALL_FILES: actual_contents = self._get_contents(info) blob_name = get_blob_name(info) diff --git a/tests/resumable_media/unit/requests/test_download.py b/tests/resumable_media/unit/requests/test_download.py index b17fbb905..25dba6e05 100644 --- a/tests/resumable_media/unit/requests/test_download.py +++ b/tests/resumable_media/unit/requests/test_download.py @@ -1370,7 +1370,9 @@ def _mock_response(status_code=http.client.OK, chunks=None, headers=None): response.__enter__.return_value = response response.__exit__.return_value = None response.iter_content.return_value = iter(chunks) - response.raw.read = mock.Mock(side_effect=lambda *args, **kwargs: b"".join(chunks)) + response.raw.read = mock.Mock( + side_effect=lambda *args, **kwargs: b"".join(chunks) + ) return response else: return mock.Mock( diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 937bebaf5..b3e7ec649 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -2099,7 +2099,7 @@ def _download_as_text_helper( properties = {} if charset is not None: - properties["contentType"] = f"text/plain; charset={charset}" + properties["contentType"] = f"text/plain; charset={charset}" # noqa: E702 elif no_charset: properties = {"contentType": "text/plain"} diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index ac9a5ede6..e494cc18a 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -4094,7 +4094,7 @@ def _generate_upload_policy_helper(self, **kwargs): break else: # pragma: NO COVER self.fail( - f"Condition {expected_condition} not found in {policy_conditions}" + f"Condition {expected_condition} not found in {policy_conditions}" # noqa: E713 ) return policy_fields, policy From c7357305495443348c0a432db649b27e6a5d8c5b Mon Sep 17 00:00:00 2001 From: Pulkit Aggarwal <54775856+Pulkit0110@users.noreply.github.com> Date: Fri, 4 Jul 2025 13:55:46 +0530 Subject: [PATCH 5/8] chore: update the source path in owlbot.yaml (#1497) * chore: update the source path in owlbot.yaml * add comments * Trigger CI --- .github/.OwlBot.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml index 837bf4fd6..67768efb5 100644 --- a/.github/.OwlBot.yaml +++ b/.github/.OwlBot.yaml @@ -18,9 +18,12 @@ docker: deep-remove-regex: - /owl-bot-staging +# In source, we've used two capturing groups (v2) and (.*) to match the version +# and the directory path. +# In dest, we use $1 to refer to the first capturing group (v2) and $2 to refer +# to the second capturing group (directory path). deep-copy-regex: - - source: /google/storage/v2/storage-v2-py/(.*) + - source: /google/storage/(v2)/storage-v2-py/(.*) dest: /owl-bot-staging/$1/$2 begin-after-commit-hash: 6acf4a0a797f1082027985c55c4b14b60f673dd7 - From 652cee3e351203796461d45e1c5bb6df34d170d1 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 4 Jul 2025 11:06:23 -0400 Subject: [PATCH 6/8] test: update constraints for python 3.9 testing (#1500) --- testing/constraints-3.9.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/testing/constraints-3.9.txt b/testing/constraints-3.9.txt index e69de29bb..2a588ced6 100644 --- a/testing/constraints-3.9.txt +++ b/testing/constraints-3.9.txt @@ -0,0 +1,14 @@ +# This constraints file is used to check that lower bounds +# are correct in setup.py +# List all library dependencies and extras in this file. +# Pin the version to the lower bound. +# e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0", +# Then this file should have google-cloud-foo==1.14.0 +google-auth==2.26.1 +google-api-core==2.15.0 +google-cloud-core==2.4.2 +google-resumable-media==2.7.2 +requests==2.22.0 +google-crc32c==1.1.3 +protobuf==3.20.2 +opentelemetry-api==1.1.0 From 93d58d0f7e7467d99a206725f5701da29f6f3595 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 4 Jul 2025 11:25:21 -0400 Subject: [PATCH 7/8] build: exclude certain autogenerated files (#1501) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * build: exclude certain autogenerated files * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * update exclude * typo --------- Co-authored-by: Owl Bot --- owlbot.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/owlbot.py b/owlbot.py index 5292db5fd..055b4db9c 100644 --- a/owlbot.py +++ b/owlbot.py @@ -37,6 +37,11 @@ "setup.py", "noxfile.py", "README.rst", + # Exclude autogenerated default import `google.cloud.storage` + "google/cloud/storage/*", + # Exclude autogenerated constraints files for Python 3.7/3.9 + "testing/constraints-3.7.txt", + "testing/constraints-3.9.txt", ], ) s.remove_staging_dirs() From 7d97a384406258e0bfce3fbb715b84e5220d6783 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 10:34:07 +0530 Subject: [PATCH 8/8] chore(main): release 3.2.0 (#1496) --- CHANGELOG.md | 7 +++++++ google/cloud/storage/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e66795a8..52b077ea5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-cloud-storage/#history +## [3.2.0](https://github.com/googleapis/python-storage/compare/v3.1.1...v3.2.0) (2025-07-04) + + +### Features + +* Adding support of single shot download ([#1493](https://github.com/googleapis/python-storage/issues/1493)) ([61c5d5f](https://github.com/googleapis/python-storage/commit/61c5d5f62c88506f200bc6d86b399a2c28715bc4)) + ## [3.1.1](https://github.com/googleapis/python-storage/compare/v3.1.0...v3.1.1) (2025-06-13) diff --git a/google/cloud/storage/version.py b/google/cloud/storage/version.py index fa35f1da3..c24ca23d6 100644 --- a/google/cloud/storage/version.py +++ b/google/cloud/storage/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.1.1" +__version__ = "3.2.0"