Merge branch 'main' into midstream-retries

.github/.OwlBot.lock.yaml

@@ -1,3 +1,3 @@
 docker:
   image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest
-  digest: sha256:4ee57a76a176ede9087c14330c625a71553cf9c72828b2c0ca12f5338171ba60
+  digest: sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3

.kokoro/release.sh

@@ -26,7 +26,7 @@ python3 -m pip install --upgrade twine wheel setuptools
 export PYTHONUNBUFFERED=1
 
 # Move into the package, build the distribution and upload.
-TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token")
+TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1")
 cd github/google-resumable-media-python
 python3 setup.py sdist bdist_wheel
 twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/*

.kokoro/release/common.cfg

@@ -23,8 +23,18 @@ env_vars: {
     value: "github/google-resumable-media-python/.kokoro/release.sh"
 }
 
+# Fetch PyPI password
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "google-cloud-pypi-token-keystore-1"
+    }
+  }
+}
+
 # Tokens needed to report release status back to GitHub
 env_vars: {
   key: "SECRET_MANAGER_KEYS"
-  value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token"
+  value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem"
 }

.repo-metadata.json

@@ -1,8 +1,8 @@
 {
     "name": "google-resumable-media",
     "name_pretty": "Google Resumable Media",
-    "client_documentation": "https://googleapis.dev/python/google-resumable-media/latest/index.html",
-    "release_level": "alpha",
+    "client_documentation": "https://cloud.google.com/python/docs/reference/google-resumable-media/latest",
+    "release_level": "preview",
     "language": "python",
     "library_type": "CORE",
     "repo": "googleapis/google-resumable-media-python",

google/__init__.py

@@ -19,4 +19,4 @@
 except ImportError:
     import pkgutil
 
-    __path__ = pkgutil.extend_path(__path__, __name__)
+    __path__ = pkgutil.extend_path(__path__, __name__)  # type: ignore

google/_async_resumable_media/_upload.py

@@ -612,7 +612,7 @@ def _make_invalid(self):
         """
         self._invalid = True
 
-    async def _process_response(self, response, bytes_sent):
+    async def _process_resumable_response(self, response, bytes_sent):
         """Process the response from an HTTP request.
 
         This is everything that must be done after a request that doesn't

google/_async_resumable_media/requests/_request_helpers.py

@@ -23,8 +23,8 @@
 from google._async_resumable_media import _helpers
 from google.resumable_media import common
 
-import google.auth.transport._aiohttp_requests as aiohttp_requests
-import aiohttp
+from google.auth.transport import _aiohttp_requests as aiohttp_requests  # type: ignore
+import aiohttp  # type: ignore
 
 _DEFAULT_RETRY_STRATEGY = common.RetryStrategy()
 _SINGLE_GET_CHUNK_SIZE = 8192

google/_async_resumable_media/requests/download.py

@@ -14,7 +14,7 @@
 
 """Support for downloading media from Google APIs."""
 
-import urllib3.response
+import urllib3.response  # type: ignore
 
 from google._async_resumable_media import _download
 from google._async_resumable_media import _helpers

google/_async_resumable_media/requests/upload.py

@@ -481,7 +481,7 @@ async def transmit_next_chunk(
             retry_strategy=self._retry_strategy,
             timeout=timeout,
         )
-        await self._process_response(response, len(payload))
+        await self._process_resumable_response(response, len(payload))
         return response
 
     async def recover(self, transport):

google/resumable_media/_helpers.py

@@ -148,12 +148,12 @@ def _get_crc32c_object():
     to use CRCMod. CRCMod might be using a 'slow' varietal. If so, warn...
     """
     try:
-        import google_crc32c
+        import google_crc32c  # type: ignore
 
         crc_obj = google_crc32c.Checksum()
     except ImportError:
         try:
-            import crcmod
+            import crcmod  # type: ignore
 
             crc_obj = crcmod.predefined.Crc("crc-32c")
             _is_fast_crcmod()

google/resumable_media/_upload.py

@@ -27,6 +27,7 @@
 import random
 import re
 import sys
+import urllib.parse
 
 from google import resumable_media
 from google.resumable_media import _helpers
@@ -462,10 +463,17 @@ def _prepare_initiate_request(
 
         self._stream = stream
         self._content_type = content_type
-        headers = {
-            _CONTENT_TYPE_HEADER: "application/json; charset=UTF-8",
-            "x-upload-content-type": content_type,
-        }
+
+        # Signed URL requires content type set directly - not through x-upload-content-type
+        parse_result = urllib.parse.urlparse(self.upload_url)
+        parsed_query = urllib.parse.parse_qs(parse_result.query)
+        if "x-goog-signature" in parsed_query or "X-Goog-Signature" in parsed_query:
+            headers = {_CONTENT_TYPE_HEADER: content_type}
+        else:
+            headers = {
+                _CONTENT_TYPE_HEADER: "application/json; charset=UTF-8",
+                "x-upload-content-type": content_type,
+            }
         # Set the total bytes if possible.
         if total_bytes is not None:
             self._total_bytes = total_bytes
@@ -647,7 +655,7 @@ def _make_invalid(self):
         """
         self._invalid = True
 
-    def _process_response(self, response, bytes_sent):
+    def _process_resumable_response(self, response, bytes_sent):
         """Process the response from an HTTP request.
 
         This is everything that must be done after a request that doesn't

google/resumable_media/common.py

@@ -26,7 +26,7 @@
 UPLOAD_CHUNK_SIZE = 262144  # 256 * 1024
 """int: Chunks in a resumable upload must come in multiples of 256 KB."""
 
-PERMANENT_REDIRECT = http.client.PERMANENT_REDIRECT
+PERMANENT_REDIRECT = http.client.PERMANENT_REDIRECT  # type: ignore
 """int: Permanent redirect status code.
 
 .. note::

google/resumable_media/py.typed

@@ -0,0 +1,2 @@
+# Marker file for PEP 561.
+# The google-resumable_media package uses inline types.

google/resumable_media/requests/_request_helpers.py

@@ -18,7 +18,7 @@
 """
 
 import requests.exceptions
-import urllib3.exceptions
+import urllib3.exceptions  # type: ignore
 
 import time
 

google/resumable_media/requests/download.py

@@ -14,7 +14,7 @@
 
 """Support for downloading media from Google APIs."""
 
-import urllib3.response
+import urllib3.response  # type: ignore
 
 from google.resumable_media import _download
 from google.resumable_media import common

google/resumable_media/requests/upload.py

@@ -508,7 +508,7 @@ def retriable_request():
                 method, url, data=payload, headers=headers, timeout=timeout
             )
 
-            self._process_response(result, len(payload))
+            self._process_resumable_response(result, len(payload))
 
             return result
 

mypy.ini

@@ -0,0 +1,3 @@
+[mypy]
+python_version = 3.6
+namespace_packages = True

noxfile.py

@@ -188,6 +188,19 @@ def blacken(session):
     )
 
 
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def mypy(session):
+    """Verify type hints are mypy compatible."""
+    session.install("-e", ".")
+    session.install(
+        "mypy",
+        "types-setuptools",
+        "types-requests",
+        "types-mock",
+    )
+    session.run("mypy", "google/", "tests/", "tests_async/")
+
+
 @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
 def system(session):
     """Run the system test suite."""

tests/system/requests/conftest.py

@@ -13,9 +13,9 @@
 # limitations under the License.
 """py.test fixtures to be shared across multiple system test modules."""
 
-import google.auth
-import google.auth.transport.requests as tr_requests
-import pytest
+import google.auth  # type: ignore
+import google.auth.transport.requests as tr_requests  # type: ignore
+import pytest  # type: ignore
 
 from tests.system import utils
 

tests/system/requests/test_download.py

@@ -19,9 +19,9 @@
 import io
 import os
 
-import google.auth
-import google.auth.transport.requests as tr_requests
-import pytest
+import google.auth  # type: ignore
+import google.auth.transport.requests as tr_requests  # type: ignore
+import pytest  # type: ignore
 
 from google.resumable_media import common
 import google.resumable_media.requests as resumable_requests

tests/system/requests/test_upload.py

@@ -19,7 +19,7 @@
 import os
 import urllib.parse
 
-import pytest
+import pytest  # type: ignore
 import mock
 
 from google.resumable_media import common

tests/system/utils.py

@@ -16,7 +16,7 @@
 import hashlib
 import time
 
-from test_utils.retry import RetryResult
+from test_utils.retry import RetryResult  # type: ignore
 
 
 BUCKET_NAME = "grpm-systest-{}".format(int(1000 * time.time()))

tests/unit/requests/test__helpers.py

@@ -15,10 +15,10 @@
 import http.client
 
 import mock
-import pytest
+import pytest  # type: ignore
 
 import requests.exceptions
-import urllib3.exceptions
+import urllib3.exceptions  # type: ignore
 
 from google.resumable_media import common
 from google.resumable_media.requests import _request_helpers

tests/unit/requests/test_download.py

@@ -16,7 +16,7 @@
 import io
 
 import mock
-import pytest
+import pytest  # type: ignore
 
 from google.resumable_media import common
 from google.resumable_media import _helpers

tests/unit/test__download.py

@@ -16,7 +16,7 @@
 import io
 
 import mock
-import pytest
+import pytest  # type: ignore
 
 from google.resumable_media import _download
 from google.resumable_media import common

tests/unit/test__helpers.py

@@ -18,7 +18,7 @@
 import http.client
 
 import mock
-import pytest
+import pytest  # type: ignore
 
 from google.resumable_media import _helpers
 from google.resumable_media import common