From 3473be976b9d6b3aca90327901d2fadabe27c95c Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 2 Dec 2021 17:56:40 -0500 Subject: [PATCH 1/6] fix: work around invalid generated typing for map fields See: https://github.com/googleapis/gapic-generator-python/issues/689 --- .../bigtable_instance_admin/async_client.py | 4 +--- .../services/bigtable_instance_admin/client.py | 4 +--- owlbot.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py b/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py index a3f4b2a72..8f2d11588 100644 --- a/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py +++ b/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py @@ -199,9 +199,7 @@ async def create_instance( parent: str = None, instance_id: str = None, instance: gba_instance.Instance = None, - clusters: Sequence[ - bigtable_instance_admin.CreateInstanceRequest.ClustersEntry - ] = None, + clusters: Dict[str, gba_instance.Cluster] = None, retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), diff --git a/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py b/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py index ce245570e..a45986b2c 100644 --- a/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py +++ b/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py @@ -430,9 +430,7 @@ def create_instance( parent: str = None, instance_id: str = None, instance: gba_instance.Instance = None, - clusters: Sequence[ - bigtable_instance_admin.CreateInstanceRequest.ClustersEntry - ] = None, + clusters: Dict[str, gba_instance.Cluster] = None, retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), diff --git a/owlbot.py b/owlbot.py index 438628413..2547428b1 100644 --- a/owlbot.py +++ b/owlbot.py @@ -175,6 +175,20 @@ def lint_setup_py(session): ''', ) +bad_clusters_typing = r""" + clusters: Sequence\[ + bigtable_instance_admin\.CreateInstanceRequest\.ClustersEntry + \] = None,""" + +good_clusters_typing = """ + clusters: Dict[str, gba_instance.Cluster] = None,""" + +s.replace( + "google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/*client.py", + bad_clusters_typing, + good_clusters_typing, +) + # ---------------------------------------------------------------------------- # Samples templates # ---------------------------------------------------------------------------- From 2ddbcdf6ed7d8d7ba98f434d66b2a637aea87bd0 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 2 Dec 2021 17:57:14 -0500 Subject: [PATCH 2/6] fix: restore missing namespace marker file --- google/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 google/__init__.py diff --git a/google/__init__.py b/google/__init__.py new file mode 100644 index 000000000..ced5017a1 --- /dev/null +++ b/google/__init__.py @@ -0,0 +1,10 @@ +from typing import List + +try: + import pkg_resources + + pkg_resources.declare_namespace(__name__) +except ImportError: + import pkgutil + + __path__: List[str] = pkgutil.extend_path(__path__, __name__) From 6cb6ca64cf06c75ee761b13a3a9e79c784dc4b28 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 2 Dec 2021 17:59:08 -0500 Subject: [PATCH 3/6] ci: get mypy running against all modules in 'google/' / 'tests/' Note we still exclude generated tests, due to broken typing emiteed from the gapic-generator. --- mypy.ini | 24 +++++++++++++++++++++++- noxfile.py | 5 +++-- owlbot.py | 5 +++-- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/mypy.ini b/mypy.ini index 9aef441de..f12ed46fc 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,28 @@ [mypy] python_version = 3.6 namespace_packages = True +exclude = tests/unit/gapic/ -[mypy-google.protobuf] +[mypy-grpc.*] +ignore_missing_imports = True + +[mypy-google.auth.*] +ignore_missing_imports = True + +[mypy-google.iam.*] +ignore_missing_imports = True + +[mypy-google.longrunning.*] +ignore_missing_imports = True + +[mypy-google.oauth2.*] +ignore_missing_imports = True + +[mypy-google.rpc.*] +ignore_missing_imports = True + +[mypy-proto.*] +ignore_missing_imports = True + +[mypy-pytest] ignore_missing_imports = True diff --git a/noxfile.py b/noxfile.py index 11ec0e948..6ae044f00 100644 --- a/noxfile.py +++ b/noxfile.py @@ -77,9 +77,10 @@ def blacken(session): def mypy(session): """Verify type hints are mypy compatible.""" session.install("-e", ".") - session.install("mypy", "types-setuptools") + session.install("mypy", "types-setuptools", "types-protobuf", "types-mock") + session.install("google-cloud-testutils") # TODO: also verify types on tests, all of google package - session.run("mypy", "-p", "google.cloud.bigtable", "--no-incremental") + session.run("mypy", "google/", "tests/") @nox.session(python=DEFAULT_PYTHON_VERSION) diff --git a/owlbot.py b/owlbot.py index 2547428b1..aee0def6b 100644 --- a/owlbot.py +++ b/owlbot.py @@ -165,9 +165,10 @@ def lint_setup_py\(session\): def mypy(session): """Verify type hints are mypy compatible.""" session.install("-e", ".") - session.install("mypy", "types-setuptools") + session.install("mypy", "types-setuptools", "types-protobuf", "types-mock") + session.install("google-cloud-testutils") # TODO: also verify types on tests, all of google package - session.run("mypy", "-p", "google.cloud.bigtable", "--no-incremental") + session.run("mypy", "google/", "tests/") @nox.session(python=DEFAULT_PYTHON_VERSION) From 08ab9dedec06005065748c0fd24f64b1f04cc0d7 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 6 Dec 2021 16:56:48 -0500 Subject: [PATCH 4/6] ci: fix coverage for restored namespace marker --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 1ba5bb57d..9b0751055 100644 --- a/.coveragerc +++ b/.coveragerc @@ -19,6 +19,7 @@ branch = True omit = google/cloud/__init__.py + google/__init__.py [report] fail_under = 100 From f7441f4e47d38f9ce5ee1b02537a90ff705da87d Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 7 Dec 2021 17:20:46 -0500 Subject: [PATCH 5/6] fix: also fix up docstring for malformed 'clusters' typing --- .../bigtable_instance_admin/async_client.py | 2 +- .../bigtable_instance_admin/client.py | 2 +- owlbot.py | 23 +++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py b/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py index 8f2d11588..15aa113fe 100644 --- a/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py +++ b/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py @@ -233,7 +233,7 @@ async def create_instance( This corresponds to the ``instance`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - clusters (:class:`Sequence[google.cloud.bigtable_admin_v2.types.CreateInstanceRequest.ClustersEntry]`): + clusters (Dict[str, gba_instance.Cluster]): Required. The clusters to be created within the instance, mapped by desired cluster ID, e.g., just ``mycluster`` rather than diff --git a/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py b/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py index a45986b2c..d59849e9c 100644 --- a/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py +++ b/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py @@ -464,7 +464,7 @@ def create_instance( This corresponds to the ``instance`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - clusters (Sequence[google.cloud.bigtable_admin_v2.types.CreateInstanceRequest.ClustersEntry]): + clusters (Dict[str, gba_instance.Cluster]): Required. The clusters to be created within the instance, mapped by desired cluster ID, e.g., just ``mycluster`` rather than diff --git a/owlbot.py b/owlbot.py index aee0def6b..ec9335683 100644 --- a/owlbot.py +++ b/owlbot.py @@ -15,6 +15,7 @@ """This script is used to synthesize generated parts of this library.""" from pathlib import Path +import re from typing import List, Optional import synthtool as s @@ -176,6 +177,7 @@ def lint_setup_py(session): ''', ) +# Work around https://github.com/googleapis/gapic-generator-python/issues/689 bad_clusters_typing = r""" clusters: Sequence\[ bigtable_instance_admin\.CreateInstanceRequest\.ClustersEntry @@ -190,6 +192,27 @@ def lint_setup_py(session): good_clusters_typing, ) +bad_clusters_docstring_1 = re.escape(r""" + clusters (:class:`Sequence[google.cloud.bigtable_admin_v2.types.CreateInstanceRequest.ClustersEntry]`):""") + +bad_clusters_docstring_2 = re.escape(r""" + clusters (Sequence[google.cloud.bigtable_admin_v2.types.CreateInstanceRequest.ClustersEntry]):""") + +good_clusters_docstring = """ + clusters (Dict[str, gba_instance.Cluster]):""" + +s.replace( + "google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/*client.py", + bad_clusters_docstring_1, + good_clusters_docstring, +) + +s.replace( + "google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/*client.py", + bad_clusters_docstring_2, + good_clusters_docstring, +) + # ---------------------------------------------------------------------------- # Samples templates # ---------------------------------------------------------------------------- From 337741484bd5b6e8482d935f569640d27db8d14b Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 7 Dec 2021 17:27:13 -0500 Subject: [PATCH 6/6] fix: indent docstring entry properly --- .../services/bigtable_instance_admin/async_client.py | 2 +- .../services/bigtable_instance_admin/client.py | 2 +- owlbot.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py b/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py index 15aa113fe..6fdda38fa 100644 --- a/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py +++ b/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/async_client.py @@ -233,7 +233,7 @@ async def create_instance( This corresponds to the ``instance`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - clusters (Dict[str, gba_instance.Cluster]): + clusters (Dict[str, gba_instance.Cluster]): Required. The clusters to be created within the instance, mapped by desired cluster ID, e.g., just ``mycluster`` rather than diff --git a/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py b/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py index d59849e9c..fca8eed03 100644 --- a/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py +++ b/google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/client.py @@ -464,7 +464,7 @@ def create_instance( This corresponds to the ``instance`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - clusters (Dict[str, gba_instance.Cluster]): + clusters (Dict[str, gba_instance.Cluster]): Required. The clusters to be created within the instance, mapped by desired cluster ID, e.g., just ``mycluster`` rather than diff --git a/owlbot.py b/owlbot.py index ec9335683..ca452ddf3 100644 --- a/owlbot.py +++ b/owlbot.py @@ -199,7 +199,7 @@ def lint_setup_py(session): clusters (Sequence[google.cloud.bigtable_admin_v2.types.CreateInstanceRequest.ClustersEntry]):""") good_clusters_docstring = """ - clusters (Dict[str, gba_instance.Cluster]):""" + clusters (Dict[str, gba_instance.Cluster]):""" s.replace( "google/cloud/bigtable_admin_v2/services/bigtable_instance_admin/*client.py",