Skip to content

Commit 47b1150

Browse files
software-dovsoftware-dov
authored
Fix for lazy import feature in alternative templates (#399)
This change places lazy imports only at the top levels of the generated GAPIC. The resulting surface is easier to test and maintains operational simplicity.
1 parent ecee115 commit 47b1150

11 files changed

Lines changed: 294 additions & 279 deletions

File tree

‎packages/gapic-generator/gapic/ads-templates/%namespace/%name/%version/%sub/__init__.py.j2‎

Lines changed: 0 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -1,100 +1,6 @@
11
{% extends '_base.py.j2' %}
22

33
{% block content %}
4-
{% if opts.lazy_import -%} {# lazy import #}
5-
import importlib
6-
import re
7-
import sys
8-
9-
from itertools import chain
10-
11-
def to_snake_case(s: str) -> str:
12-
s = re.sub(r'(?<=[a-z])([A-Z])', r'_\1', str(s))
13-
s = re.sub(r'(?<=[^_])([A-Z])(?=[a-z])', r'_\1', s)
14-
15-
# Numbers are a weird case; the goal is to spot when they _start_
16-
# some kind of name or acronym (e.g. 2FA, 3M).
17-
#
18-
# Find cases of a number preceded by a lower-case letter _and_
19-
# followed by at least two capital letters or a single capital and
20-
# end of string.
21-
s = re.sub(r'(?<=[a-z])(\d)(?=[A-Z]{2})', r'_\1', s)
22-
s = re.sub(r'(?<=[a-z])(\d)(?=[A-Z]$)', r'_\1', s)
23-
24-
return s.lower()
25-
26-
27-
def from_snake_case(s):
28-
_CHARS_TO_UPCASE_RE = re.compile(r'(?:_|^)([a-z])')
29-
return _CHARS_TO_UPCASE_RE.sub(lambda m: m.group().replace('_', '').upper(), s)
30-
31-
32-
if sys.version_info < (3, 7):
33-
raise ImportError('This module requires Python 3.7 or later.') # pragma: NO COVER
34-
35-
_lazy_name_to_package_map = {
36-
'types': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types',
37-
{%- for service in api.services.values()|sort(attribute='name')|unique(attribute='name') if service.meta.address.subpackage == api.subpackage_view %}
38-
'{{ service.client_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.{{ service.name|snake_case }}.client',
39-
'{{ service.transport_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.{{ service.name|snake_case }}.transports.base',
40-
'{{ service.grpc_transport_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.{{ service.name|snake_case }}.transports.grpc',
41-
{%- endfor %}
42-
}
43-
44-
_lazy_type_to_package_map = {
45-
{%- filter sort_lines %}
46-
{%- for proto in api.protos.values() if proto.meta.address.subpackage == api.subpackage_view %}{%- for message in proto.messages.values() %}
47-
'{{ message.name }}':'{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types.{{ proto.module_name }}',
48-
{%- endfor %}
49-
{%- for enum in proto.enums.values() %}
50-
'{{ enum.name }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types.{{ proto.module_name }}',
51-
{%- endfor %}{%- endfor %}{%- endfilter %}
52-
}
53-
54-
# Background on how this behaves: https://www.python.org/dev/peps/pep-0562/
55-
def __getattr__(name): # Requires Python >= 3.7
56-
if name == '__all__':
57-
all_names = globals()['__all__'] = sorted(
58-
chain(
59-
(from_snake_case(k) for k in _lazy_name_to_package_map if k != 'types'),
60-
_lazy_type_to_package_map,
61-
['types'],
62-
)
63-
)
64-
return all_names
65-
elif name.endswith('Transport'):
66-
module = __getattr__(to_snake_case(name))
67-
sub_mod_class = getattr(module, name)
68-
klass = type(name, (sub_mod_class,), {'__doc__': sub_mod_class.__doc__})
69-
globals()[name] = klass
70-
return klass
71-
elif name.endswith('Client'):
72-
module = __getattr__(to_snake_case(name))
73-
sub_mod_class = getattr(module, name)
74-
klass = type(
75-
name,
76-
(sub_mod_class,),
77-
{'__doc__': sub_mod_class.__doc__}
78-
)
79-
globals()[name] = klass
80-
return klass
81-
elif name in _lazy_name_to_package_map:
82-
module = importlib.import_module(f'{_lazy_name_to_package_map[name]}')
83-
globals()[name] = module
84-
return module
85-
elif name in _lazy_type_to_package_map:
86-
module = importlib.import_module(f'{_lazy_type_to_package_map[name]}')
87-
klass = getattr(module, name)
88-
{# new_klass = type(name, (klass,), {'__doc__': klass.__doc__}) #}
89-
globals()[name] = klass
90-
return klass
91-
else:
92-
raise AttributeError(f'unknown sub-module {name!r}.')
93-
94-
95-
def __dir__():
96-
return globals().get('__all__') or __getattr__('__all__')
97-
{% else -%} {# do not use lazy import #}
984
{# Import subpackages. -#}
995
{% for subpackage in api.subpackages.keys() -%}
1006
from . import {{ subpackage }}
@@ -149,5 +55,4 @@ __all__ = (
14955
{%- endfor %}
15056
{%- endfilter %}
15157
)
152-
{% endif -%} {# lazy import #}
15358
{% endblock %}

‎packages/gapic-generator/gapic/ads-templates/%namespace/%name/%version/%sub/types/__init__.py.j2‎

Lines changed: 0 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1 @@
11
{% extends '_base.py.j2' %}
2-
3-
{% block content %}
4-
{% if opts.lazy_import -%} {# lazy import #}
5-
import importlib
6-
import sys
7-
8-
9-
if sys.version_info < (3, 7):
10-
raise ImportError('This module requires Python 3.7 or later.') # pragma: NO COVER
11-
12-
13-
_lazy_type_to_package_map = {
14-
{%- filter sort_lines %}
15-
{%- for proto in api.protos.values() if proto.meta.address.subpackage == api.subpackage_view %}{%- for message in proto.messages.values() %}
16-
'{{ message.name }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types.{{ proto.module_name }}',
17-
{%- endfor %}
18-
{%- for enum in proto.enums.values() %}
19-
'{{ enum.name }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types.{{ proto.module_name }}',
20-
{%- endfor %}{%- endfor %}{%- endfilter %}
21-
}
22-
23-
24-
# Background on how this behaves: https://www.python.org/dev/peps/pep-0562/
25-
def __getattr__(name): # Requires Python >= 3.7
26-
if name == '__all__':
27-
all_names = globals()['__all__'] = sorted(_lazy_type_to_package_map)
28-
return all_names
29-
elif name in _lazy_type_to_package_map:
30-
module = importlib.import_module(f'{_lazy_type_to_package_map[name]}')
31-
klass = getattr(module, name)
32-
{# new_klass = type(name, (klass,), {'__doc__': klass.__doc__}) #}
33-
globals()[name] = klass
34-
return klass
35-
else:
36-
raise AttributeError(f'unknown sub-module {name!r}.')
37-
38-
39-
def __dir__():
40-
return globals().get('__all__') or __getattr__('__all__')
41-
42-
{% else -%}
43-
{% for p in api.protos.values() if p.file_to_generate and p.messages -%}
44-
from .{{p.module_name }} import ({% for m in p.messages.values() %}{{ m.name }}, {% endfor %})
45-
{% endfor %}
46-
47-
__all__ = (
48-
{%- for p in api.protos.values() if p.file_to_generate %}{% for m in p.messages.values() %}
49-
'{{ m.name }}',
50-
{%- endfor %}{% endfor %}
51-
)
52-
{% endif -%} {# lazy import #}
53-
{% endblock %}

‎packages/gapic-generator/gapic/ads-templates/%namespace/%name/%version/__init__.py.j2‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
{% extends '_base.py.j2' %}
2+
{% block content %}
3+
{% if opts.lazy_import -%} {# lazy import #}
4+
import importlib
5+
import sys
6+
7+
8+
if sys.version_info < (3, 7):
9+
raise ImportError('This module requires Python 3.7 or later.')
10+
11+
12+
_lazy_type_to_package_map = {
13+
# Message types
14+
{%- for message in api.top_level_messages.values() %}
15+
'{{ message.name }}': '{{ message.ident.package|join('.') }}.types.{{ message.ident.module }}',
16+
{%- endfor %}
17+
18+
# Enum types
19+
{%- for enum in api.top_level_enums.values() %}
20+
'{{ enum.name }}': '{{ enum.ident.package|join('.') }}.types.{{enum.ident.module }}',
21+
{%- endfor %}
22+
23+
# Client classes and transports
24+
{%- for service in api.services.values() %}
25+
'{{ service.client_name }}': '{{ service.meta.address.package|join('.') }}.services.{{ service.meta.address.module }}',
26+
'{{ service.transport_name }}': '{{ service.meta.address.package|join('.') }}.services.{{ service.meta.address.module }}.transports',
27+
'{{ service.grpc_transport_name }}': '{{ service.meta.address.package|join('.') }}.services.{{ service.meta.address.module }}.transports',
28+
{%- endfor %}
29+
}
30+
31+
32+
# Background on how this behaves: https://www.python.org/dev/peps/pep-0562/
33+
def __getattr__(name): # Requires Python >= 3.7
34+
if name == '__all__':
35+
all_names = globals()['__all__'] = sorted(_lazy_type_to_package_map)
36+
return all_names
37+
elif name in _lazy_type_to_package_map:
38+
module = importlib.import_module(f'{_lazy_type_to_package_map[name]}')
39+
klass = getattr(module, name)
40+
{# new_klass = type(name, (klass,), {'__doc__': klass.__doc__}) #}
41+
globals()[name] = klass
42+
return klass
43+
else:
44+
raise AttributeError(f'unknown type {name!r}.')
45+
46+
47+
def __dir__():
48+
return globals().get('__all__') or __getattr__('__all__')
49+
{% else -%} {# do not use lazy import #}
50+
{# Import subpackages. -#}
51+
{% filter sort_lines -%}
52+
{% for subpackage in api.subpackages.keys() -%}
53+
from {% if api.naming.module_namespace %}{{ api.naming.module_namespace|join('.') }}.{% endif -%}
54+
{{ api.naming.versioned_module_name }} import {{ subpackage }}
55+
{% endfor -%}
56+
57+
{# Import services for this package. -#}
58+
{% for service in api.services.values()|sort(attribute='name')
59+
if service.meta.address.subpackage == api.subpackage_view -%}
60+
from {% if api.naming.module_namespace %}{{ api.naming.module_namespace|join('.') }}.{% endif -%}
61+
{{ api.naming.versioned_module_name }}.services.{{ service.name|snake_case }}.client import {{ service.client_name }}
62+
{% endfor -%}
63+
64+
{# Import messages and enums from each proto.
65+
It is safe to import all of the messages into the same namespace here,
66+
because protocol buffers itself enforces selector uniqueness within
67+
a proto package.
68+
-#}
69+
{# Import messages from each proto.
70+
It is safe to import all of the messages into the same namespace here,
71+
because protocol buffers itself enforces selector uniqueness within
72+
a proto package.
73+
-#}
74+
{% for proto in api.protos.values()|sort(attribute='module_name')
75+
if proto.meta.address.subpackage == api.subpackage_view -%}
76+
{% for message in proto.messages.values()|sort(attribute='name') -%}
77+
from {% if api.naming.module_namespace %}{{ api.naming.module_namespace|join('.') }}.{% endif -%}
78+
{{ api.naming.versioned_module_name }}.types.{{ proto.module_name }} import {{ message.name }}
79+
{% endfor -%}
80+
{% for enum in proto.enums.values()|sort(attribute='name') -%}
81+
from {% if api.naming.module_namespace %}{{ api.naming.module_namespace|join('.') }}.{% endif -%}
82+
{{ api.naming.versioned_module_name }}.types.{{ proto.module_name }} import {{ enum.name }}
83+
{% endfor %}{% endfor -%}
84+
{% endfilter %}
85+
{# Define __all__.
86+
This requires the full set of imported names, so we iterate over
87+
them again.
88+
-#}
89+
__all__ = (
90+
{%- filter indent %}
91+
{% filter sort_lines -%}
92+
{% for subpackage in api.subpackages.keys() -%}
93+
'{{ subpackage }}',
94+
{% endfor -%}
95+
{% for service in api.services.values()|sort(attribute='name')
96+
if service.meta.address.subpackage == api.subpackage_view -%}
97+
'{{ service.client_name }}',
98+
{% endfor -%}
99+
{% for proto in api.protos.values()|sort(attribute='module_name')
100+
if proto.meta.address.subpackage == api.subpackage_view -%}
101+
{% for message in proto.messages.values()|sort(attribute='name') -%}
102+
'{{ message.name }}',
103+
{% endfor -%}
104+
{% for enum in proto.enums.values()|sort(attribute='name')
105+
if proto.meta.address.subpackage == api.subpackage_view -%}
106+
'{{ enum.name }}',
107+
{% endfor -%}{% endfor -%}
108+
{% endfilter -%}
109+
{% endfilter -%}
110+
)
111+
{% endif -%} {# lazy import #}
112+
{% endblock %}

‎packages/gapic-generator/gapic/ads-templates/%namespace/%name/__init__.py.j2‎

Lines changed: 17 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -2,92 +2,46 @@
22
{% block content %}
33
{% if opts.lazy_import -%} {# lazy import #}
44
import importlib
5-
import re
65
import sys
76

8-
from itertools import chain
9-
10-
def to_snake_case(s: str) -> str:
11-
s = re.sub(r'(?<=[a-z])([A-Z])', r'_\1', str(s))
12-
s = re.sub(r'(?<=[^_])([A-Z])(?=[a-z])', r'_\1', s)
13-
14-
# Numbers are a weird case; the goal is to spot when they _start_
15-
# some kind of name or acronym (e.g. 2FA, 3M).
16-
#
17-
# Find cases of a number preceded by a lower-case letter _and_
18-
# followed by at least two capital letters or a single capital and
19-
# end of string.
20-
s = re.sub(r'(?<=[a-z])(\d)(?=[A-Z]{2})', r'_\1', s)
21-
s = re.sub(r'(?<=[a-z])(\d)(?=[A-Z]$)', r'_\1', s)
22-
23-
return s.lower()
24-
25-
26-
def from_snake_case(s):
27-
_CHARS_TO_UPCASE_RE = re.compile(r'(?:_|^)([a-z])')
28-
return _CHARS_TO_UPCASE_RE.sub(lambda m: m.group().replace('_', '').upper(), s)
29-
307

318
if sys.version_info < (3, 7):
329
raise ImportError('This module requires Python 3.7 or later.')
3310

34-
_lazy_name_to_package_map = {
35-
'types': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types',
36-
{%- for service in api.services.values()|sort(attribute='name')|unique(attribute='name') if service.meta.address.subpackage == api.subpackage_view %}
37-
'{{ service.client_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.{{ service.name|snake_case }}.client',
38-
'{{ service.transport_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.transports.base',
39-
'{{ service.grpc_transport_name|snake_case }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.services.transports.grpc',
40-
{%- endfor %} {# Need to do types and enums #}
41-
}
4211

4312
_lazy_type_to_package_map = {
44-
{%- for proto in api.protos.values() if proto.meta.address.subpackage == api.subpackage_view %}{%- for message in proto.messages.values() %}
45-
'{{ message.name }}':'{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types.{{ proto.module_name }}',
13+
# Message types
14+
{%- for message in api.top_level_messages.values() %}
15+
'{{ message.name }}': '{{ message.ident.package|join('.') }}.types.{{ message.ident.module }}',
16+
{%- endfor %}
17+
18+
# Enum types
19+
{%- for enum in api.top_level_enums.values() %}
20+
'{{ enum.name }}': '{{ enum.ident.package|join('.') }}.types.{{enum.ident.module }}',
21+
{%- endfor %}
22+
23+
# Client classes and transports
24+
{%- for service in api.services.values() %}
25+
'{{ service.client_name }}': '{{ service.meta.address.package|join('.') }}.services.{{ service.meta.address.module }}',
26+
'{{ service.transport_name }}': '{{ service.meta.address.package|join('.') }}.services.{{ service.meta.address.module }}.transports',
27+
'{{ service.grpc_transport_name }}': '{{ service.meta.address.package|join('.') }}.services.{{ service.meta.address.module }}.transports',
4628
{%- endfor %}
47-
{%- for enum in proto.enums.values() %}
48-
'{{ enum.name }}': '{% if api.naming.module_namespace %}{{ api.naming.module_namespace|join(".") }}.{% endif -%}{{ api.naming.versioned_module_name }}.types.{{ proto.module_name }}',
49-
{%- endfor %}{%- endfor %}
5029
}
5130

5231

5332
# Background on how this behaves: https://www.python.org/dev/peps/pep-0562/
5433
def __getattr__(name): # Requires Python >= 3.7
5534
if name == '__all__':
56-
all_names = globals()['__all__'] = sorted(
57-
chain(
58-
(from_snake_case(k) for k in _lazy_name_to_package_map),
59-
_lazy_type_to_package_map,
60-
)
61-
)
35+
all_names = globals()['__all__'] = sorted(_lazy_type_to_package_map)
6236
return all_names
63-
elif name.endswith('Transport'):
64-
module = __getattr__(to_snake_case(name))
65-
sub_mod_class = getattr(module, name)
66-
klass = type(name, (sub_mod_class,), {'__doc__': sub_mod_class.__doc__})
67-
globals()[name] = klass
68-
return klass
69-
elif name.endswith('Client'):
70-
module = __getattr__(to_snake_case(name))
71-
sub_mod_class = getattr(module, name)
72-
klass = type(
73-
name,
74-
(sub_mod_class,),
75-
{'__doc__': sub_mod_class.__doc__}
76-
)
77-
globals()[name] = klass
78-
return klass
79-
elif name in _lazy_name_to_package_map:
80-
module = importlib.import_module(f'{_lazy_name_to_package_map[name]}')
81-
globals()[name] = module
82-
return module
8337
elif name in _lazy_type_to_package_map:
8438
module = importlib.import_module(f'{_lazy_type_to_package_map[name]}')
8539
klass = getattr(module, name)
8640
{# new_klass = type(name, (klass,), {'__doc__': klass.__doc__}) #}
8741
globals()[name] = klass
8842
return klass
8943
else:
90-
raise AttributeError(f'unknown sub-module {name!r}.')
44+
raise AttributeError(f'unknown type {name!r}.')
9145

9246

9347
def __dir__():

‎packages/gapic-generator/gapic/ads-templates/.coveragerc.j2‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,6 @@ exclude_lines =
1616
# generates the code and tries to run it without pip installing. This
1717
# makes it virtually impossible to test properly.
1818
except pkg_resources.DistributionNotFound
19+
# This is used to indicate a python version mismatch,
20+
# which is not easily tested in unit tests.
21+
raise ImportError

0 commit comments

Comments
 (0)