Skip to content
This repository was archived by the owner on Mar 31, 2026. It is now read-only.

Commit bd72f5d

Browse files
cojencocojenco
authored
tests: add retry conformance test framework and test always idempotent operations (#450)
* initial tests for retry strategy conformance tests implementation * revise helper methods to interact with Retry Test API * add helper to delete retry test * handle exceptions in helper methods * remove try except * change payload key to instructions to align emulator change * rename and revise to loop through each case and method * wip populate resources to conf tests * add logic to populate fixture resources * add helper method to populate fixture hmacy key * revise endpoints and 2 clients * add assertions to testdata scenarios * add logic for preconditions wip * add mapping scenarios and formatting for readability * add library methods to method invocation mapping and json * lint * refactor using **kwargs * remove unused module and fix lint * handle misused arguments following style guide * handle unused arguments * wip: add library methods to mapping and json * add client_options to resource populating client * log warnings and revise try except blocks * update schema for S1 and S2 * fix lint and mark error * move retry conformance tests to separate folder * update noxfile * relocate conformance tests * add S1 S2 tests and delete json file * lint and clean comments * add S2 object library methods * address comments * change test parametrization to separate test cases and address comments * add assertion message and display library method name * add multiple lib methods and revise assertion message * add S2 entry library methods after emulator fix * address comments * revise test case structure using python globals * revise library methods naming to start with class name * unify library method signatures * cleanup code * use pytest fixtures to populate resources * address comments and update docstrings * address comments. change to use anonymous credentials * update descriptions
1 parent 18241b2 commit bd72f5d

5 files changed

Lines changed: 1051 additions & 0 deletions

File tree

‎noxfile.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@
3030
DEFAULT_PYTHON_VERSION = "3.8"
3131
SYSTEM_TEST_PYTHON_VERSIONS = ["2.7", "3.8"]
3232
UNIT_TEST_PYTHON_VERSIONS = ["2.7", "3.6", "3.7", "3.8", "3.9"]
33+
CONFORMANCE_TEST_PYTHON_VERSIONS = ["3.8"]
34+
35+
_DEFAULT_STORAGE_HOST = "https://storage.googleapis.com"
3336

3437
CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
3538

@@ -148,6 +151,37 @@ def system(session):
148151
session.run("py.test", "--quiet", system_test_folder_path, *session.posargs)
149152

150153

154+
@nox.session(python=CONFORMANCE_TEST_PYTHON_VERSIONS)
155+
def conftest_retry(session):
156+
"""Run the retry conformance test suite."""
157+
conformance_test_path = os.path.join("tests", "conformance.py")
158+
conformance_test_folder_path = os.path.join("tests", "conformance")
159+
160+
# Environment check: Only run tests if the STORAGE_EMULATOR_HOST is set.
161+
if (
162+
os.environ.get("STORAGE_EMULATOR_HOST", _DEFAULT_STORAGE_HOST)
163+
== _DEFAULT_STORAGE_HOST
164+
):
165+
session.skip("Set STORAGE_EMULATOR_HOST to run, skipping")
166+
167+
conformance_test_exists = os.path.exists(conformance_test_path)
168+
conformance_test_folder_exists = os.path.exists(conformance_test_folder_path)
169+
# Environment check: only run tests if found.
170+
if not conformance_test_exists and not conformance_test_folder_exists:
171+
session.skip("Conformance tests were not found")
172+
173+
session.install("pytest",)
174+
session.install("-e", ".")
175+
176+
# Run py.test against the conformance tests.
177+
if conformance_test_exists:
178+
session.run("py.test", "--quiet", conformance_test_path, *session.posargs)
179+
if conformance_test_folder_exists:
180+
session.run(
181+
"py.test", "--quiet", conformance_test_folder_path, *session.posargs
182+
)
183+
184+
151185
@nox.session(python=DEFAULT_PYTHON_VERSION)
152186
def cover(session):
153187
"""Run the final coverage report.

‎tests/conformance/__init__.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import io
16+
import json
17+
import os
18+
19+
20+
def _read_local_json(json_file):
21+
here = os.path.dirname(__file__)
22+
json_path = os.path.abspath(os.path.join(here, json_file))
23+
with io.open(json_path, "r", encoding="utf-8-sig") as fileobj:
24+
return json.load(fileobj)

‎tests/conformance/retry_strategy_test_data.json‎

Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
{
2+
"retryStrategyTests": [
3+
{
4+
"id": 1,
5+
"description": "always idempotent",
6+
"cases": [
7+
{
8+
"instructions": [
9+
"return-503",
10+
"return-503"
11+
]
12+
}
13+
],
14+
"methods": [
15+
{
16+
"name": "storage.bucket_acl.get",
17+
"resources": [
18+
"BUCKET"
19+
]
20+
},
21+
{
22+
"name": "storage.bucket_acl.list",
23+
"resources": [
24+
"BUCKET"
25+
]
26+
},
27+
{
28+
"name": "storage.buckets.delete",
29+
"resources": [
30+
"BUCKET",
31+
"OBJECT"
32+
]
33+
},
34+
{
35+
"name": "storage.buckets.get",
36+
"resources": [
37+
"BUCKET"
38+
]
39+
},
40+
{
41+
"name": "storage.buckets.getIamPolicy",
42+
"resources": [
43+
"BUCKET"
44+
]
45+
},
46+
{
47+
"name": "storage.buckets.insert",
48+
"resources": []
49+
},
50+
{
51+
"name": "storage.buckets.list",
52+
"resources": [
53+
"BUCKET"
54+
]
55+
},
56+
{
57+
"name": "storage.buckets.lockRententionPolicy",
58+
"resources": [
59+
"BUCKET"
60+
]
61+
},
62+
{
63+
"name": "storage.buckets.testIamPermission",
64+
"resources": [
65+
"BUCKET"
66+
]
67+
},
68+
{
69+
"name": "storage.default_object_acl.get",
70+
"resources": [
71+
"BUCKET",
72+
"OBJECT"
73+
]
74+
},
75+
{
76+
"name": "storage.default_object_acl.list",
77+
"resources": [
78+
"BUCKET",
79+
"OBJECT"
80+
]
81+
},
82+
{
83+
"name": "storage.hmacKey.delete",
84+
"resources": []
85+
},
86+
{
87+
"name": "storage.hmacKey.get",
88+
"resources": []
89+
},
90+
{
91+
"name": "storage.hmacKey.list",
92+
"resources": []
93+
},
94+
{
95+
"name": "storage.notifications.delete",
96+
"resources": [
97+
"BUCKET",
98+
"NOTIFICATION"
99+
]
100+
},
101+
{
102+
"name": "storage.notifications.get",
103+
"resources": [
104+
"BUCKET",
105+
"NOTIFICATION"
106+
]
107+
},
108+
{
109+
"name": "storage.notifications.list",
110+
"resources": [
111+
"BUCKET",
112+
"NOTIFICATION"
113+
]
114+
},
115+
{
116+
"name": "storage.object_acl.get",
117+
"resources": [
118+
"BUCKET",
119+
"OBJECT"
120+
]
121+
},
122+
{
123+
"name": "storage.object_acl.list",
124+
"resources": [
125+
"BUCKET",
126+
"OBJECT"
127+
]
128+
},
129+
{
130+
"name": "storage.objects.get",
131+
"resources": [
132+
"BUCKET",
133+
"OBJECT"
134+
]
135+
},
136+
{
137+
"name": "storage.objects.list",
138+
"resources": [
139+
"BUCKET",
140+
"OBJECT"
141+
]
142+
},
143+
{
144+
"name": "storage.serviceaccount.get",
145+
"resources": []
146+
}
147+
],
148+
"preconditionProvided": false,
149+
"expectSuccess": true
150+
},
151+
{
152+
"id": 2,
153+
"description": "conditionally idempotent retries when precondition is present",
154+
"cases": [
155+
{
156+
"instructions": [
157+
"return-503",
158+
"return-503"
159+
]
160+
}
161+
],
162+
"methods": [
163+
{
164+
"name": "storage.buckets.patch",
165+
"resources": [
166+
"BUCKET"
167+
]
168+
},
169+
{
170+
"name": "storage.buckets.setIamPolicy",
171+
"resources": [
172+
"BUCKET"
173+
]
174+
},
175+
{
176+
"name": "storage.buckets.update",
177+
"resources": [
178+
"BUCKET"
179+
]
180+
},
181+
{
182+
"name": "storage.hmacKey.update",
183+
"resources": []
184+
},
185+
{
186+
"name": "storage.objects.compose",
187+
"resources": [
188+
"BUCKET",
189+
"OBJECT"
190+
]
191+
},
192+
{
193+
"name": "storage.objects.copy",
194+
"resources": [
195+
"BUCKET",
196+
"OBJECT"
197+
]
198+
},
199+
{
200+
"name": "storage.objects.delete",
201+
"resources": [
202+
"BUCKET",
203+
"OBJECT"
204+
]
205+
},
206+
{
207+
"name": "storage.objects.insert",
208+
"resources": [
209+
"BUCKET"
210+
]
211+
},
212+
{
213+
"name": "storage.objects.patch",
214+
"resources": [
215+
"BUCKET",
216+
"OBJECT"
217+
]
218+
},
219+
{
220+
"name": "storage.objects.rewrite",
221+
"resources": [
222+
"BUCKET",
223+
"OBJECT"
224+
]
225+
},
226+
{
227+
"name": "storage.objects.update",
228+
"resources": [
229+
"BUCKET",
230+
"OBJECT"
231+
]
232+
}
233+
],
234+
"preconditionProvided": true,
235+
"expectSuccess": true
236+
},
237+
{
238+
"id": 3,
239+
"description": "conditionally idempotent no retries when precondition is absent",
240+
"cases": [
241+
{
242+
"instructions": []
243+
}
244+
],
245+
"methods": [],
246+
"preconditionProvided": false,
247+
"expectSuccess": false
248+
},
249+
{
250+
"id": 4,
251+
"description": "non idempotent",
252+
"cases": [
253+
{
254+
"instructions": []
255+
}
256+
],
257+
"methods": [],
258+
"preconditionProvided": false,
259+
"expectSuccess": false
260+
}
261+
]
262+
}

0 commit comments

Comments
 (0)