diff --git a/benchmarks/scripts/BUILD b/benchmarks/scripts/BUILD index f5693b352..0b965b389 100644 --- a/benchmarks/scripts/BUILD +++ b/benchmarks/scripts/BUILD @@ -19,11 +19,19 @@ licenses(["notice"]) # Export for the PIP package. exports_files(["__init__.py"]) +py_library( + name = "benchmark_flags", + srcs = ["flags.py"], +) + py_test( name = "benchmark_clifford_circuit", srcs = ["benchmark_clifford_circuit.py"], python_version = "PY3", deps = [ + ":benchmark_flags", + ":benchmark_util", + "//benchmarks/scripts/models:random_clifford_circuit", "//tensorflow_quantum/core/ops:tfq_simulate_ops_py", "//tensorflow_quantum/core/serialize:serializer", "@local_config_tf//:test_log_pb2", @@ -35,6 +43,8 @@ py_test( srcs = ["benchmark_random_circuit.py"], python_version = "PY3", deps = [ + ":benchmark_flags", + ":benchmark_util", "//tensorflow_quantum/core/ops:tfq_simulate_ops_py", "//tensorflow_quantum/core/serialize:serializer", "@local_config_tf//:test_log_pb2", @@ -46,6 +56,8 @@ py_test( srcs = ["benchmark_op_gradients.py"], python_version = "PY3", deps = [ + ":benchmark_flags", + ":benchmark_util", "//tensorflow_quantum/core/ops:batch_util", "//tensorflow_quantum/core/ops:cirq_ops", "//tensorflow_quantum/core/ops:tfq_simulate_ops_py", diff --git a/benchmarks/scripts/benchmark_op_gradients.py b/benchmarks/scripts/benchmark_op_gradients.py index d90d57c83..50f42400f 100644 --- a/benchmarks/scripts/benchmark_op_gradients.py +++ b/benchmarks/scripts/benchmark_op_gradients.py @@ -127,7 +127,7 @@ def setup(self): for resolver in resolver_batch], dtype=np.float32) - self.symbol_names = symbol_names + self.symbol_names_tensor = tf.convert_to_tensor(symbol_names) self.symbol_values_tensor = tf.convert_to_tensor(symbol_values_array) self.programs = util.convert_to_tensor(circuit_batch) self.psums = util.convert_to_tensor([psums]) @@ -140,15 +140,15 @@ def _benchmark_tfq_differentiator(self, differentiator, params): analytic_op=tfq_simulate_ops.tfq_simulate_expectation) for _ in range(params.n_burn): - op(self.programs, self.symbol_names, self.symbol_values_tensor, - self.psums) + op(self.programs, self.symbol_names_tensor, + self.symbol_values_tensor, self.psums) deltas = [None] * params.n_runs for i in range(params.n_runs): start = time.perf_counter() with tf.GradientTape() as g: g.watch(self.symbol_values_tensor) - expectations = op(self.programs, self.symbol_names, + expectations = op(self.programs, self.symbol_names_tensor, self.symbol_values_tensor, self.psums) g.gradient(expectations, self.symbol_values_tensor) deltas[i] = time.perf_counter() - start diff --git a/benchmarks/scripts/benchmark_random_circuit.py b/benchmarks/scripts/benchmark_random_circuit.py index 6dee586b7..6a68818f4 100644 --- a/benchmarks/scripts/benchmark_random_circuit.py +++ b/benchmarks/scripts/benchmark_random_circuit.py @@ -1,148 +1,233 @@ -# Copyright 2020 The TensorFlow Quantum Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Benchmark simulators against classically intractable 'supremacy' circuits.""" -import os -import time - -from absl.testing import parameterized -import cirq -import tensorflow as tf -import numpy as np - -from tensorflow_quantum.core.ops import tfq_simulate_ops -from tensorflow_quantum.core.serialize.serializer import serialize_circuit -from benchmarks.scripts import flags -from benchmarks.scripts import benchmark_util - -SEED = 63536323 -SRC = os.path.dirname(os.path.realpath(__file__)) -os.environ['TEST_REPORT_FILE_PREFIX'] = os.path.join(SRC, 'reports/') -TEST_PARAMS_1 = flags.test_flags(n_rows=3, n_cols=5, n_moments=5) -TEST_PARAMS_2 = flags.test_flags(n_rows=4, n_cols=4, n_moments=20) - - -def make_random_circuit(n_rows, n_cols, depth): - """Generate a random unparameterized circuit of fixed depth.""" - return cirq.experiments.generate_boixo_2018_supremacy_circuits_v2_grid( - n_rows=n_rows, - n_cols=n_cols, - cz_depth=depth - 2, # Account for beginning/ending Hadamard layers - seed=SEED) - - -class RandomCircuitBenchmarksTest(tf.test.TestCase, parameterized.TestCase): - """Test the random circuit benchmarking class.""" - - @parameterized.named_parameters( - ("params_1", TEST_PARAMS_1), - ("params_2", TEST_PARAMS_2), - ) - def test_benchmark_random_circuit(self, params): - """Test that Op constructs and runs correctly.""" - proto_file_path = os.path.join( - SRC, "reports/", "RandomCircuitBenchmarks.benchmark_random_circuit_" - f"{params.n_rows}_{params.n_cols}_{params.n_moments}") - self.addCleanup(os.remove, proto_file_path) - - bench = RandomCircuitBenchmarks(params=params) - bench.benchmark_random_circuit() - - res = benchmark_util.read_benchmark_entry(proto_file_path) - self.assertEqual( - res.name, "RandomCircuitBenchmarks.benchmark_random_circuit_" - f"{params.n_rows}_{params.n_cols}_{params.n_moments}") - self.assertEqual(res.extras.get("n_rows").double_value, params.n_rows) - self.assertEqual(res.extras.get("n_cols").double_value, params.n_cols) - self.assertEqual( - res.extras.get("n_moments").double_value, params.n_moments) - - assert hasattr(res, 'iters') - assert hasattr(res, 'wall_time') - - @parameterized.named_parameters( - ("params_1", TEST_PARAMS_1), - ("params_2", TEST_PARAMS_2), - ) - def test_random_circuit_params(self, params): - """Ensure that the random circuits are structured as advertised.""" - circuit = make_random_circuit(params.n_rows, params.n_cols, - params.n_moments) - self.assertEqual(len(circuit), params.n_moments) - self.assertEqual(len(circuit.all_qubits()), - params.n_rows * params.n_cols) - - -class RandomCircuitBenchmarks(tf.test.Benchmark): - """Benchmark simulators against random 'supremacy' circuits. - - Flags: - --n_rows --n_cols --n_moments --batch_size --n_runs --n_burn - """ - - def __init__(self, params=None): - """Pull in command line flags or use provided flags.""" - super().__init__() - # Allow input params for testing purposes. - self.params = params if params else flags.FLAGS - - def _simulate_circuit(self, circuit, params): - # TODO: implement backend switch - return tfq_simulate_ops.tfq_simulate_state( - [str(serialize_circuit(circuit))] * params.batch_size, ["None"], - [[0]] * params.batch_size) - - def benchmark_random_circuit(self): - """Benchmark simulator performance on - a classically intractable circuit.""" - - circuit = make_random_circuit(self.params.n_rows, self.params.n_cols, - self.params.n_moments) - for _ in range(self.params.n_burn): - _ = self._simulate_circuit(circuit, self.params) - - deltas = [None] * self.params.n_runs - for i in range(self.params.n_runs): - start = time.perf_counter() - _ = self._simulate_circuit(circuit, self.params) - deltas[i] = time.perf_counter() - start - - extras = { - 'n_rows': self.params.n_rows, - 'n_cols': self.params.n_cols, - 'n_qubits': len(circuit.all_qubits()), - 'n_moments': self.params.n_moments, - 'batch_size': self.params.batch_size, - "min_time": min(deltas), - } - - name = (f"benchmark_random_circuit_{self.params.n_rows}_" - f"{self.params.n_cols}_{self.params.n_moments}") - full_path = os.path.join(os.environ['TEST_REPORT_FILE_PREFIX'], - f"{self.__class__.__name__}.{name}") - if os.path.exists(full_path): - os.remove(full_path) - - benchmark_values = { - "iters": self.params.n_runs, - "wall_time": np.median(deltas), - "extras": extras, - "name": name, - } - self.report_benchmark(**benchmark_values) - return benchmark_values - - -if __name__ == "__main__": - tf.test.main() +# Copyright 2020 The TensorFlow Quantum Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Benchmark simulators against classically intractable 'supremacy' circuits.""" +import os +import random +import time +from typing import Callable, Iterable, Sequence, TypeVar, cast + +import numpy as np +from absl.testing import parameterized +import cirq +import tensorflow as tf + +from tensorflow_quantum.core.ops import tfq_simulate_ops +from tensorflow_quantum.core.serialize.serializer import serialize_circuit +from benchmarks.scripts import flags +from benchmarks.scripts import benchmark_util + +SEED = 63536323 +SRC = os.path.dirname(os.path.realpath(__file__)) +os.environ['TEST_REPORT_FILE_PREFIX'] = os.path.join(SRC, 'reports/') +TEST_PARAMS_1 = flags.test_flags(n_rows=3, n_cols=5, n_moments=5) +TEST_PARAMS_2 = flags.test_flags(n_rows=4, n_cols=4, n_moments=20) + +T = TypeVar('T') + + +def _choice(rand_gen: Callable[[], float], sequence: Sequence[T]) -> T: + """Choose a pseudo-random element from a non-empty sequence.""" + # Keep ReCirq's float-based selection to preserve seeded circuit generation. + return sequence[int(rand_gen() * len(sequence))] + + +def _make_cz_layer(qubits: Iterable[cirq.GridQubit], layer_index: int): + """Yield a CZ interaction pattern for the given layer index.""" + offset = layer_index % 8 + for q in qubits: + for q2 in [ + cirq.GridQubit(q.row + 1, q.col), + cirq.GridQubit(q.row, q.col + 1) + ]: + if q2 in qubits and ((q.row + q.col + offset) % 2 == 0): + yield cirq.CZ(q, q2) + + +def _add_cz_layer(layer_index: int, qubits: Sequence[cirq.GridQubit], + circuit: cirq.Circuit) -> int: + """Add the next non-empty CZ layer and return the updated layer index.""" + cz_layer = None + while not cz_layer: + cz_layer = list(_make_cz_layer(qubits, layer_index)) + layer_index += 1 + + circuit.append(cz_layer, strategy=cirq.InsertStrategy.NEW_THEN_INLINE) + return layer_index + + +def generate_boixo_2018_beyond_classical_v2_grid(n_rows: int, n_cols: int, + cz_depth: int, + seed: int) -> cirq.Circuit: + """Local copy of ReCirq's v2 beyond-classical grid circuit generator. + + Source reference: + https://github.com/quantumlib/ReCirq/blob/main/recirq/beyond_classical/google_v2_beyond_classical.py + + Note: + We intentionally keep this local copy to avoid introducing broader + dependency migration. A future cleanup can switch to direct + ReCirq dependency once repository constraints are aligned. + """ + qubits = [ + cirq.GridQubit(i, j) for i in range(n_rows) for j in range(n_cols) + ] + non_diagonal_gates = [cirq.X**(1 / 2), cirq.Y**(1 / 2)] + rand_gen = random.Random(seed).random + + circuit = cirq.Circuit() + circuit.append(cirq.H(qubit) for qubit in qubits) + + layer_index = 0 + if cz_depth: + layer_index = _add_cz_layer(layer_index, qubits, circuit) + for qubit in qubits: + if not circuit.operation_at(qubit, 1): + circuit.append(cirq.T(qubit), + strategy=cirq.InsertStrategy.EARLIEST) + + for moment_index in range(2, cz_depth + 1): + layer_index = _add_cz_layer(layer_index, qubits, circuit) + for qubit in qubits: + if not circuit.operation_at(qubit, moment_index): + last_op = circuit.operation_at(qubit, moment_index - 1) + if last_op: + gate = cast(cirq.GateOperation, last_op).gate + if gate == cirq.CZ: + circuit.append(_choice(rand_gen, + non_diagonal_gates).on(qubit), + strategy=cirq.InsertStrategy.EARLIEST) + elif gate != cirq.T: + circuit.append(cirq.T(qubit), + strategy=cirq.InsertStrategy.EARLIEST) + + circuit.append([cirq.H(qubit) for qubit in qubits], + strategy=cirq.InsertStrategy.NEW_THEN_INLINE) + return circuit + + +def make_random_circuit(n_rows, n_cols, depth): + """Generate a random unparameterized circuit of fixed depth.""" + circuit = generate_boixo_2018_beyond_classical_v2_grid( + n_rows=n_rows, + n_cols=n_cols, + cz_depth=max(0, depth - 2), # Account for initial/final Hadamards. + seed=SEED) + return cirq.Circuit(circuit[:depth]) + + +class RandomCircuitBenchmarksTest(tf.test.TestCase, parameterized.TestCase): + """Test the random circuit benchmarking class.""" + + @parameterized.named_parameters( + ("params_1", TEST_PARAMS_1), + ("params_2", TEST_PARAMS_2), + ) + def test_benchmark_random_circuit(self, params): + """Test that Op constructs and runs correctly.""" + proto_file_path = os.path.join( + SRC, "reports/", "RandomCircuitBenchmarks.benchmark_random_circuit_" + f"{params.n_rows}_{params.n_cols}_{params.n_moments}") + self.addCleanup(os.remove, proto_file_path) + + bench = RandomCircuitBenchmarks(params=params) + bench.benchmark_random_circuit() + + res = benchmark_util.read_benchmark_entry(proto_file_path) + self.assertEqual( + res.name, "RandomCircuitBenchmarks.benchmark_random_circuit_" + f"{params.n_rows}_{params.n_cols}_{params.n_moments}") + self.assertEqual(res.extras.get("n_rows").double_value, params.n_rows) + self.assertEqual(res.extras.get("n_cols").double_value, params.n_cols) + self.assertEqual( + res.extras.get("n_moments").double_value, params.n_moments) + + assert hasattr(res, 'iters') + assert hasattr(res, 'wall_time') + + @parameterized.named_parameters( + ("params_1", TEST_PARAMS_1), + ("params_2", TEST_PARAMS_2), + ) + def test_random_circuit_params(self, params): + """Ensure that the random circuits are structured as advertised.""" + circuit = make_random_circuit(params.n_rows, params.n_cols, + params.n_moments) + self.assertEqual(len(circuit), params.n_moments) + self.assertEqual(len(circuit.all_qubits()), + params.n_rows * params.n_cols) + + +class RandomCircuitBenchmarks(tf.test.Benchmark): + """Benchmark simulators against random 'supremacy' circuits. + + Flags: + --n_rows --n_cols --n_moments --batch_size --n_runs --n_burn + """ + + def __init__(self, params=None): + """Pull in command line flags or use provided flags.""" + super().__init__() + # Allow input params for testing purposes. + self.params = params if params else flags.FLAGS + + def _simulate_circuit(self, circuit, params): + # TODO: implement backend switch + return tfq_simulate_ops.tfq_simulate_state( + [str(serialize_circuit(circuit))] * params.batch_size, ["None"], + [[0]] * params.batch_size) + + def benchmark_random_circuit(self): + """Benchmark simulator performance on + a classically intractable circuit.""" + + circuit = make_random_circuit(self.params.n_rows, self.params.n_cols, + self.params.n_moments) + for _ in range(self.params.n_burn): + _ = self._simulate_circuit(circuit, self.params) + + deltas = [None] * self.params.n_runs + for i in range(self.params.n_runs): + start = time.perf_counter() + _ = self._simulate_circuit(circuit, self.params) + deltas[i] = time.perf_counter() - start + + extras = { + 'n_rows': self.params.n_rows, + 'n_cols': self.params.n_cols, + 'n_qubits': len(circuit.all_qubits()), + 'n_moments': self.params.n_moments, + 'batch_size': self.params.batch_size, + "min_time": min(deltas), + } + + name = (f"benchmark_random_circuit_{self.params.n_rows}_" + f"{self.params.n_cols}_{self.params.n_moments}") + full_path = os.path.join(os.environ['TEST_REPORT_FILE_PREFIX'], + f"{self.__class__.__name__}.{name}") + if os.path.exists(full_path): + os.remove(full_path) + + benchmark_values = { + "iters": self.params.n_runs, + "wall_time": np.median(deltas), + "extras": extras, + "name": name, + } + self.report_benchmark(**benchmark_values) + return benchmark_values + + +if __name__ == "__main__": + tf.test.main() diff --git a/benchmarks/scripts/models/BUILD b/benchmarks/scripts/models/BUILD index 4036827f4..3bcf8a031 100644 --- a/benchmarks/scripts/models/BUILD +++ b/benchmarks/scripts/models/BUILD @@ -19,10 +19,9 @@ licenses(["notice"]) # Export for the PIP package. exports_files(["__init__.py"]) -py_binary( +py_library( name = "random_clifford_circuit", srcs = ["random_clifford_circuit.py"], - python_version = "PY3", ) py_test( diff --git a/scripts/test_benchmarks.sh b/scripts/test_benchmarks.sh index a37d31ec0..f4f915f34 100755 --- a/scripts/test_benchmarks.sh +++ b/scripts/test_benchmarks.sh @@ -14,16 +14,12 @@ # limitations under the License. # ============================================================================== echo "Testing all Benchmarks."; -bazel test -c opt --cxxopt="-msse2" --cxxopt="-msse3" --cxxopt="-msse4" --test_output=errors $(bazel query //benchmarks/scripts:all) -# test_outputs=$(bazel test -c opt --cxxopt="-msse2" --cxxopt="-msse3" --cxxopt="-msse4" --test_output=errors $(bazel query //benchmarks/scripts:all)) -bench_outputs=$() -# bench_outputs=$(bazel run -c opt --cxxopt="-msse2" --cxxopt="-msse3" --cxxopt="-msse4" --test_output=errors //benchmarks/scripts:benchmark_clifford_circuit) +bazel test -c opt --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=1" --cxxopt="-msse2" --cxxopt="-msse3" --cxxopt="-msse4" --test_output=errors $(bazel query //benchmarks/scripts:all) exit_code=$? if [ "$exit_code" == "0" ]; then echo "Testing Complete!"; exit 0; else echo "Testing failed, please correct errors before proceeding." - echo "{$test_outputs}" exit 64; fi