Skip to content

Commit 2aee8f5

Browse files
dfirovadfirova
authored
docs(samples): Refactoring product package (#480)
1 parent a02d40c commit 2aee8f5

9 files changed

Lines changed: 237 additions & 132 deletions

‎retail/interactive-tutorials/src/main/java/product/ImportProductsBigQueryTable.java‎

Lines changed: 18 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -31,36 +31,30 @@
3131
import com.google.longrunning.Operation;
3232
import com.google.longrunning.OperationsClient;
3333
import java.io.IOException;
34+
import java.util.concurrent.TimeUnit;
3435

3536
public class ImportProductsBigQueryTable {
3637

3738
public static void main(String[] args) throws IOException, InterruptedException {
38-
// TODO(developer): Replace these variables before running the sample.
3939
String projectId = ServiceOptions.getDefaultProjectId();
4040
String branchName =
4141
String.format(
4242
"projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId);
4343
String datasetId = "products";
4444
String tableId = "products";
4545
// TO CHECK ERROR HANDLING USE THE TABLE WITH INVALID PRODUCTS:
46-
// TABLE_ID = "products_some_invalid"
47-
String dataSchema = "product";
48-
// TRY THE FULL RECONCILIATION MODE HERE:
49-
ReconciliationMode reconciliationMode = ReconciliationMode.INCREMENTAL;
46+
// tableId = "products_some_invalid"
5047

51-
ImportProductsRequest importBigQueryRequest =
52-
getImportProductsBigQueryRequest(
53-
reconciliationMode, projectId, datasetId, tableId, dataSchema, branchName);
54-
waitForOperationCompletion(importBigQueryRequest);
48+
importProductsFromBigQuery(projectId, branchName, datasetId, tableId);
5549
}
5650

57-
public static ImportProductsRequest getImportProductsBigQueryRequest(
58-
ReconciliationMode reconciliationMode,
59-
String projectId,
60-
String datasetId,
61-
String tableId,
62-
String dataSchema,
63-
String branchName) {
51+
public static void importProductsFromBigQuery(
52+
String projectId, String branchName, String datasetId, String tableId)
53+
throws IOException, InterruptedException {
54+
// TRY THE FULL RECONCILIATION MODE HERE:
55+
ReconciliationMode reconciliationMode = ReconciliationMode.INCREMENTAL;
56+
String dataSchema = "product";
57+
6458
BigQuerySource bigQuerySource =
6559
BigQuerySource.newBuilder()
6660
.setProjectId(projectId)
@@ -80,30 +74,29 @@ public static ImportProductsRequest getImportProductsBigQueryRequest(
8074
.build();
8175
System.out.printf("Import products from big query table request: %s%n", importRequest);
8276

83-
return importRequest;
84-
}
85-
86-
public static void waitForOperationCompletion(ImportProductsRequest importRequest)
87-
throws IOException, InterruptedException {
77+
// Initialize client that will be used to send requests. This client only
78+
// needs to be created once, and can be reused for multiple requests. After
79+
// completing all of your requests, call the "close" method on the client to
80+
// safely clean up any remaining background resources.
8881
try (ProductServiceClient serviceClient = ProductServiceClient.create()) {
8982
String operationName = serviceClient.importProductsCallable().call(importRequest).getName();
90-
System.out.printf("OperationName = %s\n", operationName);
83+
System.out.printf("OperationName = %s%n", operationName);
9184

9285
OperationsClient operationsClient = serviceClient.getOperationsClient();
9386
Operation operation = operationsClient.getOperation(operationName);
9487

9588
while (!operation.getDone()) {
9689
// Keep polling the operation periodically until the import task is done.
97-
Thread.sleep(30_000);
90+
TimeUnit.SECONDS.sleep(30);
9891
operation = operationsClient.getOperation(operationName);
9992
}
10093

10194
if (operation.hasMetadata()) {
10295
ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class);
10396
System.out.printf(
104-
"Number of successfully imported products: %s\n", metadata.getSuccessCount());
97+
"Number of successfully imported products: %s%n", metadata.getSuccessCount());
10598
System.out.printf(
106-
"Number of failures during the importing: %s\n", metadata.getFailureCount());
99+
"Number of failures during the importing: %s%n", metadata.getFailureCount());
107100
}
108101

109102
if (operation.hasResponse()) {

‎retail/interactive-tutorials/src/main/java/product/ImportProductsGcs.java‎

Lines changed: 43 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020

2121
package product;
2222

23+
import com.google.api.gax.rpc.InvalidArgumentException;
24+
import com.google.api.gax.rpc.PermissionDeniedException;
2325
import com.google.cloud.ServiceOptions;
2426
import com.google.cloud.retail.v2.GcsSource;
2527
import com.google.cloud.retail.v2.ImportErrorsConfig;
@@ -32,37 +34,42 @@
3234
import com.google.longrunning.Operation;
3335
import com.google.longrunning.OperationsClient;
3436
import java.io.IOException;
37+
import java.time.Instant;
3538
import java.util.Collections;
39+
import java.util.concurrent.TimeUnit;
3640

3741
public class ImportProductsGcs {
3842

3943
public static void main(String[] args) throws IOException, InterruptedException {
40-
// TODO(developer): Replace these variables before running the sample.
4144
String projectId = ServiceOptions.getDefaultProjectId();
4245
String branchName =
4346
String.format(
4447
"projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId);
48+
49+
String bucketName = System.getenv("BUCKET_NAME");
50+
String gcsBucket = String.format("gs://%s", bucketName);
51+
String gcsErrorBucket = String.format("%s/errors", gcsBucket);
52+
4553
// To check error handling, use an invalid catalog in request
4654
// branchName =
4755
// String.format("projects/%s/locations/global/catalogs/invalid_catalog/branches/default_branch", projectId);
4856

49-
String gcsBucket = String.format("gs://%s", System.getenv("BUCKET_NAME"));
50-
String gcsErrorBucket = String.format("%s/errors", gcsBucket);
5157
String gcsProductsObject = "products.json";
5258
// To check error handling, use an invalid product JSON.
53-
// gcsProductsObject = "products_some_invalid.json";
59+
// gcsProductsObject = "products_some_invalid.json"
5460

55-
ImportProductsRequest importGcsRequest =
56-
getImportProductsGcsRequest(gcsProductsObject, gcsBucket, gcsErrorBucket, branchName);
57-
waitForOperationCompletion(importGcsRequest);
61+
importProductsFromGcs(branchName, gcsBucket, gcsProductsObject);
5862
}
5963

60-
public static ImportProductsRequest getImportProductsGcsRequest(
61-
String gcsObjectName, String gcsBucket, String gcsErrorBucket, String branchName) {
64+
public static void importProductsFromGcs(
65+
String branchName, String gcsBucket, String gcsProductsObject)
66+
throws IOException, InterruptedException {
67+
String gcsErrorBucket = String.format("%s/errors", gcsBucket);
68+
6269
GcsSource gcsSource =
6370
GcsSource.newBuilder()
6471
.addAllInputUris(
65-
Collections.singleton(String.format("%s/%s", gcsBucket, gcsObjectName)))
72+
Collections.singleton(String.format("%s/%s", gcsBucket, gcsProductsObject)))
6673
.build();
6774

6875
ProductInputConfig inputConfig =
@@ -81,39 +88,53 @@ public static ImportProductsRequest getImportProductsGcsRequest(
8188
.setErrorsConfig(errorsConfig)
8289
.build();
8390

84-
System.out.println("Import products from google cloud source request: " + importRequest);
85-
86-
return importRequest;
87-
}
91+
System.out.printf("Import products from google cloud source request: %s%n", importRequest);
8892

89-
public static void waitForOperationCompletion(ImportProductsRequest importRequest)
90-
throws IOException, InterruptedException {
93+
// Initialize client that will be used to send requests. This client only
94+
// needs to be created once, and can be reused for multiple requests. After
95+
// completing all of your requests, call the "close" method on the client to
96+
// safely clean up any remaining background resources.
9197
try (ProductServiceClient serviceClient = ProductServiceClient.create()) {
9298
String operationName = serviceClient.importProductsCallable().call(importRequest).getName();
93-
System.out.printf("OperationName = %s\n", operationName);
99+
100+
System.out.println("The operation was started.");
101+
System.out.printf("OperationName = %s%n", operationName);
94102

95103
OperationsClient operationsClient = serviceClient.getOperationsClient();
96104
Operation operation = operationsClient.getOperation(operationName);
97105

98-
while (!operation.getDone()) {
99-
// Keep polling the operation periodically until the import task is done.
100-
Thread.sleep(30_000);
106+
Instant deadline = Instant.now().plusSeconds(60);
107+
108+
while (!operation.getDone() || Instant.now().isBefore(deadline)) {
109+
System.out.println("Please wait till operation is done.");
110+
TimeUnit.SECONDS.sleep(30);
101111
operation = operationsClient.getOperation(operationName);
102112
}
103113

104114
if (operation.hasMetadata()) {
105115
ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class);
106116
System.out.printf(
107-
"Number of successfully imported products: %s\n", metadata.getSuccessCount());
117+
"Number of successfully imported products: %s%n", metadata.getSuccessCount());
108118
System.out.printf(
109-
"Number of failures during the importing: %s\n", metadata.getFailureCount());
119+
"Number of failures during the importing: %s%n", metadata.getFailureCount());
120+
} else {
121+
System.out.println("Metadata is empty.");
110122
}
111123

112124
if (operation.hasResponse()) {
113125
ImportProductsResponse response =
114126
operation.getResponse().unpack(ImportProductsResponse.class);
115127
System.out.printf("Operation result: %s%n", response);
128+
} else {
129+
System.out.println("Operation result is empty.");
116130
}
131+
} catch (InvalidArgumentException e) {
132+
System.out.printf(
133+
"%s%n'%s' file does not exist in the bucket. Please "
134+
+ "make sure you have followed the setting up instructions.",
135+
e.getMessage(), gcsProductsObject);
136+
} catch (PermissionDeniedException e) {
137+
System.out.println(e.getMessage());
117138
}
118139
}
119140
}

‎retail/interactive-tutorials/src/main/java/product/ImportProductsInlineSource.java‎

Lines changed: 41 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
package product;
2222

23+
import com.google.api.gax.rpc.InvalidArgumentException;
2324
import com.google.cloud.ServiceOptions;
2425
import com.google.cloud.retail.v2.ColorInfo;
2526
import com.google.cloud.retail.v2.FulfillmentInfo;
@@ -40,24 +41,23 @@
4041
import java.util.Collections;
4142
import java.util.List;
4243
import java.util.UUID;
44+
import java.util.concurrent.TimeUnit;
4345

4446
public class ImportProductsInlineSource {
4547

4648
public static void main(String[] args) throws IOException, InterruptedException {
47-
// TODO(developer): Replace these variables before running the sample.
4849
String projectId = ServiceOptions.getDefaultProjectId();
4950
String branchName =
5051
String.format(
5152
"projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId);
5253

53-
ImportProductsRequest importRequest = getImportProductsInlineRequest(getProducts(), branchName);
54-
waitForOperationCompletion(importRequest);
54+
importProductsInlineSource(branchName);
5555
}
5656

57-
public static ImportProductsRequest getImportProductsInlineRequest(
58-
List<Product> productsToImport, String branchName) {
57+
public static void importProductsInlineSource(String branchName)
58+
throws IOException, InterruptedException {
5959
ProductInlineSource inlineSource =
60-
ProductInlineSource.newBuilder().addAllProducts(productsToImport).build();
60+
ProductInlineSource.newBuilder().addAllProducts(getProducts()).build();
6161

6262
ProductInputConfig inputConfig =
6363
ProductInputConfig.newBuilder().setProductInlineSource(inlineSource).build();
@@ -70,7 +70,41 @@ public static ImportProductsRequest getImportProductsInlineRequest(
7070

7171
System.out.printf("Import products from inline source request: %s%n", importRequest);
7272

73-
return importRequest;
73+
// Initialize client that will be used to send requests. This client only
74+
// needs to be created once, and can be reused for multiple requests. After
75+
// completing all of your requests, call the "close" method on the client to
76+
// safely clean up any remaining background resources.
77+
try (ProductServiceClient serviceClient = ProductServiceClient.create()) {
78+
String operationName = serviceClient.importProductsCallable().call(importRequest).getName();
79+
System.out.printf("OperationName = %s%n", operationName);
80+
81+
OperationsClient operationsClient = serviceClient.getOperationsClient();
82+
Operation operation = operationsClient.getOperation(operationName);
83+
84+
long assuredBreak = System.currentTimeMillis() + 60000; // 60 seconds delay
85+
86+
while (!operation.getDone() || System.currentTimeMillis() < assuredBreak) {
87+
// Keep polling the operation periodically until the import task is done.
88+
TimeUnit.SECONDS.sleep(30);
89+
operation = operationsClient.getOperation(operationName);
90+
}
91+
92+
if (operation.hasMetadata()) {
93+
ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class);
94+
System.out.printf(
95+
"Number of successfully imported products: %s%n", metadata.getSuccessCount());
96+
System.out.printf(
97+
"Number of failures during the importing: %s%n", metadata.getFailureCount());
98+
}
99+
100+
if (operation.hasResponse()) {
101+
ImportProductsResponse response =
102+
operation.getResponse().unpack(ImportProductsResponse.class);
103+
System.out.printf("Operation result: %s%n", response);
104+
}
105+
} catch (InvalidArgumentException e) {
106+
System.out.println(e.getMessage());
107+
}
74108
}
75109

76110
public static List<Product> getProducts() {
@@ -173,35 +207,4 @@ public static List<Product> getProducts() {
173207

174208
return products;
175209
}
176-
177-
public static void waitForOperationCompletion(ImportProductsRequest importRequest)
178-
throws IOException, InterruptedException {
179-
try (ProductServiceClient serviceClient = ProductServiceClient.create()) {
180-
String operationName = serviceClient.importProductsCallable().call(importRequest).getName();
181-
System.out.printf("OperationName = %s\n", operationName);
182-
183-
OperationsClient operationsClient = serviceClient.getOperationsClient();
184-
Operation operation = operationsClient.getOperation(operationName);
185-
186-
while (!operation.getDone()) {
187-
// Keep polling the operation periodically until the import task is done.
188-
Thread.sleep(30_000);
189-
operation = operationsClient.getOperation(operationName);
190-
}
191-
192-
if (operation.hasMetadata()) {
193-
ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class);
194-
System.out.printf(
195-
"Number of successfully imported products: %s\n", metadata.getSuccessCount());
196-
System.out.printf(
197-
"Number of failures during the importing: %s\n", metadata.getFailureCount());
198-
}
199-
200-
if (operation.hasResponse()) {
201-
ImportProductsResponse response =
202-
operation.getResponse().unpack(ImportProductsResponse.class);
203-
System.out.printf("Operation result: %s%n", response);
204-
}
205-
}
206-
}
207210
}

‎retail/interactive-tutorials/src/main/java/product/setup/ProductsCreateBigqueryTable.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,16 +30,15 @@
3030

3131
public class ProductsCreateBigqueryTable {
3232

33-
public static void main(String... args) throws IOException {
33+
public static void main(String[] args) throws IOException {
3434
String dataset = "products";
3535
String validProductsTable = "products";
3636
String invalidProductsTable = "products_some_invalid";
3737
String productSchemaFilePath = "src/main/resources/product_schema.json";
3838
String validProductsSourceFile =
39-
String.format("gs://%s/products.json", ProductsCreateGcsBucket.getBucketName());
39+
String.format("gs://%s/products.json", System.getenv("BUCKET_NAME"));
4040
String invalidProductsSourceFile =
41-
String.format(
42-
"gs://%s/products_some_invalid.json", ProductsCreateGcsBucket.getBucketName());
41+
String.format("gs://%s/products_some_invalid.json", System.getenv("BUCKET_NAME"));
4342

4443
BufferedReader bufferedReader = new BufferedReader(new FileReader(productSchemaFilePath));
4544
String jsonToString = bufferedReader.lines().collect(Collectors.joining());

‎retail/interactive-tutorials/src/main/java/product/setup/ProductsCreateGcsBucket.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,16 +38,20 @@ public class ProductsCreateGcsBucket {
3838
String.format("%s_products_%s", PROJECT_ID, CURRENT_DATE.getSeconds());
3939

4040
public static void main(String... args) throws IOException {
41-
createBucket(BUCKET_NAME);
42-
System.out.printf("Products gcs bucket %s was created.", BUCKET_NAME);
41+
createGcsBucketAndUploadData(BUCKET_NAME);
42+
}
43+
44+
public static void createGcsBucketAndUploadData(String bucketName) throws IOException {
45+
createBucket(bucketName);
46+
System.out.printf("Products gcs bucket %s was created.%n", bucketName);
4347

44-
uploadObject(BUCKET_NAME, "products.json", "src/main/resources/products.json");
45-
System.out.printf("File 'products.json' was uploaded into bucket '%s'.", BUCKET_NAME);
48+
uploadObject(bucketName, "products.json", "src/main/resources/products.json");
49+
System.out.printf("File 'products.json' was uploaded into bucket '%s'.%n", bucketName);
4650

4751
uploadObject(
48-
BUCKET_NAME, "products_some_invalid.json", "src/main/resources/products_some_invalid.json");
52+
bucketName, "products_some_invalid.json", "src/main/resources/products_some_invalid.json");
4953
System.out.printf(
50-
"File 'products_some_invalid.json' was uploaded into bucket '%s'.", BUCKET_NAME);
54+
"File 'products_some_invalid.json' was uploaded into bucket '%s'.%n", bucketName);
5155
}
5256

5357
public static String getBucketName() {

0 commit comments

Comments
 (0)