Skip to content

Commit d23d4d0

Browse files
JCEmmonsJCEmmons
committed
Add support for JSON structured keys - Fixes #64 
1 parent 9e38d24 commit d23d4d0

4 files changed

Lines changed: 256 additions & 44 deletions

File tree

‎gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/JsonResource.java‎

Lines changed: 145 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright IBM Corp. 2015, 2016
2+
* Copyright IBM Corp. 2015, 2017
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -22,59 +22,105 @@
2222
import java.io.OutputStream;
2323
import java.io.OutputStreamWriter;
2424
import java.nio.charset.StandardCharsets;
25+
import java.text.StringCharacterIterator;
26+
import java.util.ArrayList;
27+
import java.util.Collections;
28+
import java.util.List;
2529
import java.util.Map;
2630
import java.util.TreeSet;
31+
import java.util.regex.Matcher;
32+
import java.util.regex.Pattern;
2733

2834
import com.google.gson.GsonBuilder;
35+
import com.google.gson.JsonArray;
2936
import com.google.gson.JsonElement;
37+
import com.google.gson.JsonNull;
3038
import com.google.gson.JsonObject;
3139
import com.google.gson.JsonParseException;
3240
import com.google.gson.JsonParser;
41+
import com.google.gson.JsonPrimitive;
42+
import com.google.gson.stream.JsonToken;
3343
import com.ibm.g11n.pipeline.resfilter.ResourceString.ResourceStringComparator;
3444

3545
/**
3646
* JSON resource filter implementation.
3747
*
38-
* @author Yoshito Umaoka
48+
* @author Yoshito Umaoka, John Emmons
3949
*/
4050
public class JsonResource implements ResourceFilter {
4151

52+
private class KeyPiece {
53+
String keyValue;
54+
JsonToken keyType;
55+
56+
KeyPiece(String keyValue, JsonToken keyType) {
57+
this.keyValue = keyValue;
58+
this.keyType = keyType;
59+
}
60+
}
61+
4262
@Override
4363
public Bundle parse(InputStream inStream) throws IOException {
4464
Bundle bundle = new Bundle();
4565
try (InputStreamReader reader = new InputStreamReader(new BomInputStream(inStream), StandardCharsets.UTF_8)) {
4666
JsonElement root = new JsonParser().parse(reader);
4767
if (!root.isJsonObject()) {
4868
throw new IllegalResourceFormatException("The root JSON element is not an JSON object.");
49-
}
50-
addBundleStrings(root.getAsJsonObject(),"", bundle, 0);
69+
}
70+
addBundleStrings(root.getAsJsonObject(), "", bundle, 0);
5171
} catch (JsonParseException e) {
5272
throw new IllegalResourceFormatException("Failed to parse the specified JSON contents.", e);
5373
}
5474
return bundle;
5575
}
5676

57-
private int addBundleStrings( JsonObject obj, String keyPrefix, Bundle bundle, int sequenceNum) {
77+
private int addBundleStrings(JsonObject obj, String keyPrefix, Bundle bundle, int sequenceNum) {
5878
for (Map.Entry<String, JsonElement> entry : obj.entrySet()) {
5979
String key = entry.getKey();
6080
JsonElement value = entry.getValue();
6181
if (value.isJsonObject()) {
62-
String newKeyPrefix;
63-
if (keyPrefix.isEmpty()) {
64-
newKeyPrefix = "$." + key + ".";
65-
} else {
66-
newKeyPrefix = keyPrefix + key + ".";
82+
sequenceNum = addBundleStrings(value.getAsJsonObject(), modifiedKeyPrefix(keyPrefix,key,"$.") , bundle, sequenceNum);
83+
} else if (value.isJsonArray()) {
84+
JsonArray ar = value.getAsJsonArray();
85+
for (int i = 0; i < ar.size(); i++) {
86+
JsonElement arrayEntry = ar.get(i);
87+
if (arrayEntry.isJsonPrimitive() && arrayEntry.getAsJsonPrimitive().isString()) {
88+
sequenceNum++;
89+
bundle.addResourceString(modifiedKeyPrefix(keyPrefix,key,"$.") + "[" + Integer.toString(i) + "]",
90+
arrayEntry.getAsString(), sequenceNum);
91+
92+
} else {
93+
sequenceNum = addBundleStrings(arrayEntry.getAsJsonObject(),
94+
modifiedKeyPrefix(keyPrefix,key,"$.") + "[" + Integer.toString(i) + "]", bundle, sequenceNum);
95+
}
6796
}
68-
sequenceNum = addBundleStrings(value.getAsJsonObject(),newKeyPrefix,bundle,sequenceNum);
6997
} else if (!value.isJsonPrimitive() || !value.getAsJsonPrimitive().isString()) {
7098
throw new IllegalResourceFormatException("The value of JSON element " + key + " is not a string.");
7199
} else {
72100
sequenceNum++;
73-
bundle.addResourceString(keyPrefix+key, value.getAsString(), sequenceNum);
101+
bundle.addResourceString(modifiedKeyPrefix(keyPrefix,key,""), value.getAsString(), sequenceNum);
74102
}
75103
}
76104
return sequenceNum;
77105
}
106+
107+
private String modifiedKeyPrefix( String keyPrefix, String key, String addPrefixIfEmpty) {
108+
109+
final Pattern specialSequences = Pattern.compile("[.'\\[\\]]");
110+
if (key.isEmpty()) {
111+
return keyPrefix;
112+
}
113+
if (specialSequences.matcher(key).find(0)) {
114+
String modifiedKey = key.replaceAll("'", "\\\\u0027");
115+
return keyPrefix + "['" + modifiedKey + "']";
116+
} else {
117+
if (keyPrefix.isEmpty()) {
118+
return addPrefixIfEmpty + key;
119+
}
120+
return keyPrefix + "." + key;
121+
}
122+
}
123+
78124
@Override
79125
public void write(OutputStream outStream, String language, Bundle bundle) throws IOException {
80126
// extracts key value pairs in original sequence order
@@ -83,19 +129,47 @@ public void write(OutputStream outStream, String language, Bundle bundle) throws
83129
JsonObject output = new JsonObject();
84130
for (ResourceString res : sortedResources) {
85131
String key = res.getKey();
86-
if (key.startsWith("$.")) {
87-
key = key.substring(2);
88-
}
89-
String[] keyPieces = key.split("\\.");
90-
JsonObject current = output;
91-
for (int i = 0 ; i < keyPieces.length ; i++ ) {
92-
if ( i + 1 < keyPieces.length ) { // There is structure under this key piece
93-
if (!current.has(keyPieces[i])) {
94-
current.add(keyPieces[i],new JsonObject());
132+
List<KeyPiece> keyPieces = splitKeyPieces(key);
133+
JsonElement current = output;
134+
for (int i = 0; i < keyPieces.size(); i++) {
135+
if (i + 1 < keyPieces.size()) { // There is structure under this key piece
136+
if (current.isJsonObject()) {
137+
JsonObject currentObject = current.getAsJsonObject();
138+
if (!currentObject.has(keyPieces.get(i).keyValue)) {
139+
if (keyPieces.get(i + 1).keyType == JsonToken.BEGIN_ARRAY) {
140+
currentObject.add(keyPieces.get(i).keyValue, new JsonArray());
141+
} else {
142+
currentObject.add(keyPieces.get(i).keyValue, new JsonObject());
143+
}
144+
}
145+
current = currentObject.get(keyPieces.get(i).keyValue);
146+
} else {
147+
JsonArray currentArray = current.getAsJsonArray();
148+
Integer idx = Integer.valueOf(keyPieces.get(i).keyValue);
149+
for ( int arrayIndex = currentArray.size(); arrayIndex <= idx ; arrayIndex++) {
150+
currentArray.add(JsonNull.INSTANCE);
151+
}
152+
if (currentArray.get(idx).isJsonNull()) {
153+
if (keyPieces.get(i + 1).keyType == JsonToken.BEGIN_ARRAY) {
154+
currentArray.set(idx, new JsonArray());
155+
} else {
156+
currentArray.set(idx, new JsonObject());
157+
}
158+
}
159+
current = currentArray.get(idx);
95160
}
96-
current = current.getAsJsonObject(keyPieces[i]);
97161
} else { // This is the leaf node
98-
current.addProperty(keyPieces[i], res.getValue());
162+
if (keyPieces.get(i).keyType == JsonToken.BEGIN_ARRAY) {
163+
JsonArray currentArray = current.getAsJsonArray();
164+
Integer idx = Integer.valueOf(keyPieces.get(i).keyValue);
165+
JsonPrimitive e = new JsonPrimitive(res.getValue());
166+
for ( int arrayIndex = currentArray.size(); arrayIndex <= idx ; arrayIndex++) {
167+
currentArray.add(JsonNull.INSTANCE);
168+
}
169+
current.getAsJsonArray().set(idx, e);
170+
} else {
171+
current.getAsJsonObject().addProperty(keyPieces.get(i).keyValue, res.getValue());
172+
}
99173
}
100174
}
101175
}
@@ -105,10 +179,55 @@ public void write(OutputStream outStream, String language, Bundle bundle) throws
105179
}
106180
}
107181

182+
private List<KeyPiece> splitKeyPieces(String key) {
183+
List<KeyPiece> result = new ArrayList<KeyPiece>();
184+
Matcher onlyDigits = Pattern.compile("^\\d+$").matcher("");
185+
// Disregard $. at the beginning - it's not really part of the key...
186+
List<String> tokens = findTokens(key.startsWith("$.") ? key.substring(2) : key);
187+
for (String s : tokens) {
188+
if (s.startsWith("'")) {
189+
// Turn any "\u0027" in the key back into '
190+
String modifiedKeyPiece = s.substring(1, s.length() - 1).replaceAll("\\\\u0027", "'");
191+
result.add(new KeyPiece(modifiedKeyPiece, JsonToken.BEGIN_OBJECT));
192+
} else if (onlyDigits.reset(s).matches()) {
193+
result.add(new KeyPiece(s, JsonToken.BEGIN_ARRAY));
194+
} else {
195+
for (String s2 : s.split("\\.")) {
196+
if (!s2.isEmpty()) {
197+
result.add(new KeyPiece(s2, JsonToken.BEGIN_OBJECT));
198+
}
199+
}
200+
}
201+
}
202+
return Collections.unmodifiableList(result);
203+
}
204+
205+
private static List<String> findTokens(String data) {
206+
List<String> tokens = new ArrayList<String>();
207+
boolean inQuotes = false;
208+
StringBuilder currentToken = new StringBuilder();
209+
StringCharacterIterator i = new StringCharacterIterator(data);
210+
while (i.current() != StringCharacterIterator.DONE) {
211+
char c = i.current();
212+
if ( c == '\'' ) {
213+
inQuotes = !inQuotes;
214+
}
215+
if (!inQuotes && ( c == '.' || c == '[' || c == ']')) {
216+
tokens.add(currentToken.toString());
217+
currentToken.setLength(0);
218+
} else {
219+
currentToken.append(c);
220+
}
221+
i.next();
222+
}
223+
tokens.add(currentToken.toString());
224+
return Collections.unmodifiableList(tokens);
225+
}
226+
108227
@Override
109-
public void merge(InputStream base, OutputStream outStream, String language, Bundle bundle)
110-
throws IOException {
111-
//TODO: Add merge implementation here. For now, fallback to write() operation.
228+
public void merge(InputStream base, OutputStream outStream, String language, Bundle bundle) throws IOException {
229+
// TODO: Add merge implementation here. For now, fallback to write()
230+
// operation.
112231
write(outStream, language, bundle);
113232
}
114233
}

‎gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/JsonResourceTest.java‎

Lines changed: 48 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright IBM Corp. 2016
2+
* Copyright IBM Corp. 2016, 2017
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -30,7 +30,6 @@
3030
import java.util.LinkedList;
3131
import java.util.List;
3232

33-
import org.junit.Ignore;
3433
import org.junit.Test;
3534

3635
import com.ibm.g11n.pipeline.resfilter.ResourceString.ResourceStringComparator;
@@ -44,17 +43,35 @@ public class JsonResourceTest {
4443

4544
private static final File EXPECTED_WRITE_FILE = new File("src/test/resource/resfilter/json/write-output.json");
4645

47-
private static final File EXPECTED_MERGE_FILE = new File("src/test/resource/resfilter/json/merge-output.json");
48-
4946
private static final Collection<ResourceString> EXPECTED_INPUT_RES_LIST;
5047

5148
static {
5249
List<ResourceString> lst = new LinkedList<>();
5350
lst.add(new ResourceString("$.bears.grizzly.brown", "Brown Bear", 1));
5451
lst.add(new ResourceString("$.bears.grizzly.black", "Black Bear", 2));
5552
lst.add(new ResourceString("$.bears.white", "Polar Bear", 3));
56-
lst.add(new ResourceString("frog 2", "Red-eyed Tree Frog", 4));
57-
lst.add(new ResourceString("owl 3", "Great Horned Owl", 5));
53+
lst.add(new ResourceString("$.countries[0].Europe[0]", "Germany", 4));
54+
lst.add(new ResourceString("$.countries[0].Europe[1]", "Italy", 5));
55+
lst.add(new ResourceString("$.countries[0].Europe[2]", "France", 6));
56+
lst.add(new ResourceString("$.countries[0].Europe[3]", "Spain", 7));
57+
lst.add(new ResourceString("$.countries[1].Asia[0]", "China", 8));
58+
lst.add(new ResourceString("$.countries[1].Asia[1]", "Japan", 9));
59+
lst.add(new ResourceString("$.countries[1].Asia[2]", "India", 10));
60+
lst.add(new ResourceString("$.countries[2].Americas['S. America'][0]", "Brazil", 11));
61+
lst.add(new ResourceString("$.countries[2].Americas['S. America'][1]", "Venezuela", 12));
62+
lst.add(new ResourceString("$.countries[2].Americas['N. America'][0]", "United States [USA]", 13));
63+
lst.add(new ResourceString("$.countries[2].Americas['N. America'][1]", "Canada", 14));
64+
lst.add(new ResourceString("$.countries[2].Americas['N. America'][2]", "Mexico", 15));
65+
lst.add(new ResourceString("$.countries[3].Africa[0]", "Egypt", 16));
66+
lst.add(new ResourceString("$.countries[3].Africa[1]", "Somalia", 17));
67+
lst.add(new ResourceString("$.countries[3].Africa[2]", "S. Africa", 18));
68+
lst.add(new ResourceString("$.colors[0]", "red", 19));
69+
lst.add(new ResourceString("$.colors[1]", "blue", 20));
70+
lst.add(new ResourceString("$.colors[2]", "yellow", 21));
71+
lst.add(new ResourceString("$.colors[3]", "orange", 22));
72+
lst.add(new ResourceString("['frog[\\u00272\\u0027]']", "Red-eyed Tree Frog", 23));
73+
lst.add(new ResourceString("['owl[3]']", "Great Horned Owl", 24));
74+
lst.add(new ResourceString("some_text", "Just a plain old string", 25));
5875

5976
Collections.sort(lst, new ResourceStringComparator());
6077
EXPECTED_INPUT_RES_LIST = lst;
@@ -64,11 +81,31 @@ public class JsonResourceTest {
6481

6582
static {
6683
WRITE_BUNDLE = new Bundle();
67-
WRITE_BUNDLE.addResourceString("owl 3", "Great Horned Owl - translated", 5);
68-
WRITE_BUNDLE.addResourceString("$.bears.grizzly.brown", "Brown Bear - translated", 1);
69-
WRITE_BUNDLE.addResourceString("$.bears.grizzly.black", "Black Bear - translated", 2);
70-
WRITE_BUNDLE.addResourceString("$.bears.white", "Polar Bear - translated", 3);
71-
WRITE_BUNDLE.addResourceString("frog 2", "Red-eyed Tree Frog - translated", 4);
84+
WRITE_BUNDLE.addResourceString("$.bears.grizzly.brown", "Brown Bear - XL", 1);
85+
WRITE_BUNDLE.addResourceString("$.bears.grizzly.black", "Black Bear - XL", 2);
86+
WRITE_BUNDLE.addResourceString("$.bears.white", "Polar Bear - XL", 3);
87+
WRITE_BUNDLE.addResourceString("$.countries[0].Europe[0]", "Germany - XL", 4);
88+
WRITE_BUNDLE.addResourceString("$.countries[0].Europe[1]", "Italy - XL", 5);
89+
WRITE_BUNDLE.addResourceString("$.countries[0].Europe[2]", "France - XL", 6);
90+
WRITE_BUNDLE.addResourceString("$.countries[0].Europe[3]", "Spain - XL", 7);
91+
WRITE_BUNDLE.addResourceString("$.countries[1].Asia[0]", "China - XL", 8);
92+
WRITE_BUNDLE.addResourceString("$.countries[1].Asia[1]", "Japan - XL", 9);
93+
WRITE_BUNDLE.addResourceString("$.countries[1].Asia[2]", "India - XL", 10);
94+
WRITE_BUNDLE.addResourceString("$.countries[2].Americas['S. America'][0]", "Brazil - XL", 11);
95+
WRITE_BUNDLE.addResourceString("$.countries[2].Americas['S. America'][1]", "Venezuela - XL", 12);
96+
WRITE_BUNDLE.addResourceString("$.countries[2].Americas['N. America'][0]", "United States [USA] - XL", 13);
97+
WRITE_BUNDLE.addResourceString("$.countries[2].Americas['N. America'][1]", "Canada - XL", 14);
98+
WRITE_BUNDLE.addResourceString("$.countries[2].Americas['N. America'][2]", "Mexico - XL", 15);
99+
WRITE_BUNDLE.addResourceString("$.countries[3].Africa[0]", "Egypt - XL", 16);
100+
WRITE_BUNDLE.addResourceString("$.countries[3].Africa[1]", "Somalia - XL", 17);
101+
WRITE_BUNDLE.addResourceString("$.countries[3].Africa[2]", "S. Africa - XL", 18);
102+
WRITE_BUNDLE.addResourceString("$.colors[0]", "red - XL", 19);
103+
WRITE_BUNDLE.addResourceString("$.colors[1]", "blue - XL", 20);
104+
WRITE_BUNDLE.addResourceString("$.colors[2]", "yellow - XL", 21);
105+
WRITE_BUNDLE.addResourceString("$.colors[3]", "orange - XL", 22);
106+
WRITE_BUNDLE.addResourceString("['frog[\\u00272\\u0027]']", "Red-eyed Tree Frog - XL", 23);
107+
WRITE_BUNDLE.addResourceString("['owl[3]']", "Great Horned Owl - XL", 24);
108+
WRITE_BUNDLE.addResourceString("some_text", "Just a plain old string - XL", 25);
72109
}
73110

74111
private static final JsonResource res = new JsonResource();

‎gp-res-filter/src/test/resource/resfilter/json/input.json‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,18 @@
66
},
77
"white": "Polar Bear"
88
},
9-
"frog 2": "Red-eyed Tree Frog",
10-
"owl 3": "Great Horned Owl"
9+
"countries": [
10+
{ "Europe": [ "Germany", "Italy", "France", "Spain" ]},
11+
{ "Asia": [ "China", "Japan", "India"] },
12+
{ "Americas": {
13+
"S. America": [ "Brazil", "Venezuela"],
14+
"N. America": [ "United States [USA]", "Canada", "Mexico"]
15+
}
16+
},
17+
{ "Africa": [ "Egypt", "Somalia", "S. Africa" ]}
18+
],
19+
"colors": ["red", "blue", "yellow", "orange" ],
20+
"frog['2']": "Red-eyed Tree Frog",
21+
"owl[3]": "Great Horned Owl",
22+
"some_text": "Just a plain old string"
1123
}

0 commit comments

Comments
 (0)