Skip to content

Commit 8bcd488

Browse files
srl295srl295
committed
Fix processing of JSON keys
- added unit test for splitKeys - inverted the relationship between findTokens and splitKeys to track array subscript Fixes: #103
1 parent b34bcc3 commit 8bcd488

3 files changed

Lines changed: 513 additions & 41 deletions

File tree

‎gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/impl/JsonResource.java‎

Lines changed: 36 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import java.util.Collections;
2828
import java.util.List;
2929
import java.util.Map;
30-
import java.util.regex.Matcher;
3130
import java.util.regex.Pattern;
3231

3332
import com.google.gson.GsonBuilder;
@@ -251,60 +250,56 @@ public void write(OutputStream outStream, LanguageBundle languageBundle,
251250
static List<KeyPiece> splitKeyPieces(String key) {
252251
if (USE_JSONPATH_PATTERN.matcher(key).matches()) {
253252
List<KeyPiece> result = new ArrayList<KeyPiece>();
254-
Matcher onlyDigits = Pattern.compile("^\\d+$").matcher("");
253+
boolean inQuotes = false;
254+
StringBuilder currentToken = new StringBuilder();
255255
// Disregard $ at the beginning - it's not really part of the key...
256-
List<String> tokens = findTokens(key.substring(JSONPATH_ROOT.length()));
257-
for (String s : tokens) {
258-
if (s.startsWith("'")) {
259-
// Turn any "\u0027" in the key back into '
260-
String modifiedKeyPiece = s.substring(1, s.length() - 1).replaceAll("\\\\u0027", "'");
261-
result.add(new KeyPiece(modifiedKeyPiece, JsonToken.BEGIN_OBJECT));
262-
} else if (onlyDigits.reset(s).matches()) { // BAD
263-
result.add(new KeyPiece(s, JsonToken.BEGIN_ARRAY));
264-
} else if (false && s.startsWith("[") && (s.length() > 1)) {
265-
result.add(new KeyPiece(s.substring(1, s.length() - 1), JsonToken.BEGIN_ARRAY));
266-
} else {
267-
for (String s2 : s.split("\\.")) {
268-
if (!s2.isEmpty()) {
269-
result.add(new KeyPiece(s2, JsonToken.BEGIN_OBJECT));
256+
StringCharacterIterator i = new StringCharacterIterator(key.substring(JSONPATH_ROOT.length()));
257+
boolean inSubscript = false;
258+
while (i.current() != StringCharacterIterator.DONE) {
259+
char c = i.current();
260+
if (c == '\'') {
261+
inQuotes = !inQuotes;
262+
}
263+
if (!inQuotes && (c == '.' || c == '[' || c == ']')) {
264+
if (currentToken.length() > 0) {
265+
addToken(result, currentToken.toString(), inSubscript);
266+
currentToken.setLength(0);
267+
if (inSubscript) {
268+
inSubscript = false;
270269
}
271270
}
271+
if (c == '[') {
272+
inSubscript = true; // Record that the next token had an
273+
// array subscript on it.
274+
}
275+
} else {
276+
currentToken.append(c);
272277
}
278+
i.next();
273279
}
280+
addToken(result, currentToken.toString(), inSubscript);
281+
274282
return Collections.unmodifiableList(result);
275283
}
276284
// Otherwise, this is a plain JSON object label
277285
return Collections.singletonList(new KeyPiece(key, JsonToken.BEGIN_OBJECT));
278286
}
279287

280-
static List<String> findTokens(String data) {
281-
List<String> tokens = new ArrayList<String>();
282-
boolean inQuotes = false;
283-
StringBuilder currentToken = new StringBuilder();
284-
StringCharacterIterator i = new StringCharacterIterator(data);
285-
while (i.current() != StringCharacterIterator.DONE) {
286-
char c = i.current();
287-
if (c == '\'') {
288-
inQuotes = !inQuotes;
289-
}
290-
if (!inQuotes && (c == '.' || c == '[' || c == ']')) {
291-
// if (c == ']') {
292-
// currentToken.append(c);
293-
// }
294-
if (currentToken.length() > 0) {
295-
tokens.add(currentToken.toString());
296-
currentToken.setLength(0);
288+
static void addToken(List<KeyPiece> result, String s, boolean inSubscript) {
289+
if (s.startsWith("'")) {
290+
// Turn any "\u0027" in the key back into '
291+
String modifiedKeyPiece = s.substring(1, s.length() - 1).replaceAll("\\\\u0027", "'");
292+
result.add(new KeyPiece(modifiedKeyPiece, JsonToken.BEGIN_OBJECT));
293+
} else if (inSubscript) {
294+
// [0] produces an array
295+
result.add(new KeyPiece(s, JsonToken.BEGIN_ARRAY));
296+
} else {
297+
for (String s2 : s.split("\\.")) {
298+
if (!s2.isEmpty()) {
299+
result.add(new KeyPiece(s2, JsonToken.BEGIN_OBJECT));
297300
}
298-
// if (c == '[') {
299-
// currentToken.append(c);
300-
// }
301-
} else {
302-
currentToken.append(c);
303301
}
304-
i.next();
305302
}
306-
tokens.add(currentToken.toString());
307-
return Collections.unmodifiableList(tokens);
308303
}
309304

310305
// TODO: Implement merge method

‎gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/impl/JsonResourceTest.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@
1616
package com.ibm.g11n.pipeline.resfilter.impl;
1717

1818
import static org.junit.Assert.assertArrayEquals;
19+
import static org.junit.Assert.assertEquals;
1920
import static org.junit.Assert.assertTrue;
2021

2122
import java.io.File;
2223
import java.io.FileInputStream;
24+
import java.io.FileNotFoundException;
2325
import java.io.FileOutputStream;
2426
import java.io.IOException;
2527
import java.io.InputStream;
@@ -29,14 +31,19 @@
2931
import java.util.Collections;
3032
import java.util.LinkedList;
3133
import java.util.List;
34+
import java.util.Map.Entry;
3235

3336
import org.junit.Test;
3437

38+
import com.google.gson.JsonArray;
39+
import com.google.gson.JsonElement;
40+
import com.google.gson.JsonObject;
3541
import com.ibm.g11n.pipeline.resfilter.LanguageBundle;
3642
import com.ibm.g11n.pipeline.resfilter.LanguageBundleBuilder;
3743
import com.ibm.g11n.pipeline.resfilter.ResourceFilterException;
3844
import com.ibm.g11n.pipeline.resfilter.ResourceString;
3945
import com.ibm.g11n.pipeline.resfilter.ResourceString.ResourceStringComparator;
46+
import com.ibm.g11n.pipeline.resfilter.impl.JsonResource.KeyPiece;
4047

4148
/**
4249
* @author farhan
@@ -45,6 +52,7 @@
4552
public class JsonResourceTest {
4653
private static final File INPUT_FILE = new File("src/test/resource/resfilter/json/input.json");
4754
private static final File INPUT_FILE2 = new File("src/test/resource/resfilter/json/other-input.json");
55+
private static final File SPLITKEYS = new File("src/test/resource/resfilter/json/testSplitKeys.json");
4856

4957
private static final File EXPECTED_WRITE_FILE = new File("src/test/resource/resfilter/json/write-output.json");
5058

@@ -216,6 +224,43 @@ public void testReWriteOther() throws IOException, ResourceFilterException {
216224
}
217225
}
218226
}
227+
228+
/**
229+
* @throws IOException
230+
* @throws FileNotFoundException
231+
* @See {@link JsonResource#splitKeyPieces(String)}
232+
*/
233+
@Test
234+
public void testSplitKeyPieces() throws FileNotFoundException, IOException {
235+
/*
236+
* Initial data generated with: Gson g = new GsonBuilder().create();
237+
* TreeMap<String, List<KeyPiece>> tm = new TreeMap<String,
238+
* List<KeyPiece>>();
239+
*
240+
* for (final ResourceString s : EXPECTED_INPUT_RES_LIST) {
241+
* tm.put(s.getKey(), JsonResource.splitKeyPieces(s.getKey())); }
242+
* System.out.println(g.toJson(tm));
243+
*/
244+
245+
JsonObject SPLITKEY_DATA = ResourceTestUtil.parseJson(SPLITKEYS).getAsJsonObject();
246+
for (final Entry<String, JsonElement> e : SPLITKEY_DATA.entrySet()) {
247+
final String key = e.getKey();
248+
final JsonArray expectList = e.getValue().getAsJsonArray();
249+
250+
List<KeyPiece> actualList = JsonResource.splitKeyPieces(key);
251+
252+
String prefix = "‘" + key + "’: ";
253+
assertEquals(prefix + "key count", expectList.size(), actualList.size());
254+
for (int n = 0; n < expectList.size(); n++) {
255+
JsonObject expectObject = expectList.get(n).getAsJsonObject();
256+
KeyPiece actualObject = actualList.get(n);
257+
String subPrefix = prefix + " key " + n;
258+
assertEquals(subPrefix + " value ", expectObject.get("keyValue").getAsString(), actualObject.keyValue);
259+
assertEquals(subPrefix + " type ", expectObject.get("keyType").getAsString(),
260+
actualObject.keyType.name());
261+
}
262+
}
263+
}
219264

220265
// TODO: Not ready yet
221266
// @Test

0 commit comments

Comments
 (0)