Merge pull request #65 from JCEmmons/issue62

yumaoka · web-flow · commit 0528e99697ee · 2017-04-17T15:46:08.000-04:00
JSON Structured Key support
diff --git a/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/JsonResource.java b/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/JsonResource.java
@@ -1,5 +1,5 @@
 /*
- * Copyright IBM Corp. 2015, 2016
+ * Copyright IBM Corp. 2015, 2017
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,23 +22,43 @@
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.nio.charset.StandardCharsets;
-import java.util.LinkedHashMap;
+import java.text.StringCharacterIterator;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
+import com.google.gson.JsonNull;
+import com.google.gson.JsonObject;
 import com.google.gson.JsonParseException;
 import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.stream.JsonToken;
 import com.ibm.g11n.pipeline.resfilter.ResourceString.ResourceStringComparator;
 
 /**
  * JSON resource filter implementation.
  *
- * @author Yoshito Umaoka
+ * @author Yoshito Umaoka, John Emmons
  */
 public class JsonResource implements ResourceFilter {
 
+    private class KeyPiece {
+        String keyValue;
+        JsonToken keyType;
+
+        KeyPiece(String keyValue, JsonToken keyType) {
+            this.keyValue = keyValue;
+            this.keyType = keyType;
+        }
+    }
+
     @Override
     public Bundle parse(InputStream inStream) throws IOException {
         Bundle bundle = new Bundle();
@@ -47,41 +67,167 @@ public Bundle parse(InputStream inStream) throws IOException {
             if (!root.isJsonObject()) {
                 throw new IllegalResourceFormatException("The root JSON element is not an JSON object.");
             }
-            int sequenceNum = 0;
-            for (Map.Entry<String, JsonElement> entry : root.getAsJsonObject().entrySet()) {
-                String key = entry.getKey();
-                JsonElement value = entry.getValue();
-                if (!value.isJsonPrimitive() || !value.getAsJsonPrimitive().isString()) {
-                    throw new IllegalResourceFormatException("The value of JSON element " + key + " is not a string.");
-                }
-                sequenceNum++;
-                bundle.addResourceString(key, value.getAsString(), sequenceNum);
-            }
+            addBundleStrings(root.getAsJsonObject(), "", bundle, 0);
         } catch (JsonParseException e) {
             throw new IllegalResourceFormatException("Failed to parse the specified JSON contents.", e);
         }
         return bundle;
     }
 
+    private int addBundleStrings(JsonObject obj, String keyPrefix, Bundle bundle, int sequenceNum) {
+        for (Map.Entry<String, JsonElement> entry : obj.entrySet()) {
+            String key = entry.getKey();
+            JsonElement value = entry.getValue();
+            if (value.isJsonObject()) {
+                sequenceNum = addBundleStrings(value.getAsJsonObject(), modifiedKeyPrefix(keyPrefix,key,"$.") , bundle, sequenceNum);
+            } else if (value.isJsonArray()) {
+                JsonArray ar = value.getAsJsonArray();
+                for (int i = 0; i < ar.size(); i++) {
+                    JsonElement arrayEntry = ar.get(i);
+                    if (arrayEntry.isJsonPrimitive() && arrayEntry.getAsJsonPrimitive().isString()) {
+                        sequenceNum++;
+                        bundle.addResourceString(modifiedKeyPrefix(keyPrefix,key,"$.") + "[" + Integer.toString(i) + "]",
+                                arrayEntry.getAsString(), sequenceNum);
+
+                    } else {
+                        sequenceNum = addBundleStrings(arrayEntry.getAsJsonObject(),
+                                modifiedKeyPrefix(keyPrefix,key,"$.") + "[" + Integer.toString(i) + "]", bundle, sequenceNum);
+                    }
+                }
+            } else if (!value.isJsonPrimitive() || !value.getAsJsonPrimitive().isString()) {
+                throw new IllegalResourceFormatException("The value of JSON element " + key + " is not a string.");
+            } else {
+                sequenceNum++;
+                bundle.addResourceString(modifiedKeyPrefix(keyPrefix,key,""), value.getAsString(), sequenceNum);
+            }
+        }
+        return sequenceNum;
+    }
+
+    private String modifiedKeyPrefix( String keyPrefix, String key, String addPrefixIfEmpty) {
+        
+        final Pattern specialSequences = Pattern.compile("[.'\\[\\]]");
+        if (key.isEmpty()) {
+            return keyPrefix;
+        }
+        if (specialSequences.matcher(key).find(0)) {
+            String modifiedKey = key.replaceAll("'", "\\\\u0027");
+            return keyPrefix + "['" + modifiedKey + "']";
+        } else {
+            if (keyPrefix.isEmpty()) {
+                return addPrefixIfEmpty + key;
+            }
+            return keyPrefix + "." + key;
+        }
+    }
+    
     @Override
     public void write(OutputStream outStream, String language, Bundle bundle) throws IOException {
         // extracts key value pairs in original sequence order
         TreeSet<ResourceString> sortedResources = new TreeSet<>(new ResourceStringComparator());
         sortedResources.addAll(bundle.getResourceStrings());
-        LinkedHashMap<String, String> kvmap = new LinkedHashMap<>(sortedResources.size());
+        JsonObject output = new JsonObject();
         for (ResourceString res : sortedResources) {
-            kvmap.put(res.getKey(), res.getValue());
+            String key = res.getKey();
+            List<KeyPiece> keyPieces = splitKeyPieces(key);
+            JsonElement current = output;
+            for (int i = 0; i < keyPieces.size(); i++) {
+                if (i + 1 < keyPieces.size()) { // There is structure under this key piece
+                    if (current.isJsonObject()) {
+                        JsonObject currentObject = current.getAsJsonObject();
+                        if (!currentObject.has(keyPieces.get(i).keyValue)) {
+                            if (keyPieces.get(i + 1).keyType == JsonToken.BEGIN_ARRAY) {
+                                currentObject.add(keyPieces.get(i).keyValue, new JsonArray());
+                            } else {
+                                currentObject.add(keyPieces.get(i).keyValue, new JsonObject());
+                            }
+                        }
+                        current = currentObject.get(keyPieces.get(i).keyValue);
+                    } else {
+                        JsonArray currentArray = current.getAsJsonArray();
+                        Integer idx = Integer.valueOf(keyPieces.get(i).keyValue);
+                        for ( int arrayIndex = currentArray.size(); arrayIndex <= idx ; arrayIndex++) {
+                            currentArray.add(JsonNull.INSTANCE);
+                        }
+                        if (currentArray.get(idx).isJsonNull()) {
+                            if (keyPieces.get(i + 1).keyType == JsonToken.BEGIN_ARRAY) {
+                                currentArray.set(idx, new JsonArray());
+                            } else {
+                                currentArray.set(idx, new JsonObject());
+                            }
+                        } 
+                        current = currentArray.get(idx);
+                    }
+                } else { // This is the leaf node
+                    if (keyPieces.get(i).keyType == JsonToken.BEGIN_ARRAY) {
+                        JsonArray currentArray = current.getAsJsonArray();
+                        Integer idx = Integer.valueOf(keyPieces.get(i).keyValue);
+                        JsonPrimitive e = new JsonPrimitive(res.getValue());
+                        for ( int arrayIndex = currentArray.size(); arrayIndex <= idx ; arrayIndex++) {
+                            currentArray.add(JsonNull.INSTANCE);
+                        }
+                        current.getAsJsonArray().set(idx, e);
+                    } else {
+                        current.getAsJsonObject().addProperty(keyPieces.get(i).keyValue, res.getValue());
+                    }
+                }
+            }
         }
         try (OutputStreamWriter writer = new OutputStreamWriter(new BufferedOutputStream(outStream),
                 StandardCharsets.UTF_8)) {
-            new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create().toJson(kvmap, writer);
+            new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create().toJson(output, writer);
+        }
+    }
+
+    private List<KeyPiece> splitKeyPieces(String key) {
+        List<KeyPiece> result = new ArrayList<KeyPiece>();
+        Matcher onlyDigits = Pattern.compile("^\\d+$").matcher("");
+        // Disregard $. at the beginning - it's not really part of the key...
+        List<String> tokens = findTokens(key.startsWith("$.") ? key.substring(2) : key);
+        for (String s : tokens) {
+            if (s.startsWith("'")) {
+                // Turn any "\u0027" in the key back into '
+                String modifiedKeyPiece = s.substring(1, s.length() - 1).replaceAll("\\\\u0027", "'");
+                result.add(new KeyPiece(modifiedKeyPiece, JsonToken.BEGIN_OBJECT));
+            } else if (onlyDigits.reset(s).matches()) {
+                result.add(new KeyPiece(s, JsonToken.BEGIN_ARRAY));
+            } else {
+                for (String s2 : s.split("\\.")) {
+                    if (!s2.isEmpty()) {
+                        result.add(new KeyPiece(s2, JsonToken.BEGIN_OBJECT));
+                    }
+                }
+            }
+        }
+        return Collections.unmodifiableList(result);
+    }
+
+    private static List<String> findTokens(String data) {
+        List<String> tokens = new ArrayList<String>();
+        boolean inQuotes = false;
+        StringBuilder currentToken = new StringBuilder();
+        StringCharacterIterator i = new StringCharacterIterator(data);
+        while (i.current() != StringCharacterIterator.DONE) {
+            char c = i.current();
+            if ( c == '\'' ) {
+                inQuotes = !inQuotes;
+            }
+            if (!inQuotes && ( c == '.' || c == '[' || c == ']')) {
+                tokens.add(currentToken.toString());
+                currentToken.setLength(0);
+            } else {
+                currentToken.append(c);
+            }
+            i.next();
         }
+        tokens.add(currentToken.toString());
+        return Collections.unmodifiableList(tokens);
     }
 
     @Override
-    public void merge(InputStream base, OutputStream outStream, String language, Bundle bundle)
-            throws IOException {
-        //TODO: Add merge implementation here. For now, fallback to write() operation.
+    public void merge(InputStream base, OutputStream outStream, String language, Bundle bundle) throws IOException {
+        // TODO: Add merge implementation here. For now, fallback to write()
+        // operation.
         write(outStream, language, bundle);
     }
 }
diff --git a/gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/JsonResourceTest.java b/gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/JsonResourceTest.java
@@ -1,5 +1,5 @@
 /*
- * Copyright IBM Corp. 2016
+ * Copyright IBM Corp. 2016, 2017
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -30,7 +30,6 @@
 import java.util.LinkedList;
 import java.util.List;
 
-import org.junit.Ignore;
 import org.junit.Test;
 
 import com.ibm.g11n.pipeline.resfilter.ResourceString.ResourceStringComparator;
@@ -44,15 +43,35 @@ public class JsonResourceTest {
 
     private static final File EXPECTED_WRITE_FILE = new File("src/test/resource/resfilter/json/write-output.json");
 
-    private static final File EXPECTED_MERGE_FILE = new File("src/test/resource/resfilter/json/merge-output.json");
-
     private static final Collection<ResourceString> EXPECTED_INPUT_RES_LIST;
 
     static {
         List<ResourceString> lst = new LinkedList<>();
-        lst.add(new ResourceString("bear 1", "Brown Bear", 1));
-        lst.add(new ResourceString("frog 2", "Red-eyed Tree Frog", 2));
-        lst.add(new ResourceString("owl 3", "Great Horned Owl", 3));
+        lst.add(new ResourceString("$.bears.grizzly.brown", "Brown Bear", 1));
+        lst.add(new ResourceString("$.bears.grizzly.black", "Black Bear", 2));
+        lst.add(new ResourceString("$.bears.white", "Polar Bear", 3));
+        lst.add(new ResourceString("$.countries[0].Europe[0]", "Germany", 4));
+        lst.add(new ResourceString("$.countries[0].Europe[1]", "Italy", 5));
+        lst.add(new ResourceString("$.countries[0].Europe[2]", "France", 6));
+        lst.add(new ResourceString("$.countries[0].Europe[3]", "Spain", 7));
+        lst.add(new ResourceString("$.countries[1].Asia[0]", "China", 8));
+        lst.add(new ResourceString("$.countries[1].Asia[1]", "Japan", 9));
+        lst.add(new ResourceString("$.countries[1].Asia[2]", "India", 10));
+        lst.add(new ResourceString("$.countries[2].Americas['S. America'][0]", "Brazil", 11));
+        lst.add(new ResourceString("$.countries[2].Americas['S. America'][1]", "Venezuela", 12));
+        lst.add(new ResourceString("$.countries[2].Americas['N. America'][0]", "United States [USA]", 13));
+        lst.add(new ResourceString("$.countries[2].Americas['N. America'][1]", "Canada", 14));
+        lst.add(new ResourceString("$.countries[2].Americas['N. America'][2]", "Mexico", 15));
+        lst.add(new ResourceString("$.countries[3].Africa[0]", "Egypt", 16));
+        lst.add(new ResourceString("$.countries[3].Africa[1]", "Somalia", 17));
+        lst.add(new ResourceString("$.countries[3].Africa[2]", "S. Africa", 18));
+        lst.add(new ResourceString("$.colors[0]", "red", 19));
+        lst.add(new ResourceString("$.colors[1]", "blue", 20));
+        lst.add(new ResourceString("$.colors[2]", "yellow", 21));
+        lst.add(new ResourceString("$.colors[3]", "orange", 22));
+        lst.add(new ResourceString("['frog[\\u00272\\u0027]']", "Red-eyed Tree Frog", 23));
+        lst.add(new ResourceString("['owl[3]']", "Great Horned Owl", 24));
+        lst.add(new ResourceString("some_text", "Just a plain old string", 25));
 
         Collections.sort(lst, new ResourceStringComparator());
         EXPECTED_INPUT_RES_LIST = lst;
@@ -62,9 +81,31 @@ public class JsonResourceTest {
 
     static {
         WRITE_BUNDLE = new Bundle();
-        WRITE_BUNDLE.addResourceString("owl 3", "Great Horned Owl - translated", 3);
-        WRITE_BUNDLE.addResourceString("bear 1", "Brown Bear - translated", 1);
-        WRITE_BUNDLE.addResourceString("frog 2", "Red-eyed Tree Frog - translated", 2);
+        WRITE_BUNDLE.addResourceString("$.bears.grizzly.brown", "Brown Bear - XL", 1);
+        WRITE_BUNDLE.addResourceString("$.bears.grizzly.black", "Black Bear - XL", 2);
+        WRITE_BUNDLE.addResourceString("$.bears.white", "Polar Bear - XL", 3);
+        WRITE_BUNDLE.addResourceString("$.countries[0].Europe[0]", "Germany - XL", 4);
+        WRITE_BUNDLE.addResourceString("$.countries[0].Europe[1]", "Italy - XL", 5);
+        WRITE_BUNDLE.addResourceString("$.countries[0].Europe[2]", "France - XL", 6);
+        WRITE_BUNDLE.addResourceString("$.countries[0].Europe[3]", "Spain - XL", 7);
+        WRITE_BUNDLE.addResourceString("$.countries[1].Asia[0]", "China - XL", 8);
+        WRITE_BUNDLE.addResourceString("$.countries[1].Asia[1]", "Japan - XL", 9);
+        WRITE_BUNDLE.addResourceString("$.countries[1].Asia[2]", "India - XL", 10);
+        WRITE_BUNDLE.addResourceString("$.countries[2].Americas['S. America'][0]", "Brazil - XL", 11);
+        WRITE_BUNDLE.addResourceString("$.countries[2].Americas['S. America'][1]", "Venezuela - XL", 12);
+        WRITE_BUNDLE.addResourceString("$.countries[2].Americas['N. America'][0]", "United States [USA] - XL", 13);
+        WRITE_BUNDLE.addResourceString("$.countries[2].Americas['N. America'][1]", "Canada - XL", 14);
+        WRITE_BUNDLE.addResourceString("$.countries[2].Americas['N. America'][2]", "Mexico - XL", 15);
+        WRITE_BUNDLE.addResourceString("$.countries[3].Africa[0]", "Egypt - XL", 16);
+        WRITE_BUNDLE.addResourceString("$.countries[3].Africa[1]", "Somalia - XL", 17);
+        WRITE_BUNDLE.addResourceString("$.countries[3].Africa[2]", "S. Africa - XL", 18);
+        WRITE_BUNDLE.addResourceString("$.colors[0]", "red - XL", 19);
+        WRITE_BUNDLE.addResourceString("$.colors[1]", "blue - XL", 20);
+        WRITE_BUNDLE.addResourceString("$.colors[2]", "yellow - XL", 21);
+        WRITE_BUNDLE.addResourceString("$.colors[3]", "orange - XL", 22);
+        WRITE_BUNDLE.addResourceString("['frog[\\u00272\\u0027]']", "Red-eyed Tree Frog - XL", 23);
+        WRITE_BUNDLE.addResourceString("['owl[3]']", "Great Horned Owl - XL", 24);
+        WRITE_BUNDLE.addResourceString("some_text", "Just a plain old string - XL", 25);
     }
 
     private static final JsonResource res = new JsonResource();
@@ -93,16 +134,16 @@ public void testWrite() throws IOException {
         }
     }
 
-    @Test
-    public void testMerge() throws IOException {
-        File tempFile = File.createTempFile(this.getClass().getSimpleName(), ".json");
-        tempFile.deleteOnExit();
+//    @Test
+//    public void testMerge() throws IOException {
+//        File tempFile = File.createTempFile(this.getClass().getSimpleName(), ".json");
+//        tempFile.deleteOnExit();
 
-        try (OutputStream os = new FileOutputStream(tempFile); InputStream is = new FileInputStream(INPUT_FILE)) {
-            res.merge(is, os, null, WRITE_BUNDLE);
-            os.flush();
+//        try (OutputStream os = new FileOutputStream(tempFile); InputStream is = new FileInputStream(INPUT_FILE)) {
+//            res.merge(is, os, null, WRITE_BUNDLE);
+//            os.flush();
             // TODO: Not ready yet
             // assertTrue(ResourceTestUtil.compareFiles(EXPECTED_MERGE_FILE, tempFile));
-        }
-    }
+//        }
+//    }
 }
diff --git a/gp-res-filter/src/test/resource/resfilter/json/input.json b/gp-res-filter/src/test/resource/resfilter/json/input.json
@@ -1,5 +1,23 @@
 {
-    "bear 1": "Brown Bear",
-    "frog 2": "Red-eyed Tree Frog",
-    "owl 3": "Great Horned Owl"
+    "bears": {
+    	"grizzly": {
+    	    "brown": "Brown Bear",
+    	    "black": "Black Bear"
+    	},
+        "white": "Polar Bear"
+    },
+    "countries": [ 
+    	{ "Europe": [ "Germany", "Italy", "France", "Spain" ]},
+    	{ "Asia": [ "China", "Japan", "India"] },
+        { "Americas": {
+            "S. America": [ "Brazil", "Venezuela"],
+            "N. America": [ "United States [USA]", "Canada", "Mexico"]
+            }
+        },
+        { "Africa": [ "Egypt", "Somalia", "S. Africa" ]}
+    ],
+    "colors": ["red", "blue", "yellow", "orange" ],
+    "frog['2']": "Red-eyed Tree Frog",
+    "owl[3]": "Great Horned Owl",
+    "some_text": "Just a plain old string"
 }
diff --git a/gp-res-filter/src/test/resource/resfilter/json/write-output.json b/gp-res-filter/src/test/resource/resfilter/json/write-output.json
