Merge pull request #69 from JCEmmons/issue68

yumaoka · web-flow · commit 8e480970251d · 2017-05-08T10:54:18.000-04:00
Implement support for globalizejs JSON resources. Fixes #68
diff --git a/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/ImportCmd.java b/gp-cli/src/main/java/com/ibm/g11n/pipeline/tools/cli/ImportCmd.java
@@ -1,5 +1,5 @@
 /*  
- * Copyright IBM Corp. 2015
+ * Copyright IBM Corp. 2015, 2017
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -69,6 +69,12 @@ protected void _execute() {
         File f = new File(fileName);
         try (FileInputStream fis = new FileInputStream(f)) {
             Bundle bundle = filter.parse(fis);
+            if (!bundle.getLanguage().isEmpty() &&
+                    !bundle.getLanguage().equals(languageId)) {
+                throw new RuntimeException("Language ID \"" + bundle.getLanguage() +
+                    "\" in the upload file does not match the specified language ID \"" +
+                    languageId + "\"");
+            }
             resEntries = new HashMap<>(bundle.getResourceStrings().size());
             for (ResourceString resString : bundle.getResourceStrings()) {
                 NewResourceEntryData resEntryData = new NewResourceEntryData(resString.getValue());
diff --git a/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/Bundle.java b/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/Bundle.java
@@ -1,5 +1,5 @@
 /*
- * Copyright IBM Corp. 2016
+ * Copyright IBM Corp. 2016, 2017
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@
 import java.util.List;
 
 public final class Bundle {
+    private String language = "";
     private List<String> notes;
     private Collection<ResourceString> resStrings;
 
@@ -80,4 +81,12 @@ public List<String> getNotes() {
         }
         return Collections.unmodifiableList(notes);
     }
+    
+    public String getLanguage() {
+        return language;
+    }
+    
+    public void setLanguage(String language) {
+        this.language = language;
+    }
 }
diff --git a/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/GlobalizeJsResource.java b/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/GlobalizeJsResource.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright IBM Corp. 2017
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.ibm.g11n.pipeline.resfilter;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonParser;
+
+/**
+ * globalizejs resource filter implementation.
+ *
+ * @author John Emmons
+ */
+public class GlobalizeJsResource extends JsonResource {
+
+    @Override
+    public Bundle parse(InputStream inStream) throws IOException {
+        Bundle bundle = new Bundle();
+        try (InputStreamReader reader = new InputStreamReader(new BomInputStream(inStream), StandardCharsets.UTF_8)) {
+            JsonElement root = new JsonParser().parse(reader);
+            if (!root.isJsonObject()) {
+                throw new IllegalResourceFormatException("The root JSON element is not a JSON object.");
+            }
+            JsonObject root_obj = root.getAsJsonObject();
+            Set<Map.Entry<String, JsonElement>> root_elements = root_obj.entrySet();
+            if (root_elements.size() != 1) {
+                throw new IllegalResourceFormatException(
+                        "Only one top level language tag element is allowed per file.");
+            }
+            Map.Entry<String, JsonElement> top_level = root_elements.iterator().next();
+            String language = top_level.getKey();
+            // We just hang on to the language tag as part of the bundle.
+            // When doing an import, we can validate that the language tag matches what they
+            // say they are importing.
+            bundle.setLanguage(language);
+            JsonElement language_obj = top_level.getValue();
+            if (!language_obj.isJsonObject()) {
+                throw new IllegalResourceFormatException("The top level language element is not a JSON object.");
+            }
+            addBundleStrings(language_obj.getAsJsonObject(), "", bundle, 0);
+        } catch (JsonParseException e) {
+            throw new IllegalResourceFormatException("Failed to parse the specified JSON contents.", e);
+        }
+        return bundle;
+    }
+
+    /**
+     * The override of addBundleStrings is necessary because globalizejs treats arrays of strings
+     * as a single long string, with each piece separated by a space, rather than treating it like
+     * a real JSON array.
+     */
+    @Override
+    protected int addBundleStrings(JsonObject obj, String keyPrefix, Bundle bundle, int sequenceNum) {
+        for (Map.Entry<String, JsonElement> entry : obj.entrySet()) {
+            String key = entry.getKey();
+            JsonElement value = entry.getValue();
+            if (value.isJsonObject()) {
+                sequenceNum = addBundleStrings(value.getAsJsonObject(), modifiedKeyPrefix(keyPrefix, key, "$."), bundle,
+                        sequenceNum);
+            } else if (value.isJsonArray()) {
+                JsonArray ar = value.getAsJsonArray();
+                StringBuilder sb = new StringBuilder();
+                for (int i = 0; i < ar.size(); i++) {
+                    JsonElement arrayEntry = ar.get(i);
+                    if (arrayEntry.isJsonPrimitive() && arrayEntry.getAsJsonPrimitive().isString()) {
+                        if (i > 0) {
+                            sb.append(" "); //
+                        }
+                        sb.append(arrayEntry.getAsString());
+                    } else {
+                        throw new IllegalResourceFormatException(
+                                "Arrays must contain only strings in a globalizejs resource.");
+                    }
+                }
+                sequenceNum++;
+                bundle.addResourceString(modifiedKeyPrefix(keyPrefix, key, ""), sb.toString(), sequenceNum);
+            } else if (!value.isJsonPrimitive() || !value.getAsJsonPrimitive().isString()) {
+                throw new IllegalResourceFormatException("The value of JSON element " + key + " is not a string.");
+            } else {
+                sequenceNum++;
+                bundle.addResourceString(modifiedKeyPrefix(keyPrefix, key, ""), value.getAsString(), sequenceNum);
+            }
+        }
+        return sequenceNum;
+    }
+}
diff --git a/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/JsonResource.java b/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/JsonResource.java
@@ -74,38 +74,41 @@ public Bundle parse(InputStream inStream) throws IOException {
         return bundle;
     }
 
-    private int addBundleStrings(JsonObject obj, String keyPrefix, Bundle bundle, int sequenceNum) {
+    protected int addBundleStrings(JsonObject obj, String keyPrefix, Bundle bundle, int sequenceNum) {
         for (Map.Entry<String, JsonElement> entry : obj.entrySet()) {
             String key = entry.getKey();
             JsonElement value = entry.getValue();
             if (value.isJsonObject()) {
-                sequenceNum = addBundleStrings(value.getAsJsonObject(), modifiedKeyPrefix(keyPrefix,key,"$.") , bundle, sequenceNum);
+                sequenceNum = addBundleStrings(value.getAsJsonObject(), modifiedKeyPrefix(keyPrefix, key, "$."), bundle,
+                        sequenceNum);
             } else if (value.isJsonArray()) {
                 JsonArray ar = value.getAsJsonArray();
                 for (int i = 0; i < ar.size(); i++) {
                     JsonElement arrayEntry = ar.get(i);
                     if (arrayEntry.isJsonPrimitive() && arrayEntry.getAsJsonPrimitive().isString()) {
                         sequenceNum++;
-                        bundle.addResourceString(modifiedKeyPrefix(keyPrefix,key,"$.") + "[" + Integer.toString(i) + "]",
+                        bundle.addResourceString(
+                                modifiedKeyPrefix(keyPrefix, key, "$.") + "[" + Integer.toString(i) + "]",
                                 arrayEntry.getAsString(), sequenceNum);
 
                     } else {
                         sequenceNum = addBundleStrings(arrayEntry.getAsJsonObject(),
-                                modifiedKeyPrefix(keyPrefix,key,"$.") + "[" + Integer.toString(i) + "]", bundle, sequenceNum);
+                                modifiedKeyPrefix(keyPrefix, key, "$.") + "[" + Integer.toString(i) + "]", bundle,
+                                sequenceNum);
                     }
                 }
             } else if (!value.isJsonPrimitive() || !value.getAsJsonPrimitive().isString()) {
                 throw new IllegalResourceFormatException("The value of JSON element " + key + " is not a string.");
             } else {
                 sequenceNum++;
-                bundle.addResourceString(modifiedKeyPrefix(keyPrefix,key,""), value.getAsString(), sequenceNum);
+                bundle.addResourceString(modifiedKeyPrefix(keyPrefix, key, ""), value.getAsString(), sequenceNum);
             }
         }
         return sequenceNum;
     }
 
-    private String modifiedKeyPrefix( String keyPrefix, String key, String addPrefixIfEmpty) {
-        
+    protected String modifiedKeyPrefix(String keyPrefix, String key, String addPrefixIfEmpty) {
+
         final Pattern specialSequences = Pattern.compile("[.'\\[\\]]");
         if (key.isEmpty()) {
             return keyPrefix;
@@ -120,19 +123,29 @@ private String modifiedKeyPrefix( String keyPrefix, String key, String addPrefix
             return keyPrefix + "." + key;
         }
     }
-    
+
     @Override
     public void write(OutputStream outStream, String language, Bundle bundle) throws IOException {
         // extracts key value pairs in original sequence order
         TreeSet<ResourceString> sortedResources = new TreeSet<>(new ResourceStringComparator());
         sortedResources.addAll(bundle.getResourceStrings());
         JsonObject output = new JsonObject();
+        JsonObject top_level;
+        
+        if (this instanceof GlobalizeJsResource) {
+            top_level = new JsonObject();
+            top_level.add(language, output);
+        } else {
+            top_level = output;
+        }
+        
         for (ResourceString res : sortedResources) {
             String key = res.getKey();
             List<KeyPiece> keyPieces = splitKeyPieces(key);
             JsonElement current = output;
             for (int i = 0; i < keyPieces.size(); i++) {
-                if (i + 1 < keyPieces.size()) { // There is structure under this key piece
+                if (i + 1 < keyPieces.size()) { // There is structure under this
+                                                // key piece
                     if (current.isJsonObject()) {
                         JsonObject currentObject = current.getAsJsonObject();
                         if (!currentObject.has(keyPieces.get(i).keyValue)) {
@@ -146,7 +159,7 @@ public void write(OutputStream outStream, String language, Bundle bundle) throws
                     } else {
                         JsonArray currentArray = current.getAsJsonArray();
                         Integer idx = Integer.valueOf(keyPieces.get(i).keyValue);
-                        for ( int arrayIndex = currentArray.size(); arrayIndex <= idx ; arrayIndex++) {
+                        for (int arrayIndex = currentArray.size(); arrayIndex <= idx; arrayIndex++) {
                             currentArray.add(JsonNull.INSTANCE);
                         }
                         if (currentArray.get(idx).isJsonNull()) {
@@ -155,15 +168,15 @@ public void write(OutputStream outStream, String language, Bundle bundle) throws
                             } else {
                                 currentArray.set(idx, new JsonObject());
                             }
-                        } 
+                        }
                         current = currentArray.get(idx);
                     }
                 } else { // This is the leaf node
                     if (keyPieces.get(i).keyType == JsonToken.BEGIN_ARRAY) {
                         JsonArray currentArray = current.getAsJsonArray();
                         Integer idx = Integer.valueOf(keyPieces.get(i).keyValue);
                         JsonPrimitive e = new JsonPrimitive(res.getValue());
-                        for ( int arrayIndex = currentArray.size(); arrayIndex <= idx ; arrayIndex++) {
+                        for (int arrayIndex = currentArray.size(); arrayIndex <= idx; arrayIndex++) {
                             currentArray.add(JsonNull.INSTANCE);
                         }
                         current.getAsJsonArray().set(idx, e);
@@ -175,7 +188,7 @@ public void write(OutputStream outStream, String language, Bundle bundle) throws
         }
         try (OutputStreamWriter writer = new OutputStreamWriter(new BufferedOutputStream(outStream),
                 StandardCharsets.UTF_8)) {
-            new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create().toJson(output, writer);
+            new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create().toJson(top_level, writer);
         }
     }
 
@@ -209,10 +222,10 @@ private static List<String> findTokens(String data) {
         StringCharacterIterator i = new StringCharacterIterator(data);
         while (i.current() != StringCharacterIterator.DONE) {
             char c = i.current();
-            if ( c == '\'' ) {
+            if (c == '\'') {
                 inQuotes = !inQuotes;
             }
-            if (!inQuotes && ( c == '.' || c == '[' || c == ']')) {
+            if (!inQuotes && (c == '.' || c == '[' || c == ']')) {
                 tokens.add(currentToken.toString());
                 currentToken.setLength(0);
             } else {
diff --git a/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/ResourceFilterFactory.java b/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/ResourceFilterFactory.java
@@ -1,5 +1,5 @@
 /*
- * Copyright IBM Corp. 2015
+ * Copyright IBM Corp. 2015, 2017
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,6 +27,8 @@ public static ResourceFilter getX(String type) {
             return new JavaPropertiesResource();
         } else if (type.equalsIgnoreCase("json")) {
             return new JsonResource();
+        } else if (type.equalsIgnoreCase("globalizejs")) {
+            return new GlobalizeJsResource();
         } else if (type.equalsIgnoreCase("amdjs")) {
             return new AmdJsResource();
         } else if (type.equalsIgnoreCase("yml")){
@@ -51,6 +53,8 @@ public static ResourceFilter get(ResourceType type) {
             return new JavaPropertiesResource();
         case JSON:
             return new JsonResource();
+        case GLOBALIZEJS:
+            return new GlobalizeJsResource();
         case AMDJS:
             return new AmdJsResource();
         case YML:
diff --git a/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/ResourceType.java b/gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/ResourceType.java
@@ -1,5 +1,5 @@
 /*
- * Copyright IBM Corp. 2015
+ * Copyright IBM Corp. 2015, 2017
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@
 public enum ResourceType {
     JAVA,
     JSON,
+    GLOBALIZEJS,
     AMDJS,
     YML,
     XLIFF,
diff --git a/gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/AllTests.java b/gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/AllTests.java
@@ -1,5 +1,5 @@
 /*
- * Copyright IBM Corp. 2016
+ * Copyright IBM Corp. 2016, 2017
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,6 +24,7 @@
 @SuiteClasses({
     AmdJsResourceTest.class,
     AndroidStringsResourceTest.class,
+    GlobalizeJsResourceTest.class,
     IOSStringsResourceTest.class,
     JavaPropertiesResourceTest.class,
     JsonResourceTest.class,
diff --git a/gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/GlobalizeJsResourceTest.java b/gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/GlobalizeJsResourceTest.java
diff --git a/gp-res-filter/src/test/resource/resfilter/globalizejs/input.json b/gp-res-filter/src/test/resource/resfilter/globalizejs/input.json
diff --git a/gp-res-filter/src/test/resource/resfilter/globalizejs/write-output.json b/gp-res-filter/src/test/resource/resfilter/globalizejs/write-output.json

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`	`1`	`/*`
`2`		`- * Copyright IBM Corp. 2016`
	`2`	`+ * Copyright IBM Corp. 2016, 2017`
`3`	`3`	`*`
`4`	`4`	`* Licensed under the Apache License, Version 2.0 (the "License");`
`5`	`5`	`* you may not use this file except in compliance with the License.`
`@@ -22,6 +22,7 @@`
`22`	`22`	`import java.util.List;`
`23`	`23`
`24`	`24`	`public final class Bundle {`
	`25`	`+ private String language = "";`
`25`	`26`	`private List<String> notes;`
`26`	`27`	`private Collection<ResourceString> resStrings;`
`27`	`28`
`@@ -80,4 +81,12 @@ public List<String> getNotes() {`
`80`	`81`	`}`
`81`	`82`	`return Collections.unmodifiableList(notes);`
`82`	`83`	`}`
	`84`	`+`
	`85`	`+ public String getLanguage() {`
	`86`	`+ return language;`
	`87`	`+ }`
	`88`	`+`
	`89`	`+ public void setLanguage(String language) {`
	`90`	`+ this.language = language;`
	`91`	`+ }`
`83`	`92`	`}`