Skip to content

Commit 13032c0

Browse files
yumaokayumaoka
authored
Merge pull request #30 from JCEmmons/jprops_parser
Implementation of notes feature for Java properties
2 parents 8426395 + 8a0bc4b commit 13032c0

6 files changed

Lines changed: 157 additions & 31 deletions

File tree

‎gp-res-filter/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,13 @@
8383
<version>2.6.3</version>
8484
</dependency>
8585

86+
<!-- StringEscapeUtils -->
87+
<dependency>
88+
<groupId>org.apache.commons</groupId>
89+
<artifactId>commons-lang3</artifactId>
90+
<version>3.4</version>
91+
</dependency>
92+
8693
<!-- JUnit -->
8794
<dependency>
8895
<groupId>junit</groupId>

‎gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/Bundle.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,12 @@ public void addNote(String note) {
5656
}
5757
notes.add(note);
5858
}
59+
60+
public void addNotes(List<String> inputNotes) {
61+
for (String note : inputNotes) {
62+
notes.add(note);
63+
}
64+
}
5965

6066
public Collection<ResourceString> getResourceStrings() {
6167
if (resStrings == null) {

‎gp-res-filter/src/main/java/com/ibm/g11n/pipeline/resfilter/JavaPropertiesResource.java‎

Lines changed: 128 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright IBM Corp. 2015
2+
* Copyright IBM Corp. 2015, 2016
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -40,6 +40,8 @@
4040

4141
import com.ibm.g11n.pipeline.resfilter.ResourceString.ResourceStringComparator;
4242

43+
import org.apache.commons.lang3.StringEscapeUtils;
44+
4345
/**
4446
* Java properties resource filter implementation.
4547
*
@@ -49,7 +51,8 @@ public class JavaPropertiesResource implements ResourceFilter {
4951

5052
// TODO:
5153
// This is not a good idea. This implementation might work,
52-
// but it depends on an assumption that java.util.Properties#load(InputStream)
54+
// but it depends on an assumption that
55+
// java.util.Properties#load(InputStream)
5356
// calls Properties#put(Object, Object).
5457

5558
@SuppressWarnings("serial")
@@ -65,7 +68,7 @@ public Iterable<Object> orderedKeys() {
6568

6669
@Override
6770
public Enumeration<Object> keys() {
68-
return Collections.<Object>enumeration(keys);
71+
return Collections.<Object> enumeration(keys);
6972
}
7073

7174
@Override
@@ -78,17 +81,85 @@ public Object put(Object key, Object value) {
7881
@Override
7982
public Bundle parse(InputStream inStream) throws IOException {
8083
LinkedProperties props = new LinkedProperties();
81-
props.load(inStream);
84+
BufferedReader inStreamReader = new BufferedReader(new InputStreamReader(inStream, PROPS_ENC));
85+
String line;
86+
Map<String, List<String>> notesMap = new HashMap<>();
87+
List<String> currentNotes = new ArrayList<>();
88+
boolean globalNotesAvailable = true;
89+
List<String> globalNotes = null;
90+
while ((line = inStreamReader.readLine()) != null) {
91+
line = line.trim();
92+
// Comment line - Add to list of comments (notes) until we find
93+
// either
94+
// a blank line (global comment) or a key/value pair
95+
if (line.startsWith("#") || line.startsWith("!")) {
96+
// Strip off the leading comment marker, and perform any
97+
// necessary unescaping here.
98+
currentNotes.add(StringEscapeUtils.unescapeJava(line.substring(1)));
99+
} else if (line.isEmpty()) {
100+
// We are following the convention that the first blank line in
101+
// a properties
102+
// file signifies the end of a global comment.
103+
if (globalNotesAvailable && !currentNotes.isEmpty()) {
104+
globalNotes = new ArrayList<>(currentNotes);
105+
currentNotes.clear();
106+
} else {
107+
// Just a generic blank line - treat it like a comment.
108+
currentNotes.add(line);
109+
}
110+
globalNotesAvailable = false;
111+
} else {
112+
// Regular non-comment line. If there are notes outstanding that
113+
// apply
114+
// to this line, we find its key and add it to the notes map.
115+
StringBuffer sb = new StringBuffer(line);
116+
while (isContinuationLine(sb.toString())) {
117+
String continuationLine = inStreamReader.readLine();
118+
sb.setLength(sb.length() - 1); // Remove the continuation
119+
// "\"
120+
if (continuationLine != null) {
121+
sb.append(continuationLine.trim());
122+
}
123+
}
124+
String logicalLine = sb.toString();
125+
PropDef pd = PropDef.parseLine(logicalLine);
126+
props.setProperty(pd.getKey(), pd.getValue());
127+
if (!currentNotes.isEmpty()) {
128+
notesMap.put(pd.getKey(), new ArrayList<>(currentNotes));
129+
currentNotes.clear();
130+
}
131+
}
132+
}
133+
82134
Iterator<Object> i = props.orderedKeys().iterator();
83135
Bundle result = new Bundle();
84136
int sequenceNum = 0;
85137
while (i.hasNext()) {
86138
String key = (String) i.next();
87-
result.addResourceString(key, props.getProperty(key), ++sequenceNum);
139+
List<String> notes = notesMap.get(key);
140+
ResourceString rs = new ResourceString(key, props.getProperty(key), ++sequenceNum, notes);
141+
result.addResourceString(rs);
142+
}
143+
if (globalNotes != null) {
144+
result.addNotes(globalNotes);
88145
}
89146
return result;
90147
}
91148

149+
// This method handles the bizarre edge case where someone might have
150+
// multiple backslashes at the end of a line. An even number of them
151+
// isn't really a continuation, but a backslash in the property value.
152+
private boolean isContinuationLine(String s) {
153+
int backslashCount = 0;
154+
for (int index = s.length() - 1; index >= 0; index--) {
155+
if (s.charAt(index) != '\\') {
156+
break;
157+
}
158+
backslashCount++;
159+
}
160+
return backslashCount % 2 == 1;
161+
}
162+
92163
@Override
93164
public void write(OutputStream outStream, String language, Bundle resource) throws IOException {
94165
TreeSet<ResourceString> sortedResources = new TreeSet<>(new ResourceStringComparator());
@@ -109,9 +180,7 @@ static class PropDef {
109180
private PropSeparator separator;
110181

111182
public enum PropSeparator {
112-
EQUAL('='),
113-
COLON(':'),
114-
SPACE(' ');
183+
EQUAL('='), COLON(':'), SPACE(' ');
115184

116185
private char sepChar;
117186

@@ -129,7 +198,7 @@ public char getCharacter() {
129198

130199
public PropDef(String key, String value, PropSeparator separator) {
131200
this.key = key;
132-
this.value = value; // This is raw value in file, and may contain escaped Unicode (e.g. \u00C1)
201+
this.value = value;
133202
this.separator = separator;
134203
};
135204

@@ -150,7 +219,7 @@ public static PropDef parseLine(String line) {
150219
sep = PropSeparator.SPACE;
151220
}
152221
} else {
153-
if (i > 0 && line.charAt(i-1) != '\\') {
222+
if (i > 0 && line.charAt(i - 1) != '\\') {
154223
if (iChar == ' ') {
155224
sawSpace = true;
156225
} else if (iChar == PropSeparator.EQUAL.getCharacter()) {
@@ -171,8 +240,10 @@ public static PropDef parseLine(String line) {
171240
return null;
172241
}
173242

174-
PropDef pl = new PropDef(line.substring(0, sepIdx).trim(),
175-
line.substring(sepIdx + 1).trim(), sep);
243+
String key = unescapePropKey(line.substring(0, sepIdx).trim());
244+
String value = unescapePropValue(line.substring(sepIdx + 1).trim());
245+
246+
PropDef pl = new PropDef(key, value, sep);
176247
return pl;
177248
}
178249

@@ -190,14 +261,15 @@ public PropSeparator getSeparator() {
190261

191262
public void print(PrintWriter pw, String language) throws IOException {
192263
StringBuilder buf = new StringBuilder(100);
193-
int len = key.length() + value.length() + 3; /* 3 - length of separator plus two SPs */
264+
int len = key.length() + value.length()
265+
+ 3; /* 3 - length of separator plus two SPs */
194266

195267
if (len <= COLMAX) {
196268
// Print this property in a single line
197269
if (separator.getCharacter() == PropSeparator.SPACE.getCharacter()) {
198-
buf.append(key).append(separator.getCharacter());
270+
buf.append(escapePropKey(key)).append(separator.getCharacter());
199271
} else {
200-
buf.append(key).append(' ').append(separator.getCharacter()).append(' ');
272+
buf.append(escapePropKey(key)).append(' ').append(separator.getCharacter()).append(' ');
201273
}
202274
buf.append(escapePropValue(value));
203275
pw.println(buf.toString());
@@ -208,9 +280,9 @@ public void print(PrintWriter pw, String language) throws IOException {
208280

209281
// always prints out key and separator in a single line
210282
if (separator.getCharacter() == PropSeparator.SPACE.getCharacter()) {
211-
buf.append(key).append(separator.getCharacter());
283+
buf.append(escapePropKey(key)).append(separator.getCharacter());
212284
} else {
213-
buf.append(key).append(' ').append(separator.getCharacter()).append(' ');
285+
buf.append(escapePropKey(key)).append(' ').append(separator.getCharacter()).append(' ');
214286
}
215287

216288
if (buf.length() > COLMAX) {
@@ -264,7 +336,8 @@ public void print(PrintWriter pw, String language) throws IOException {
264336

265337
@Override
266338
public boolean equals(Object obj) {
267-
if (obj.getClass() != PropDef.class) return false;
339+
if (obj.getClass() != PropDef.class)
340+
return false;
268341
PropDef p = (PropDef) obj;
269342
return getKey().equals(p.getKey()) && getValue().equals(p.getValue())
270343
&& getSeparator().getCharacter() == p.getSeparator().getCharacter();
@@ -290,8 +363,7 @@ private static String escapePropValue(String s) {
290363
if (c == '\\') {
291364
escaped.append("\\\\");
292365
} else if (c > 0x7F) {
293-
escaped.append("\\u")
294-
.append(String.format("%04X", (int)c));
366+
escaped.append("\\u").append(String.format("%04X", (int) c));
295367
} else if (c == ':') {
296368
escaped.append("\\:");
297369
} else if (c == '=') {
@@ -303,23 +375,54 @@ private static String escapePropValue(String s) {
303375
return escaped.toString();
304376
}
305377

378+
private static String unescapePropValue(String s) {
379+
StringBuilder unescaped = new StringBuilder();
380+
StringCharacterIterator itr = new StringCharacterIterator(s);
381+
for (char c = itr.first(); c != CharacterIterator.DONE; c = itr.next()) {
382+
if (c == '\\' && itr.getIndex() < itr.getEndIndex()) {
383+
char n = itr.next();
384+
if (n == '\\' || n == ':' || n == '=') {
385+
unescaped.append(n);
386+
} else if (n == 'u' && itr.getIndex() + 4 <= itr.getEndIndex()) {
387+
StringBuilder unicodeEscape = new StringBuilder("\\u");
388+
for (int i = 0; i < 4; i++) {
389+
unicodeEscape.append(itr.next());
390+
}
391+
unescaped.append(StringEscapeUtils.unescapeJava(unicodeEscape.toString()));
392+
} else {
393+
unescaped.append(c);
394+
unescaped.append(n);
395+
}
396+
} else {
397+
unescaped.append(c);
398+
}
399+
}
400+
return unescaped.toString();
401+
}
402+
306403
private static String escapePropKey(String s) {
307404
return s.replace(" ", "\\ ");
308405
}
309406

407+
private static String unescapePropKey(String s) {
408+
return s.replaceAll("\\\\ ", " ");
409+
}
410+
310411
@Override
311412
public void merge(InputStream base, OutputStream outStream, String language, Bundle resource) throws IOException {
312-
Map<String, String> resMap = new HashMap<String, String>(resource.getResourceStrings().size() * 4/3 + 1);
413+
Map<String, String> resMap = new HashMap<String, String>(resource.getResourceStrings().size() * 4 / 3 + 1);
313414
for (ResourceString res : resource.getResourceStrings()) {
314-
resMap.put(escapePropKey(res.getKey()), res.getValue());
415+
resMap.put(res.getKey(), res.getValue());
315416
}
316417

317418
BufferedReader baseReader = new BufferedReader(new InputStreamReader(base, PROPS_ENC));
318419
PrintWriter outWriter = new PrintWriter(new OutputStreamWriter(outStream, PROPS_ENC));
319420

320421
String line = null;
321422
StringBuilder logicalLineBuf = new StringBuilder();
322-
List<String> orgLines = new ArrayList<String>(8); // default size - up to 8 continuous lines
423+
List<String> orgLines = new ArrayList<String>(8); // default size - up
424+
// to 8 continuous
425+
// lines
323426
do {
324427
// logical line that may define a single property, or empty line
325428
String logicalLine = null;
@@ -339,7 +442,7 @@ public void merge(InputStream base, OutputStream outStream, String language, Bun
339442
if (normLine.startsWith("#") || normLine.startsWith("!")) {
340443
// Comment line - print the original line
341444
outWriter.println(line);
342-
} else if (normLine.endsWith("\\")) {
445+
} else if (isContinuationLine(normLine)) {
343446
// Continue to the next line
344447
logicalLineBuf.append(normLine, 0, normLine.length() - 1);
345448
orgLines.add(line);
@@ -351,7 +454,7 @@ public void merge(InputStream base, OutputStream outStream, String language, Bun
351454
if (normLine.endsWith("\\")) {
352455
// continues to the next line
353456
logicalLineBuf.append(normLine.substring(0, normLine.length() - 1));
354-
orgLines.add(line); // preserve the original line
457+
orgLines.add(line); // preserve the original line
355458
} else {
356459
// terminating the current logical property line
357460
logicalLineBuf.append(normLine);

‎gp-res-filter/src/test/java/com/ibm/g11n/pipeline/resfilter/JavaPropertiesResourceTest.java‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,18 @@ public class JavaPropertiesResourceTest {
6060
List<ResourceString> lst = new LinkedList<>();
6161
lst.add(new ResourceString("website", "http://en.wikipedia.org/", 1));
6262
lst.add(new ResourceString("language", "English", 2));
63-
lst.add(new ResourceString("message", "Welcome to Wikipedia!", 3));
63+
List<String> resourceNotes = new ArrayList<>();
64+
resourceNotes.add(" The backslash below tells the application to continue reading");
65+
resourceNotes.add(" the value onto the next line.");
66+
lst.add(new ResourceString("message", "Welcome to Wikipedia!", 3, resourceNotes));
67+
resourceNotes.clear();
68+
resourceNotes.add(" Add spaces to the key");
6469
lst.add(new ResourceString("key with spaces",
65-
"This is the value that could be looked up with the key \"key with spaces\".", 4));
66-
lst.add(new ResourceString("tab", "pick up the\u00A5 tab", 5));
70+
"This is the value that could be looked up with the key \"key with spaces\".", 4, resourceNotes));
71+
72+
ResourceString rs5 = new ResourceString("tab", "pick up the\u00A5 tab", 5);
73+
rs5.addNote(" Unicode");
74+
lst.add(rs5);
6775
Collections.sort(lst, new ResourceStringComparator());
6876
EXPECTED_INPUT_RES_LIST = lst;
6977
}
@@ -84,12 +92,12 @@ public class JavaPropertiesResourceTest {
8492

8593
static {
8694
EXPECTED_PROP_DEF_LIST = new LinkedList<PropDef>();
87-
EXPECTED_PROP_DEF_LIST.add(new PropDef("website", "http\\://en.wikipedia.org/", PropSeparator.EQUAL));
95+
EXPECTED_PROP_DEF_LIST.add(new PropDef("website", "http://en.wikipedia.org/", PropSeparator.EQUAL));
8896
EXPECTED_PROP_DEF_LIST.add(new PropDef("language", "English", PropSeparator.SPACE));
8997
EXPECTED_PROP_DEF_LIST.add(new PropDef("message", "Welcome to Wikipedia!", PropSeparator.COLON));
90-
EXPECTED_PROP_DEF_LIST.add(new PropDef("key\\ with\\ spaces",
98+
EXPECTED_PROP_DEF_LIST.add(new PropDef("key with spaces",
9199
"This is the value that could be looked up with the key \"key with spaces\".", PropSeparator.EQUAL));
92-
EXPECTED_PROP_DEF_LIST.add(new PropDef("tab", "pick up the\\u00A5 tab", PropSeparator.COLON));
100+
EXPECTED_PROP_DEF_LIST.add(new PropDef("tab", "pick up the\u00A5 tab", PropSeparator.COLON));
93101
}
94102

95103
private static final JavaPropertiesResource res = new JavaPropertiesResource();

‎gp-res-filter/src/test/resource/resfilter/properties/input.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
! The exclamation mark can also mark text as comments.
33
# The key and element characters #, !, =, and : are written with
44
# a preceding backslash to ensure that they are properly loaded.
5+
56
website = http\://en.wikipedia.org/
67
language English
78
# The backslash below tells the application to continue reading

‎gp-res-filter/src/test/resource/resfilter/properties/merge-output.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
! The exclamation mark can also mark text as comments.
33
# The key and element characters #, !, =, and : are written with
44
# a preceding backslash to ensure that they are properly loaded.
5+
56
website = http\://en.wikipedia.org/translated
67
language Not-English
78
# The backslash below tells the application to continue reading

0 commit comments

Comments
 (0)