Skip to main content
summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorpelder2009-04-07 13:44:54 -0400
committerpelder2009-04-07 13:44:54 -0400
commit20424a453b966e5df26b996670172f1ff4add4a4 (patch)
treeb6364a7ae44351339f9af3a46523bfaa06b9304d
parentd144ba8d45d13151937914937e6a88d96a04b045 (diff)
downloadorg.eclipse.jet-20424a453b966e5df26b996670172f1ff4add4a4.tar.gz
org.eclipse.jet-20424a453b966e5df26b996670172f1ff4add4a4.tar.xz
org.eclipse.jet-20424a453b966e5df26b996670172f1ff4add4a4.zip
[270985] International characters is JET templates cause PDE build problems with the generated Java source
-rw-r--r--plugins/org.eclipse.jet.core/src/org/eclipse/jet/core/compiler/JETCompilerOptions.java6
-rw-r--r--plugins/org.eclipse.jet.editor/plugin.xml2
-rw-r--r--plugins/org.eclipse.jet/plugin.properties1
-rw-r--r--plugins/org.eclipse.jet/plugin.xml8
-rw-r--r--plugins/org.eclipse.jet/src/org/eclipse/jet/internal/builder/WorkspaceCompiler.java56
-rw-r--r--plugins/org.eclipse.jet/src/org/eclipse/jet/internal/compiler/SimplifiedCompiler.java7
-rw-r--r--plugins/org.eclipse.jet/src/org/eclipse/jet/internal/compiler/UnicodeEscapeUtil.java131
-rw-r--r--tests/org.eclipse.jet.tests/src/org/eclipse/jet/tests/compiler/TestSimplifiedCompiler.java6
8 files changed, 188 insertions, 29 deletions
diff --git a/plugins/org.eclipse.jet.core/src/org/eclipse/jet/core/compiler/JETCompilerOptions.java b/plugins/org.eclipse.jet.core/src/org/eclipse/jet/core/compiler/JETCompilerOptions.java
index 8c92c3f..bceb55d 100644
--- a/plugins/org.eclipse.jet.core/src/org/eclipse/jet/core/compiler/JETCompilerOptions.java
+++ b/plugins/org.eclipse.jet.core/src/org/eclipse/jet/core/compiler/JETCompilerOptions.java
@@ -1,7 +1,7 @@
/**
* <copyright>
*
- * Copyright (c) 2007, 2008 IBM Corporation and others.
+ * Copyright (c) 2007, 2009 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -12,7 +12,7 @@
*
* </copyright>
*
- * $Id: JETCompilerOptions.java,v 1.4 2008/04/02 16:05:11 pelder Exp $
+ * $Id: JETCompilerOptions.java,v 1.5 2009/04/07 17:44:50 pelder Exp $
*/
package org.eclipse.jet.core.compiler;
@@ -40,7 +40,7 @@ public final class JETCompilerOptions {
/**
* The default value of the {@link #OPTION_TEMPLATE_EXT} option; value: "jet".
*/
- public static final String DEFAULT_TEMPLATE_EXT = "jet"; //$NON-NLS-1$
+ public static final String DEFAULT_TEMPLATE_EXT = "jet,jet2"; //$NON-NLS-1$
/**
* The default value of the {@link #OPTION_SET_JAVA_FILES_AS_DERIVED} option (Boolean.TRUE).
diff --git a/plugins/org.eclipse.jet.editor/plugin.xml b/plugins/org.eclipse.jet.editor/plugin.xml
index 8edf41c..3383cb2 100644
--- a/plugins/org.eclipse.jet.editor/plugin.xml
+++ b/plugins/org.eclipse.jet.editor/plugin.xml
@@ -9,7 +9,7 @@
class="org.eclipse.jet.internal.editor.JETTextEditor"
contributorClass="org.eclipse.ui.texteditor.BasicTextEditorActionContributor"
default="true"
- extensions="jet"
+ extensions="jet,jet2"
icon="icons/full/obj16/JETTemplateFile.gif"
id="org.eclipse.jet.editor.JETEditor"
name="%editor.name.0">
diff --git a/plugins/org.eclipse.jet/plugin.properties b/plugins/org.eclipse.jet/plugin.properties
index c7d383b..1594422 100644
--- a/plugins/org.eclipse.jet/plugin.properties
+++ b/plugins/org.eclipse.jet/plugin.properties
@@ -42,6 +42,7 @@ markers.compileProblem.name = JET Template Compile Problem
markers.taskmarker.name = Task
contentTypes.template.name = JET Template
+contentTypes.templateUnicode.name = JET Template (Unicode)
extpoint.tagLibraries.name = Tag Libraries
extpoint.deployedTransforms.name = Deployed Transformations
diff --git a/plugins/org.eclipse.jet/plugin.xml b/plugins/org.eclipse.jet/plugin.xml
index a60f533..0d01f26 100644
--- a/plugins/org.eclipse.jet/plugin.xml
+++ b/plugins/org.eclipse.jet/plugin.xml
@@ -1422,6 +1422,14 @@ See the documentation for the Java class java.util.Locale for details on languag
id="template"
name="%contentTypes.template.name"
priority="normal"/>
+ <content-type
+ base-type="org.eclipse.core.runtime.text"
+ default-charset="UTF-8"
+ file-extensions="jet2"
+ id="templateUnicode"
+ name="%contentTypes.templateUnicode.name"
+ priority="normal">
+ </content-type>
</extension>
<extension
point="org.eclipse.core.runtime.preferences">
diff --git a/plugins/org.eclipse.jet/src/org/eclipse/jet/internal/builder/WorkspaceCompiler.java b/plugins/org.eclipse.jet/src/org/eclipse/jet/internal/builder/WorkspaceCompiler.java
index f62ab3b..f6cee99 100644
--- a/plugins/org.eclipse.jet/src/org/eclipse/jet/internal/builder/WorkspaceCompiler.java
+++ b/plugins/org.eclipse.jet/src/org/eclipse/jet/internal/builder/WorkspaceCompiler.java
@@ -12,7 +12,7 @@
*
* </copyright>
*
- * $Id: WorkspaceCompiler.java,v 1.5 2009/04/06 17:55:12 pelder Exp $
+ * $Id: WorkspaceCompiler.java,v 1.6 2009/04/07 17:44:54 pelder Exp $
*/
package org.eclipse.jet.internal.builder;
@@ -20,11 +20,14 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
-import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -52,6 +55,7 @@ import org.eclipse.jet.core.parser.ProblemSeverity;
import org.eclipse.jet.core.parser.ast.Problem;
import org.eclipse.jet.internal.InternalJET2Platform;
import org.eclipse.jet.internal.compiler.SimplifiedCompiler;
+import org.eclipse.jet.internal.compiler.UnicodeEscapeUtil;
import org.eclipse.jet.internal.compiler.SimplifiedCompiler.Builder;
import org.eclipse.jet.internal.core.compiler.ICompilerOutput;
import org.eclipse.jet.internal.core.compiler.IJETCompiler;
@@ -232,24 +236,22 @@ public class WorkspaceCompiler implements IJETCompiler, ICompilerOutput, ITagLib
}
}
+ /* (non-Javadoc)
+ * @see org.eclipse.jet.internal.core.compiler.ICompilerOutput#writeOutput(java.lang.String, java.lang.String, java.lang.String)
+ */
public void writeOutput(String outputFilePath, String contents, String encoding)
{
+ // NOTE: encoding is ignored. Now always write Java files using the default encoding
+ // for those files. This plays better with PDE build and ant, which are incapable of dealing
+ // with individual Java files having non-standard encodings.
try
{
final IFile file = outputLocation.getFile(new Path(outputFilePath));
ensureDirsExist(file.getParent());
- if (!file.exists())
- {
- file.create(new ByteArrayInputStream(new byte []{}), true, new SubProgressMonitor(monitor, 1));
- }
-
- if (encoding != null && !file.getCharset().equals(encoding))
- {
- file.setCharset(encoding, new SubProgressMonitor(monitor, 1));
- }
- byte[] newBytes = contents.getBytes(file.getCharset());
+ final Charset cs = Charset.forName(file.getCharset());
+ final ByteBuffer newContents = UnicodeEscapeUtil.encode(contents, cs);
// check if files exists
if(file.exists())
{
@@ -258,10 +260,12 @@ public class WorkspaceCompiler implements IJETCompiler, ICompilerOutput, ITagLib
try
{
InputStream inputStream = file.getContents();
- byte [] oldBytes = new byte[inputStream.available()];
- inputStream.read(oldBytes);
- inputStream.close();
- writeNewContents = !Arrays.equals(oldBytes, newBytes);
+ final ReadableByteChannel channel = Channels.newChannel(inputStream);
+ final ByteBuffer originalContents = ByteBuffer.allocate(inputStream.available());
+ channel.read(originalContents);
+ channel.close();
+ writeNewContents = !newContents.equals(originalContents);
+ newContents.rewind();
}
catch (IOException exception)
{
@@ -282,15 +286,27 @@ public class WorkspaceCompiler implements IJETCompiler, ICompilerOutput, ITagLib
throw new CoreException(status);
}
}
+
+ final ByteArrayInputStream stream = new ByteArrayInputStream(newContents.array(), 0, newContents.remaining());
+ if (!file.exists())
+ {
+ file.create(stream, true, new SubProgressMonitor(monitor, 1));
+ } else {
+ file.setContents(stream,
+ true, false, new SubProgressMonitor(monitor, 1));
+ }
- file.setContents(new ByteArrayInputStream(newBytes), true, false, new SubProgressMonitor(monitor, 1));
file.setDerived(writeFilesAsDerived);
}
- catch (UnsupportedEncodingException e)
+// catch (UnsupportedEncodingException e)
+// {
+// InternalJET2Platform.logError(JET2Messages.JET2Compiler_ErrorWritingJava, e);
+// }
+ catch (CoreException e)
{
InternalJET2Platform.logError(JET2Messages.JET2Compiler_ErrorWritingJava, e);
}
- catch (CoreException e)
+ catch (CharacterCodingException e)
{
InternalJET2Platform.logError(JET2Messages.JET2Compiler_ErrorWritingJava, e);
}
diff --git a/plugins/org.eclipse.jet/src/org/eclipse/jet/internal/compiler/SimplifiedCompiler.java b/plugins/org.eclipse.jet/src/org/eclipse/jet/internal/compiler/SimplifiedCompiler.java
index e345b44..f96272f 100644
--- a/plugins/org.eclipse.jet/src/org/eclipse/jet/internal/compiler/SimplifiedCompiler.java
+++ b/plugins/org.eclipse.jet/src/org/eclipse/jet/internal/compiler/SimplifiedCompiler.java
@@ -12,7 +12,7 @@
*
* </copyright>
*
- * $Id: SimplifiedCompiler.java,v 1.4 2009/04/06 17:55:11 pelder Exp $
+ * $Id: SimplifiedCompiler.java,v 1.5 2009/04/07 17:44:54 pelder Exp $
*/
package org.eclipse.jet.internal.compiler;
@@ -375,7 +375,10 @@ public final class SimplifiedCompiler implements IJETCompiler
compilerOutput.removeOutput(oldOutputPath);
}
- compilerOutput.writeOutput(outputPath, NewLineUtil.setLineTerminator(code, NL), cu.getOutputEncoding());
+ // Don't encode the .java file the same as the template. This causes
+ // problems with ANT/PDE build, which typically expects default encodings on all Java
+ // files.
+ compilerOutput.writeOutput(outputPath, NewLineUtil.setLineTerminator(code, NL), null);
if(cu.hasErrors() || cu.hasWarnings()) {
compilerOutput.recordProblems(templatePath, cu.getProblems());
diff --git a/plugins/org.eclipse.jet/src/org/eclipse/jet/internal/compiler/UnicodeEscapeUtil.java b/plugins/org.eclipse.jet/src/org/eclipse/jet/internal/compiler/UnicodeEscapeUtil.java
new file mode 100644
index 0000000..aa28b6f
--- /dev/null
+++ b/plugins/org.eclipse.jet/src/org/eclipse/jet/internal/compiler/UnicodeEscapeUtil.java
@@ -0,0 +1,131 @@
+/**
+ * Copyright (c) 2009 IBM Corporation and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ * IBM - Initial API and implementation
+ *
+ */
+package org.eclipse.jet.internal.compiler;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CoderResult;
+import java.nio.charset.CodingErrorAction;
+
+/**
+ * Utility class for encoding Java code
+ *
+ */
+public class UnicodeEscapeUtil {
+
+ private final CharsetEncoder encoder;
+ private final CharBuffer in;
+ private ByteBuffer out;
+ private CharBuffer unicodeEscapes;
+
+ private UnicodeEscapeUtil(Charset charset, CharBuffer in) {
+ this.in = in;
+ encoder = charset.newEncoder()
+ .onUnmappableCharacter(CodingErrorAction.REPORT)
+ .onMalformedInput(CodingErrorAction.REPLACE);
+ int capacity = (int) (in.remaining() * encoder.averageBytesPerChar());
+ out = ByteBuffer.allocate(capacity);
+ // for convenience, ensure uncideEscapes is never null
+ unicodeEscapes = CharBuffer.wrap(""); //$NON-NLS-1$
+ }
+
+ /**
+ * Follow the contract of {@link CharsetEncoder#encode(CharBuffer, ByteBuffer, boolean)},
+ * with {@link CharsetEncoder#onUnmappableCharacter(CodingErrorAction)} set to
+ * {@link CodingErrorAction#REPORT}. On such report, replace the offending characters
+ * with their Unicode escape equivalent \\uXXXX, and continue encoding.
+ * @return
+ * @throws CharacterCodingException
+ */
+ private ByteBuffer doEncode()
+ throws CharacterCodingException {
+
+ if (in.remaining() == 0) {
+ return out;
+ }
+ encoder.reset();
+ for (;;) {
+ CoderResult cr;
+ if(unicodeEscapes.hasRemaining()) {
+ cr = encoder.encode(unicodeEscapes, out, false);
+ } else {
+ //
+ cr = encoder.encode(in, out, false);
+ }
+ if (cr.isUnderflow()) {
+ if(in.hasRemaining()) {
+ continue;
+ } else {
+ break;
+ }
+ }
+ if (cr.isOverflow()) {
+ // re-allocate and copy the output buffer
+ final int newCapacity = out.capacity()
+ + (int)((in.remaining() + unicodeEscapes.remaining()) * encoder.averageBytesPerChar());
+ final ByteBuffer newOut = ByteBuffer.allocate(newCapacity);
+ out.flip();
+ newOut.put(out);
+ out = newOut;
+ continue;
+ }
+ if(cr.isUnmappable()) {
+ // re-write any unmappable characters as unicode escapes
+ StringBuffer buffer = new StringBuffer(6 * cr.length());
+ for(int i = 0; i < cr.length(); i++ ) {
+ int ch = in.get();
+ buffer.append("\\u"); //$NON-NLS-1$
+ String hex = Integer.toHexString(ch);
+ for(int j = 4 - hex.length(); j > 0; j--) buffer.append('0');
+ buffer.append(hex);
+ }
+ unicodeEscapes = CharBuffer.wrap(buffer.toString());
+ continue;
+ }
+ cr.throwException();
+ }
+ // religiously follow the contract for encode.
+ encoder.encode(in, out, true);
+ encoder.flush(out);
+ out.flip();
+ return out;
+
+ }
+
+ /**
+ * Encode the passed contents in the specified character set, re-writting any characters
+ * not representable in that character set as a Java unicode escape (\\uXXXX0).
+ * @param buffer the contents to encode
+ * @param charset the character set or {@link NullPointerException}
+ * @return a ByteBuffer containing the encoded contents
+ * @throws CharacterCodingException if an encoding error occurs.
+ */
+ public static ByteBuffer encode(CharBuffer buffer, Charset charset) throws CharacterCodingException {
+ return new UnicodeEscapeUtil(charset, buffer).doEncode();
+ }
+
+
+ /**
+ * Encode the passed contents in the specified character set, re-writting any characters
+ * not representable in that character set as a Java unicode escape (\\uXXXX0).
+ * @param contents the contents to encode
+ * @param charset the character set or {@link NullPointerException}
+ * @return a ByteBuffer containing the encoded contents
+ * @throws CharacterCodingException if an encoding error occurs.
+ */
+ public static ByteBuffer encode(CharSequence contents, Charset charset) throws CharacterCodingException {
+ return encode(CharBuffer.wrap(contents), charset);
+ }
+}
diff --git a/tests/org.eclipse.jet.tests/src/org/eclipse/jet/tests/compiler/TestSimplifiedCompiler.java b/tests/org.eclipse.jet.tests/src/org/eclipse/jet/tests/compiler/TestSimplifiedCompiler.java
index f02e067..575867c 100644
--- a/tests/org.eclipse.jet.tests/src/org/eclipse/jet/tests/compiler/TestSimplifiedCompiler.java
+++ b/tests/org.eclipse.jet.tests/src/org/eclipse/jet/tests/compiler/TestSimplifiedCompiler.java
@@ -113,7 +113,7 @@ public class TestSimplifiedCompiler extends TestCase {
final String expectedJavaCode = FileUtils.getFileContents(baseLocation.resolve(templatePath + ".expected.java").toString(), true);
assertEquals(expectedJavaCode, compilerOutput.getCode());
- assertEquals("UTF-8", compilerOutput.getEncoding());
+ assertEquals(null, compilerOutput.getEncoding());
}
/**
@@ -155,7 +155,7 @@ public class TestSimplifiedCompiler extends TestCase {
final String expectedJavaCode = FileUtils.getFileContents(baseLocation.resolve(templatePath + ".predefs.expected.java").toString(), true);
assertEquals(expectedJavaCode, compilerOutput.getCode());
- assertEquals("UTF-8", compilerOutput.getEncoding());
+ assertEquals(null, compilerOutput.getEncoding());
}
/**
@@ -175,7 +175,7 @@ public class TestSimplifiedCompiler extends TestCase {
final String expectedJavaCode = FileUtils.getFileContents(baseLocation.resolve(templatePath + ".nopredefs.expected.java").toString(), true);
assertEquals(expectedJavaCode, compilerOutput.getCode());
- assertEquals("UTF-8", compilerOutput.getEncoding());
+ assertEquals(null, compilerOutput.getEncoding());
}

Back to the top