[272375] support for UTF string literals in LR parser

10 files changed, 90 insertions, 69 deletions

diff --git a/lrparser/org.eclipse.cdt.core.lrparser/META-INF/MANIFEST.MF b/lrparser/org.eclipse.cdt.core.lrparser/META-INF/MANIFEST.MF
index 403832ca95b..0f1f07b721f 100644
--- a/lrparser/org.eclipse.cdt.core.lrparser/META-INF/MANIFEST.MF
+++ b/lrparser/org.eclipse.cdt.core.lrparser/META-INF/MANIFEST.MF
@@ -12,6 +12,7 @@ Export-Package: org.eclipse.cdt.core.dom.lrparser,
  org.eclipse.cdt.core.dom.lrparser.action,
  org.eclipse.cdt.core.dom.lrparser.action.c99,
  org.eclipse.cdt.core.dom.lrparser.action.cpp,
+ org.eclipse.cdt.core.dom.lrparser.action.gnu,
  org.eclipse.cdt.core.dom.lrparser.c99,
  org.eclipse.cdt.core.dom.lrparser.cpp,
  org.eclipse.cdt.core.dom.lrparser.gnu,
diff --git a/lrparser/org.eclipse.cdt.core.lrparser/grammar/build.xml b/lrparser/org.eclipse.cdt.core.lrparser/grammar/build.xml
index f1a71b9c8de..2dfb0d5191b 100644
--- a/lrparser/org.eclipse.cdt.core.lrparser/grammar/build.xml
+++ b/lrparser/org.eclipse.cdt.core.lrparser/grammar/build.xml
@@ -12,17 +12,8 @@
 <project name="CDT Extensible LR Parser Framework" basedir=".">
     <description>Generates LPG parsers from grammar files</description>
 	
-	
-	<fail unless="lpg_exe">
-		Property $${lpg_exe} not set.
-		This property must be set to the full path to the LPG generator executable.
-	</fail>
-	
-	<fail unless="lpg_template">
-		Property $${lpg_template} not set.
-		This property must be set to the full path to the LPG templates folder.
-	</fail>
-	
+	<import file="generate.xml"/>
+
 	<property name="c99_location" value="../src/org/eclipse/cdt/internal/core/dom/lrparser/c99"/>
 	<property name="gcc_location" value="../src/org/eclipse/cdt/internal/core/dom/lrparser/gcc"/>
 	<property name="cpp_location" value="../src/org/eclipse/cdt/internal/core/dom/lrparser/cpp"/>
@@ -148,24 +139,6 @@
 	
 
 	
-	<target name="generate">
-		<property name="grammar_file" value="${grammar_dir}/${grammar_name}.g"/>
-		<echo message="lpg_exe=${lpg_exe}"/>
-		<echo message="lpg_template=${lpg_template}"/>
-		<echo message="grammar_file=${grammar_file}.g"/>
-		<echo message="output_dir=${output_dir}"/>
-
-		<exec executable="${lpg_exe}">
-			<arg value="${grammar_file}"/>
-			<env key="LPG_TEMPLATE" path="${lpg_template}"/>
-			
-		</exec>
-		
-		<move overwrite="true" toDir="${output_dir}">
-		    <fileset dir=".">
-			    <include name="${grammar_name}*.*"/>
-			</fileset>
-		</move>
-	</target> 
+	
 	
 </project>  
\ No newline at end of file
diff --git a/lrparser/org.eclipse.cdt.core.lrparser/grammar/generate.xml b/lrparser/org.eclipse.cdt.core.lrparser/grammar/generate.xml
new file mode 100644
index 00000000000..6241db95945
--- /dev/null
+++ b/lrparser/org.eclipse.cdt.core.lrparser/grammar/generate.xml
@@ -0,0 +1,64 @@
+<!--
+ Copyright (c) 2009 IBM Corporation and others.
+ All rights reserved. This program and the accompanying materials
+ are made available under the terms of the Eclipse Public License v1.0
+ which accompanies this distribution, and is available at
+ http://www.eclipse.org/legal/epl-v10.html
+
+ Contributors:
+     IBM Corporation - initial API and implementation
+-->
+
+<project name="CDT Extensible LR Parser Framework" basedir=".">
+
+	<!--
+		Common tasks that can be used to generate a parser using LPG and the
+		LR parser framework.
+		
+		There are two ANT properties that must be defined:
+		1) lpg_exe - This property must be set to the full path to the LPG generator executable (lpg.exe).
+	    2) lpg_template - This property must be set to the full path to the folder that contains the LRParserTemplate.g file.
+	
+		Additionally if the $Import or $Include directives are being used in a grammar
+		file then the LPG_INCLUDE environment variable must be set to the directory
+		of the files being included.
+	-->
+	
+	<fail unless="lpg_exe">
+		Property $${lpg_exe} not set.
+		This property must be set to the full path to the LPG generator executable.
+	</fail>
+		
+	<fail unless="lpg_template">
+		Property $${lpg_template} not set.
+		This property must be set to the full path to the LPG templates folder.
+	</fail>
+	
+	
+	<!--
+	Parameters: 
+	${grammar_dir} - directory that contains the grammar files
+	${grammar_name} - the name of the main grammar file to run LPG on (not including the .g extension)
+	${output_dir} - name of directory where generated files should go
+	-->
+	<target name="generate">
+		<property name="grammar_file" value="${grammar_dir}/${grammar_name}.g"/>
+		<echo message="lpg_exe=${lpg_exe}"/>
+		<echo message="lpg_template=${lpg_template}"/>
+		<echo message="grammar_file=${grammar_file}.g"/>
+		<echo message="output_dir=${output_dir}"/>
+
+		<exec executable="${lpg_exe}">
+			<arg value="${grammar_file}"/>
+			<env key="LPG_TEMPLATE" path="${lpg_template}"/>
+		</exec>
+		
+		<move overwrite="true" toDir="${output_dir}">
+		    <fileset dir=".">
+			    <include name="${grammar_name}*.*"/>
+			</fileset>
+		</move>
+	</target>
+	
+	
+</project>
\ No newline at end of file
diff --git a/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/c99/DOMToC99TokenMap.java b/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/c99/DOMToC99TokenMap.java
index f6b679a14a1..e5ec6bf4d94 100644
--- a/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/c99/DOMToC99TokenMap.java
+++ b/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/c99/DOMToC99TokenMap.java
@@ -127,8 +127,12 @@ public final class DOMToC99TokenMap implements IDOMTokenMap {
 			case tFLOATINGPT   : return TK_floating;
 			case tSTRING       : return TK_stringlit;
 			case tLSTRING      : return TK_stringlit;
+			case tUTF16STRING  : return TK_stringlit;
+			case tUTF32STRING  : return TK_stringlit;
 			case tCHAR         : return TK_charconst;
 			case tLCHAR        : return TK_charconst;
+	        case tUTF16CHAR    : return TK_charconst;
+	        case tUTF32CHAR    : return TK_charconst;
 			case t__Bool       : return TK__Bool;
 			case t__Complex    : return TK__Complex;
 			case t__Imaginary  : return TK__Imaginary;
diff --git a/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/cpp/DOMToISOCPPTokenMap.java b/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/cpp/DOMToISOCPPTokenMap.java
index a937207c53d..19cc66af087 100644
--- a/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/cpp/DOMToISOCPPTokenMap.java
+++ b/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/cpp/DOMToISOCPPTokenMap.java
@@ -162,8 +162,12 @@ public class DOMToISOCPPTokenMap implements IDOMTokenMap {
 			case tFLOATINGPT   : return TK_floating;
 			case tSTRING       : return TK_stringlit;
 			case tLSTRING      : return TK_stringlit;
+			case tUTF16STRING  : return TK_stringlit;
+			case tUTF32STRING  : return TK_stringlit;
 			case tCHAR         : return TK_charconst;
 			case tLCHAR        : return TK_charconst;
+	        case tUTF16CHAR    : return TK_charconst;
+	        case tUTF32CHAR    : return TK_charconst;
 			case tCOMPLETION   : return TK_Completion;
 			case tEOC          : return TK_EndOfCompletion;
 			case tEND_OF_INPUT : return TK_EOF_TOKEN;
diff --git a/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/gnu/DOMToGCCTokenMap.java b/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/gnu/DOMToGCCTokenMap.java
index 41832355ad4..335b0e5d9b1 100644
--- a/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/gnu/DOMToGCCTokenMap.java
+++ b/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/gnu/DOMToGCCTokenMap.java
@@ -128,8 +128,12 @@ public final class DOMToGCCTokenMap implements IDOMTokenMap {
 			case tFLOATINGPT   : return TK_floating;
 			case tSTRING       : return TK_stringlit;
 			case tLSTRING      : return TK_stringlit;
+			case tUTF16STRING  : return TK_stringlit;
+			case tUTF32STRING  : return TK_stringlit;
 			case tCHAR         : return TK_charconst;
 			case tLCHAR        : return TK_charconst;
+	        case tUTF16CHAR    : return TK_charconst;
+	        case tUTF32CHAR    : return TK_charconst;
 			case t__Bool       : return TK__Bool;
 			case t__Complex    : return TK__Complex;
 			case t__Imaginary  : return TK__Imaginary;
diff --git a/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/gnu/DOMToGPPTokenMap.java b/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/gnu/DOMToGPPTokenMap.java
index 7e64aed9399..8ceea4f9966 100644
--- a/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/gnu/DOMToGPPTokenMap.java
+++ b/lrparser/org.eclipse.cdt.core.lrparser/src/org/eclipse/cdt/core/dom/lrparser/gnu/DOMToGPPTokenMap.java
@@ -163,6 +163,10 @@ public class DOMToGPPTokenMap implements IDOMTokenMap {
 			case tFLOATINGPT   : return TK_floating;
 			case tSTRING       : return TK_stringlit;
 			case tLSTRING      : return TK_stringlit;
+			case tUTF16STRING  : return TK_stringlit;
+			case tUTF32STRING  : return TK_stringlit;
+	        case tUTF16CHAR    : return TK_charconst;
+	        case tUTF32CHAR    : return TK_charconst;
 			case tCHAR         : return TK_charconst;
 			case tLCHAR        : return TK_charconst;
 			case tCOMPLETION   : return TK_Completion;
diff --git a/upc/org.eclipse.cdt.core.parser.upc/grammar/build.xml b/upc/org.eclipse.cdt.core.parser.upc/grammar/build.xml
index fb3fb979f0b..14946c96cc2 100644
--- a/upc/org.eclipse.cdt.core.parser.upc/grammar/build.xml
+++ b/upc/org.eclipse.cdt.core.parser.upc/grammar/build.xml
@@ -1,5 +1,5 @@
 <!--
- Copyright (c) 2006, 2008 IBM Corporation and others.
+ Copyright (c) 2006, 2009 IBM Corporation and others.
  All rights reserved. This program and the accompanying materials
  are made available under the terms of the Eclipse Public License v1.0
  which accompanies this distribution, and is available at
@@ -12,15 +12,7 @@
 <project name="UPC Parser" default="upc" basedir=".">
     <description>Generates the UPC parser from LPG grammar files</description>
 	
-	<fail unless="lpg_exe">
-		Property $${lpg_exe} not set.
-		This property must be set to the full path to the LPG generator executable.
-	</fail>
-	
-	<fail unless="lpg_template">
-		Property $${lpg_template} not set.
-		This property must be set to the full path to the LPG templates folder.
-	</fail>
+	<import file="../../org.eclipse.cdt.core.lrparser/grammar/generate.xml"/>
 	
 	<property name="upc_location" value="../src/org/eclipse/cdt/internal/core/dom/parser/upc"/>
 
@@ -52,25 +44,5 @@
 			<param name="grammar_name" value="${grammar_name}"/>
 		</antcall>
 	</target>
-
-	
-	<target name="generate">
-		<property name="grammar_file" value="${grammar_dir}/${grammar_name}.g"/>
-		<echo message="lpg_exe=${lpg_exe}"/>
-		<echo message="lpg_template=${lpg_template}"/>
-		<echo message="grammar_file=${grammar_file}.g"/>
-		<echo message="output_dir=${output_dir}"/>
-
-		<exec executable="${lpg_exe}">
-			<arg value="${grammar_file}"/>
-			<env key="LPG_TEMPLATE" path="${lpg_template}"/>
-		</exec>
-		
-		<move overwrite="true" toDir="${output_dir}">
-		    <fileset dir=".">
-			    <include name="${grammar_name}*.*"/>
-			</fileset>
-		</move>
-	</target> 
 	
 </project>  
\ No newline at end of file
diff --git a/upc/org.eclipse.cdt.core.parser.upc/src/org/eclipse/cdt/core/dom/parser/upc/DOMToUPCTokenMap.java b/upc/org.eclipse.cdt.core.parser.upc/src/org/eclipse/cdt/core/dom/parser/upc/DOMToUPCTokenMap.java
index d626097072b..1a25a3d5f0d 100644
--- a/upc/org.eclipse.cdt.core.parser.upc/src/org/eclipse/cdt/core/dom/parser/upc/DOMToUPCTokenMap.java
+++ b/upc/org.eclipse.cdt.core.parser.upc/src/org/eclipse/cdt/core/dom/parser/upc/DOMToUPCTokenMap.java
@@ -130,8 +130,12 @@ public class DOMToUPCTokenMap implements IDOMTokenMap {
 			case tFLOATINGPT   : return TK_floating;
 			case tSTRING       : return TK_stringlit;
 			case tLSTRING      : return TK_stringlit;
+			case tUTF16STRING  : return TK_stringlit;
+			case tUTF32STRING  : return TK_stringlit;
 			case tCHAR         : return TK_charconst;
 			case tLCHAR        : return TK_charconst;
+	        case tUTF16CHAR    : return TK_charconst;
+	        case tUTF32CHAR    : return TK_charconst;
 			case t__Bool       : return TK__Bool;
 			case t__Complex    : return TK__Complex;
 			case t__Imaginary  : return TK__Imaginary;
diff --git a/upc/org.eclipse.cdt.core.parser.upc/src/org/eclipse/cdt/core/dom/upc/UPCLanguage.java b/upc/org.eclipse.cdt.core.parser.upc/src/org/eclipse/cdt/core/dom/upc/UPCLanguage.java
index 2286fb1b05a..e8fc84afb49 100644
--- a/upc/org.eclipse.cdt.core.parser.upc/src/org/eclipse/cdt/core/dom/upc/UPCLanguage.java
+++ b/upc/org.eclipse.cdt.core.parser.upc/src/org/eclipse/cdt/core/dom/upc/UPCLanguage.java
@@ -25,8 +25,6 @@ import org.eclipse.cdt.core.dom.parser.upc.DOMToUPCTokenMap;
 import org.eclipse.cdt.core.dom.parser.upc.UPCLanguageKeywords;
 import org.eclipse.cdt.core.index.IIndex;
 import org.eclipse.cdt.core.model.ICLanguageKeywords;
-import org.eclipse.cdt.core.model.IContributedModelBuilder;
-import org.eclipse.cdt.core.model.ITranslationUnit;
 import org.eclipse.cdt.core.parser.IScanner;
 import org.eclipse.cdt.core.parser.ParserLanguage;
 import org.eclipse.cdt.internal.core.dom.parser.upc.UPCParser;
@@ -51,13 +49,6 @@ public class UPCLanguage extends BaseExtensibleLanguage {
 	protected IParser<IASTTranslationUnit> getParser(IScanner scanner, IIndex index, Set<Options> options) {
 		return new UPCParser(scanner, new DOMToUPCTokenMap(), getBuiltinBindingsProvider(), index, options);
 	}
-
-	/**
-	 * @param tu Not used, default model builder used. 
-	 */
-	public IContributedModelBuilder createModelBuilder(ITranslationUnit tu) {
-		return null;
-	}
 	
 	public String getId() {
 		return ID;