fix bug 102825: problems in scanner around macro pasting

4 files changed, 112 insertions, 110 deletions

diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CSpecFailingTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CSpecFailingTest.java
index 5c5334bcb7c..ac34a41d6e3 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CSpecFailingTest.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CSpecFailingTest.java
@@ -48,52 +48,6 @@ public class AST2CSpecFailingTest extends AST2SpecBaseTest {
 	}
 	
 	/**
-	 [--Start Example(C 6.10.3.5-6):
-	#define str(s) # s
-	#define xstr(s) str(s)
-	#define debug(s, t) printf("x" # s "= %d, x" # t "= %s", \
-	x ## s, x ## t)
-	#define INCFILE(n) vers ## n
-	#define glue(a, b) a ## b
-	#define xglue(a, b) glue(a, b)
-	#define HIGHLOW "hello"
-	#define LOW LOW ", world"
-	int f() {
-	debug(1, 2);
-	fputs(str(strncmp("abc\0d", "abc", '\4') // this goes away
-	== 0) str(: @\n), s);
-	//#include xstr(INCFILE(2).h)
-	glue(HIGH, LOW);
-	xglue(HIGH, LOW)
-	}
-	 --End Example]
-	 */
-	public void test6_10_3_5s6() throws Exception {
-		StringBuffer buffer = new StringBuffer();
-		buffer.append("#define str(s) # s\n"); //$NON-NLS-1$
-		buffer.append("#define xstr(s) str(s)\n"); //$NON-NLS-1$
-		buffer.append("#define debug(s, t) printf(\"x\" # s \"= %d, x\" # t \"= %s\", \\n"); //$NON-NLS-1$
-		buffer.append("x ## s, x ## t)\n"); //$NON-NLS-1$
-		buffer.append("#define INCFILE(n) vers ## n\n"); //$NON-NLS-1$
-		buffer.append("#define glue(a, b) a ## b\n"); //$NON-NLS-1$
-		buffer.append("#define xglue(a, b) glue(a, b)\n"); //$NON-NLS-1$
-		buffer.append("#define HIGHLOW \"hello\"\n"); //$NON-NLS-1$
-		buffer.append("#define LOW LOW \", world\"\n"); //$NON-NLS-1$
-		buffer.append("int f() {\n"); //$NON-NLS-1$
-		buffer.append("debug(1, 2);\n"); //$NON-NLS-1$
-		buffer.append("fputs(str(strncmp(\"abc\0d\", \"abc\", '\4') // this goes away\n"); //$NON-NLS-1$
-		buffer.append("== 0) str(: @\n), s);\n"); //$NON-NLS-1$
-		buffer.append("//#include xstr(INCFILE(2).h)\n"); //$NON-NLS-1$
-		buffer.append("glue(HIGH, LOW);\n"); //$NON-NLS-1$
-		buffer.append("xglue(HIGH, LOW)\n"); //$NON-NLS-1$
-		buffer.append("}\n"); //$NON-NLS-1$
-		try {
-		parseCandCPP(buffer.toString(), false, 0);
-		assertTrue(false);
-		} catch (Exception e) {}
-	}
-	
-	/**
 	 [--Start Example(C 6.10.3.5-5):
 	#define x 3
 	#define f(a) f(x * (a))
@@ -158,36 +112,4 @@ public class AST2CSpecFailingTest extends AST2SpecBaseTest {
 		assertTrue(false);
 		} catch (Exception e) {}
 	}
-	
-	/**
-	 [--Start Example(C 6.10.3.5-9):
-	#define debug(...) fprintf(stderr, _ _VA_ARGS_ _)
-	#define showlist(...) puts(#_ _VA_ARGS_ _)
-	#define report(test, ...) ((test)?puts(#test):\
-	printf(_ _VA_ARGS_ _))
-	int f() {
-	debug("Flag");
-	debug("X = %d\n", x);
-	showlist(The first, second, and third items.);
-	report(x>y, "x is %d but y is %d", x, y);
-	}
-	 --End Example]
-	 */
-	public void test6_10_3_5s9() throws Exception {
-		StringBuffer buffer = new StringBuffer();
-		buffer.append("#define debug(...) fprintf(stderr, _ _VA_ARGS_ _)\n"); //$NON-NLS-1$
-		buffer.append("#define showlist(...) puts(#_ _VA_ARGS_ _)\n"); //$NON-NLS-1$
-		buffer.append("#define report(test, ...) ((test)?puts(#test):\\n"); //$NON-NLS-1$
-		buffer.append("printf(_ _VA_ARGS_ _))\n"); //$NON-NLS-1$
-		buffer.append("int f() {\n"); //$NON-NLS-1$
-		buffer.append("debug(\"Flag\");\n"); //$NON-NLS-1$
-		buffer.append("debug(\"X = %d\n\", x);\n"); //$NON-NLS-1$
-		buffer.append("showlist(The first, second, and third items.);\n"); //$NON-NLS-1$
-		buffer.append("report(x>y, \"x is %d but y is %d\", x, y);\n"); //$NON-NLS-1$
-		buffer.append("}\n"); //$NON-NLS-1$
-		try {
-		parseCandCPP(buffer.toString(), false, 0);
-		assertTrue(false);
-		} catch (Exception e) {}
-	}
 }
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CSpecTest.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CSpecTest.java
index 823d2aad29e..fc518382263 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CSpecTest.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/ast2/AST2CSpecTest.java
@@ -1905,6 +1905,52 @@ public class AST2CSpecTest extends AST2SpecBaseTest {
 	}
 	
 	/**
+	 [--Start Example(C 6.10.3.5-6):
+	#define str(s) # s
+	#define xstr(s) str(s)
+	#define debug(s, t) printf("x" # s "= %d, x" # t "= %s", \
+	x ## s, x ## t)
+	#define INCFILE(n) vers ## n
+	#define glue(a, b) a ## b
+	#define xglue(a, b) glue(a, b)
+	#define HIGHLOW "hello"
+	#define LOW LOW ", world"
+	int f() {
+	debug(1, 2);
+	fputs(str(strncmp("abc\0d", "abc", '\4') // this goes away
+	== 0) str(: @\n), s);
+	//#include xstr(INCFILE(2).h)
+	glue(HIGH, LOW);
+	xglue(HIGH, LOW)
+	}
+	 --End Example]
+	 */
+	public void test6_10_3_5s6() throws Exception {
+		StringBuffer buffer = new StringBuffer();
+		buffer.append("#define str(s) # s                    \n"); //$NON-NLS-1$
+		buffer.append("#define xstr(s) str(s)                \n"); //$NON-NLS-1$
+		buffer.append("#define debug(s, t) printf(\"x\" # s \"= %d, x\" # t \"= %s\", \\\n"); //$NON-NLS-1$
+		buffer.append("x ## s, x ## t)                       \n"); //$NON-NLS-1$
+		buffer.append("#define INCFILE(n) vers ## n          \n"); //$NON-NLS-1$
+		buffer.append("#define glue(a, b) a ## b             \n"); //$NON-NLS-1$
+		buffer.append("#define xglue(a, b) glue(a, b)        \n"); //$NON-NLS-1$
+		buffer.append("#define HIGHLOW \"hello\"             \n"); //$NON-NLS-1$
+		buffer.append("#define LOW LOW \", world\"           \n"); //$NON-NLS-1$
+		buffer.append("void printf( char *, ...);            \n"); //$NON-NLS-1$
+		buffer.append("void fputs( char *, ... );            \n"); //$NON-NLS-1$
+		buffer.append("int x1, x2, s;                        \n"); //$NON-NLS-1$
+		buffer.append("int f() {                             \n"); //$NON-NLS-1$
+		buffer.append("   debug(1, 2);                       \n"); //$NON-NLS-1$
+		buffer.append("   fputs(str(strncmp(\"abc\0d\", \"abc\", '\4') // this goes away\n"); //$NON-NLS-1$
+		buffer.append("   == 0) str(: @\n), s);              \n"); //$NON-NLS-1$
+		buffer.append("   char * c = glue(HIGH, LOW);        \n"); //$NON-NLS-1$
+		buffer.append("   c = xglue(HIGH, LOW);              \n"); //$NON-NLS-1$
+		buffer.append("}                                     \n"); //$NON-NLS-1$
+
+		parseCandCPP(buffer.toString(), false, 0);
+	}
+	
+	/**
 	 [--Start Example(C 6.10.3.5-8):
 	#define OBJ_LIKE1 (1-1)
 	#define OBJ_LIKE2    \
@@ -1927,4 +1973,37 @@ public class AST2CSpecTest extends AST2SpecBaseTest {
 		parseCandCPP(buffer.toString(), true, 0);
 	}
 	
+	/**
+	 [--Start Example(C 6.10.3.5-9):
+	#define debug(...) fprintf(stderr, __VA_ARGS__)
+	#define showlist(...) puts(#__VA_ARGS__)
+	#define report(test, ...) ((test)?puts(#test):\
+	printf(__VA_ARGS__))
+	int f() {
+	debug("Flag");
+	debug("X = %d\n", x);
+	showlist(The first, second, and third items.);
+	report(x>y, "x is %d but y is %d", x, y);
+	}
+	 --End Example]
+	 */
+	public void test6_10_3_5s9() throws Exception {
+		StringBuffer buffer = new StringBuffer();
+		buffer.append("#define debug(...) fprintf(stderr, __VA_ARGS__)\n"); //$NON-NLS-1$
+		buffer.append("#define showlist(...) puts(#__VA_ARGS__)\n"); //$NON-NLS-1$
+		buffer.append("#define report(test, ...) ((test)?puts(#test):\\\n"); //$NON-NLS-1$
+		buffer.append("printf(__VA_ARGS__))                             \n"); //$NON-NLS-1$
+		buffer.append("void fprintf( ... );                             \n"); //$NON-NLS-1$
+		buffer.append("void puts(char * );                              \n"); //$NON-NLS-1$
+		buffer.append("void printf( char *, ... );                      \n"); //$NON-NLS-1$
+		buffer.append("int stderr, x, y;                                \n");//$NON-NLS-1$
+		buffer.append("int f() {                                        \n"); //$NON-NLS-1$
+		buffer.append("   debug(\"Flag\");                              \n"); //$NON-NLS-1$
+		buffer.append("   debug(\"X = %d\\n\", x);                      \n"); //$NON-NLS-1$
+		buffer.append("   showlist(The first, second, and third items.);\n"); //$NON-NLS-1$
+		buffer.append("   report(x>y, \"x is %d but y is %d\", x, y);   \n"); //$NON-NLS-1$
+		buffer.append("}                                                \n"); //$NON-NLS-1$
+
+		parseCandCPP(buffer.toString(), false, 0);
+	}
 }
diff --git a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java
index 9d31c45c019..98c8b7b3bd0 100644
--- a/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java
+++ b/core/org.eclipse.cdt.core.tests/parser/org/eclipse/cdt/core/parser/tests/scanner2/Scanner2Test.java
@@ -45,24 +45,37 @@ public class Scanner2Test extends BaseScanner2Test
     }
 
     
-//    public void testBug102825_2() throws Exception {
-//        StringBuffer buffer = new StringBuffer("#define CURLOPTTYPE_OBJECTPOINT   10000\n" ); //$NON-NLS-1$
-//        buffer.append("#define CINIT(name,type,number) = CURLOPTTYPE_##type + number\n" ); //$NON-NLS-1$
-//        buffer.append("CINIT(FILE, OBJECTPOINT, 1)\n" ); //$NON-NLS-1$
-//        initializeScanner(buffer.toString());
-//        validateToken( IToken.tASSIGN );
-//        validateInteger( "10000"); //$NON-NLS-1$
-//    }
-
-//    public void testBug102825_3() throws Exception {
-//        StringBuffer buffer = new StringBuffer("#define CURLOPTTYPE_OBJECTPOINT   10000\n" ); //$NON-NLS-1$
-//        buffer.append("#define CINIT(name,type,number) CURLOPT_##name = CURLOPTTYPE_##type + number\n" ); //$NON-NLS-1$
-//        buffer.append("CINIT(FILE, OBJECTPOINT, 1)\n" ); //$NON-NLS-1$
-//        initializeScanner(buffer.toString());
-//        validateIdentifier( "CURLOPT_FILE"); //$NON-NLS-1$
-//        validateToken( IToken.tASSIGN );
-//        validateInteger( "10000"); //$NON-NLS-1$
-//    }
+    public void testBug102825_2() throws Exception {
+        StringBuffer buffer = new StringBuffer("#define CURLOPTTYPE_OBJECTPOINT   10000\n" ); //$NON-NLS-1$
+        buffer.append("#define CINIT(name,type,number) = CURLOPTTYPE_##type + number\n" ); //$NON-NLS-1$
+        buffer.append("CINIT(FILE, OBJECTPOINT, 1)\n" ); //$NON-NLS-1$
+        initializeScanner(buffer.toString());
+        validateToken( IToken.tASSIGN );
+        validateInteger( "10000"); //$NON-NLS-1$
+    }
+
+    public void testBug102825_3() throws Exception {
+        StringBuffer buffer = new StringBuffer("#define CURLOPTTYPE_OBJECTPOINT   10000\n" ); //$NON-NLS-1$
+        buffer.append("#define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_	## type + number\n" ); //$NON-NLS-1$
+        buffer.append("CINIT(FILE, OBJECTPOINT, 1)\n" ); //$NON-NLS-1$
+        initializeScanner(buffer.toString());
+        validateIdentifier( "CURLOPT_FILE"); //$NON-NLS-1$
+        validateToken( IToken.tASSIGN );
+        validateInteger( "10000"); //$NON-NLS-1$
+    }
+    
+    public void testBug102825_4() throws Exception {
+    	StringBuffer buffer = new StringBuffer();
+    	buffer.append("#define glue( a, b ) a ## b\n"); //$NON-NLS-1$
+    	buffer.append("#define HIGHLOW \"hello\"\n"); //$NON-NLS-1$
+    	buffer.append("glue( HIGH, LOW )\n"); //$NON-NLS-1$
+    	
+    	initializeScanner(buffer.toString());
+        Callback callback = new Callback( ParserMode.QUICK_PARSE );
+		initializeScanner( buffer.toString(), ParserMode.QUICK_PARSE, callback );
+		validateString( "hello"); //$NON-NLS-1$
+		assertEquals( callback.problems.size(), 0 );
+    }
     
 	public class TableRow
 	{
diff --git a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/BaseScanner.java b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/BaseScanner.java
index f908673f59b..68f1f2a97a1 100644
--- a/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/BaseScanner.java
+++ b/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/scanner2/BaseScanner.java
@@ -4243,7 +4243,7 @@ abstract class BaseScanner implements IScanner {
                         System.arraycopy(repObject, 0, newRep, l1, l2);
                     else
                         System.arraycopy(expansion, idstart, newRep, l1, l2);
-
+                    idstart = prevArgStart;
                     repObject = newRep;
                 }
                 if (repObject != null) {
@@ -4396,19 +4396,7 @@ abstract class BaseScanner implements IScanner {
                         }
                         break;
                     }
-
-                    // copy everything up to the whitespace
-                    int n = wsstart - (++lastcopy);
-                    if (n > 0 && result != null)
-                        System
-                                .arraycopy(expansion, lastcopy, result, outpos,
-                                        n);
-                    outpos += n;
-
-                    // skip over the ## and the whitespace around it
-                    lastcopy = --pos;
-                    wsstart = -1;
-
+                    --pos;
                 } else {
                     prevConcat = false;
                     // stringify