Bug 490356 - String literals in AST Tree are escaped twice.

This commit fixes the problem by replacing using obsolete Scanner with
new utility class JsStringScanner.

Change-Id: I416de43d5f708f4c1a581619f04d95e737b6c91c
Signed-off-by: Eugene Melekhov <emvv@mail.ru>

5 files changed, 694 insertions, 140 deletions

diff --git a/bundles/org.eclipse.wst.jsdt.core/src/org/eclipse/wst/jsdt/core/dom/StringLiteral.java b/bundles/org.eclipse.wst.jsdt.core/src/org/eclipse/wst/jsdt/core/dom/StringLiteral.java
index 3e7420479..5839d3515 100644
--- a/bundles/org.eclipse.wst.jsdt.core/src/org/eclipse/wst/jsdt/core/dom/StringLiteral.java
+++ b/bundles/org.eclipse.wst.jsdt.core/src/org/eclipse/wst/jsdt/core/dom/StringLiteral.java
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2011 IBM Corporation and others.
+ * Copyright (c) 2000, 2016 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -14,23 +14,21 @@ package org.eclipse.wst.jsdt.core.dom;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.eclipse.wst.jsdt.core.compiler.InvalidInputException;
-import org.eclipse.wst.jsdt.internal.compiler.parser.Scanner;
-import org.eclipse.wst.jsdt.internal.compiler.parser.TerminalTokens;
+import org.eclipse.wst.jsdt.core.util.JsStringScanner;
 
 /**
  * String literal nodes.
- * 
- * Provisional API: This class/interface is part of an interim API that is still under development and expected to 
- * change significantly before reaching stability. It is being made available at this early stage to solicit feedback 
- * from pioneering adopters on the understanding that any code that uses this API will almost certainly be broken 
+ *
+ * Provisional API: This class/interface is part of an interim API that is still under development and expected to
+ * change significantly before reaching stability. It is being made available at this early stage to solicit feedback
+ * from pioneering adopters on the understanding that any code that uses this API will almost certainly be broken
  * (repeatedly) as the API evolves.
  */
 public class StringLiteral extends Expression {
 
 	/**
 	 * The "escapedValue" structural property of this node type.
-	 *  
+	 *
 	 */
 	public static final SimplePropertyDescriptor ESCAPED_VALUE_PROPERTY =
 		new SimplePropertyDescriptor(StringLiteral.class, "escapedValue", String.class, MANDATORY); //$NON-NLS-1$
@@ -58,7 +56,7 @@ public class StringLiteral extends Expression {
 
 	 * @return a list of property descriptors (element type:
 	 * {@link StructuralPropertyDescriptor})
-	 *  
+	 *
 	 */
 	public static List propertyDescriptors(int apiLevel) {
 		return PROPERTY_DESCRIPTORS;
@@ -172,19 +170,10 @@ public class StringLiteral extends Expression {
 		if (token == null) {
 			throw new IllegalArgumentException("Token cannot be null"); //$NON-NLS-1$
 		}
-		Scanner scanner = this.ast.scanner;
-		char[] source = token.toCharArray();
-		scanner.setSource(source);
-		scanner.resetTo(0, source.length);
 		try {
-			int tokenType = scanner.getNextToken();
-			switch(tokenType) {
-				case TerminalTokens.TokenNameStringLiteral:
-					break;
-				default:
-					throw new IllegalArgumentException("Invalid string literal : >" + token + "<"); //$NON-NLS-1$//$NON-NLS-2$
-			}
-		} catch(InvalidInputException e) {
+			new JsStringScanner(token).scan();
+		}
+		catch (IllegalArgumentException e) {
 			throw new IllegalArgumentException("Invalid string literal : >" + token + "<");//$NON-NLS-1$//$NON-NLS-2$
 		}
 		preValueChange(ESCAPED_VALUE_PROPERTY);
@@ -221,30 +210,7 @@ public class StringLiteral extends Expression {
 	 * @exception IllegalArgumentException if the literal value cannot be converted
 	 */
 	public String getLiteralValue() {
-		String s = getEscapedValue();
-		int len = s.length();
-		char zeroth = s.charAt(0);
-		char last = s.charAt(len - 1);
-		if (len < 2 || (zeroth != '\"' && zeroth != '\'') || (last != '\"' && last != '\'')) {
-			throw new IllegalArgumentException();
-		}
-
-		Scanner scanner = this.ast.scanner;
-		char[] source = s.toCharArray();
-		scanner.setSource(source);
-		scanner.resetTo(0, source.length);
-		try {
-			int tokenType = scanner.getNextToken();
-			switch(tokenType) {
-				case TerminalTokens.TokenNameStringLiteral:
-				case TerminalTokens.TokenNameCharacterLiteral:
-					return scanner.getCurrentStringLiteral();
-				default:
-					throw new IllegalArgumentException();
-			}
-		} catch(InvalidInputException e) {
-			throw new IllegalArgumentException();
-		}
+		return new JsStringScanner(getEscapedValue()).scan();
 	}
 
 	/**
@@ -293,6 +259,15 @@ public class StringLiteral extends Expression {
 				case '\r' :
 					b.append("\\r"); //$NON-NLS-1$
 					break;
+				case '\u000B' :
+					b.append("\\v"); //$NON-NLS-1$
+					break;
+				case '\u2028' :
+					b.append("\\u2028"); //$NON-NLS-1$
+					break;
+				case '\u2029' :
+					b.append("\\u2029"); //$NON-NLS-1$
+					break;
 				case '\"':
 					b.append("\\\""); //$NON-NLS-1$
 					break;
diff --git a/bundles/org.eclipse.wst.jsdt.core/src/org/eclipse/wst/jsdt/core/util/JsStringScanner.java b/bundles/org.eclipse.wst.jsdt.core/src/org/eclipse/wst/jsdt/core/util/JsStringScanner.java
new file mode 100644
index 000000000..a3ff91a6b
--- /dev/null
+++ b/bundles/org.eclipse.wst.jsdt.core/src/org/eclipse/wst/jsdt/core/util/JsStringScanner.java
@@ -0,0 +1,352 @@
+/*******************************************************************************
+ * Copyright (c) 2016 Eugene Melekhov and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Distribution License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * Contributors:
+ *   Eugene Melekhov - initial API and implementation
+ *******************************************************************************/
+
+package org.eclipse.wst.jsdt.core.util;
+
+/**
+ * Scanner to parse JavaScript string as it is written in JavaSript source
+ * code like {@code "aaa\n"} or {@code 'aaa\n"\ u{58}"'}. It's used in
+ * {@link StringLiteral} to convert JavaScript string into native Java String
+ *
+ * @since 2.0
+ */
+public class JsStringScanner {
+
+	/**
+	 * JavaScript string source to scan
+	 */
+	private String source;
+
+	/**
+	 * Current scanner position
+	 */
+	private int index;
+
+	/**
+	 * Source length
+	 */
+	private int length;
+
+	/**
+	 * Current read symbol
+	 */
+	private int current;
+
+	/**
+	 * Output accumulator
+	 */
+	private StringBuilder result = new StringBuilder();
+
+	/**
+	 * Create new Scanner
+	 *
+	 * @param source
+	 *            JavaScript string to scan
+	 */
+	public JsStringScanner(String source) {
+		setSource(source);
+	}
+
+	/**
+	 * Set JavaScript string to parse
+	 *
+	 * @param source
+	 *            JavaScript string to scan
+	 */
+	public void setSource(String source) {
+		this.source = source;
+		index = 0;
+		length = source.length();
+		result.setLength(0);
+	}
+
+	/**
+	 * Scans JavaScript string set in the constructor or via
+	 * {@link #setSource(String)}
+	 *
+	 * @exception IllegalArgumentException
+	 *                in case of parse error
+	 * @return literal representation of the scanned JavaScript string
+	 */
+	public String scan() {
+		read();
+		int quote = current;
+		if (quote != '"' && quote != '\'') {
+			error();
+		}
+		read();
+		while (!isEof()) {
+			if (current == quote) {
+				return result.toString();
+			}
+			else if (current == '\\') {
+				scanEscape();
+			}
+			else if (isEol()) {
+				error();
+			}
+			else {
+				append();
+				read();
+			}
+		}
+
+		throw new IllegalArgumentException();
+	}
+
+	/**
+	 * Scan Escape Sequence
+	 *
+	 * @exception IllegalArgumentException
+	 *                in case of parse error
+	 */
+	private void scanEscape() {
+		read();
+		checkEof();
+		if (isEol()) {
+			int ch = current;
+			read();
+			if (ch == '\r' && current == '\n') {
+				read();
+			}
+		}
+		else {
+			switch (current) {
+				case 'n' :
+					append('\n');
+					read();
+					break;
+				case 'r' :
+					append('\r');
+					read();
+					break;
+				case 't' :
+					append('\t');
+					read();
+					break;
+				case 'u' :
+					read();
+					scanUnicodeEscape();
+					break;
+				case 'x' :
+					read();
+					scanHexDigits(2);
+					break;
+				case 'b' :
+					append('\b');
+					read();
+					break;
+				case 'f' :
+					append('\f');
+					read();
+					break;
+				case 'v' :
+					append("\u000B"); //$NON-NLS-1$
+					read();
+					break;
+				default :
+					if (current >= '0' && current <= '7') {
+						int code = 0;
+						for (int toRead = (current >= '0' && current <= '3') ? 3 : 2; toRead > 0 && current >= '0' && current <= '7'; toRead--) {
+							code = (code << 3) + (current - '0');
+							read();
+							checkEof();
+						}
+						append(codePointToString(code));
+					}
+					else if (current == '8' || current == '9') {
+						error();
+					}
+					else {
+						append();
+						read();
+					}
+			}
+		}
+	}
+
+	/**
+	 * Scan unicode escape sequence
+	 *
+	 * @exception IllegalArgumentException
+	 *                in case of parse error
+	 */
+	private void scanUnicodeEscape() {
+		checkEof();
+		if (current == '{') {
+			read();
+			scanHexDigits();
+			if (current != '}') {
+				error();
+			}
+			read();
+		}
+		else {
+			scanHexDigits(4);
+		}
+	}
+
+	/**
+	 * Scans arbitrary number of hex digits and adds corresponding unicode
+	 * symbol to output
+	 *
+	 * @exception IllegalArgumentException
+	 *                in case of parse error
+	 */
+	private void scanHexDigits() {
+		int hexDigits = 0;
+		while (!isEof()) {
+			int hex = hexValue(true);
+			if (hex == -1) {
+				break;
+			}
+			hexDigits = (hexDigits << 4) | hex;
+			if (hexDigits > 0x10FFFF) {
+				error();
+			}
+			read();
+		}
+		append(codePointToString(hexDigits));
+	}
+
+	/**
+	 * Scans defined number of hex digits and adds corresponding unicode
+	 * symbol to output.
+	 *
+	 * @exception IllegalArgumentException
+	 *                in case of parse error
+	 * @param number
+	 *            expected number of digits
+	 */
+	private void scanHexDigits(int number) {
+		int hex = 0;
+		for (int i = 0; i < number; i++, read()) {
+			hex = (hex << 4) | hexValue(false);
+		}
+		append(codePointToString(hex));
+	}
+
+	/**
+	 * Scans one hex digit and returns value. In case of error if
+	 * {@code optional = true} then {@code -1} is returned else
+	 * {@code IllegalArgumentException} is thrown
+	 *
+	 * @exception IllegalArgumentException
+	 *                in case of parse error
+	 * @param optional
+	 *            is digit optional
+	 * @return value of scanned hex digit or {@code -1} if error is detected
+	 *         and {@code optional = true}
+	 */
+	private int hexValue(boolean optional) {
+		if (current >= '0' && current <= '9') {
+			return current - '0';
+		}
+		else if (current >= 'a' && current <= 'f') {
+			return 10 + (current - 'a');
+		}
+		else if (current >= 'A' && current <= 'F') {
+			return 10 + (current - 'A');
+		}
+		if (optional) {
+			return -1;
+		}
+		throw new IllegalArgumentException();
+	}
+
+	/**
+	 * Reads one symbol, makes it available in {@link #current}
+	 */
+	private void read() {
+		if (index < length) {
+			current = source.charAt(index++);
+		}
+		else {
+			current = -1;
+		}
+	}
+
+	/**
+	 * Appends current symbol to the output
+	 */
+	private void append() {
+		result.append((char) current);
+	}
+
+	/**
+	 * Appends given symbol to output
+	 *
+	 * @param ch
+	 *            symbol to add to the output
+	 */
+	private void append(char ch) {
+		result.append(ch);
+	}
+
+	/**
+	 * Appends given string to output
+	 *
+	 * @param s
+	 *            string to add to the output
+	 */
+	private void append(String s) {
+		result.append(s);
+	}
+
+	/**
+	 * Signal error
+	 */
+	private static void error() {
+		throw new IllegalArgumentException();
+	}
+
+	/**
+	 * Check if unexpected end of input is reached
+	 */
+	private void checkEof() {
+		if (isEof()) {
+			error();
+		}
+	}
+
+	/**
+	 * Returns {@code true} if current symbol is one of the line terminators
+	 *
+	 * @return
+	 */
+	private boolean isEol() {
+		return (current == '\r') || (current == '\n') || (current == '\u2028') || (current == '\u2029');
+	}
+
+	/**
+	 * Returns {@code true} if end of input is reached
+	 *
+	 * @return
+	 */
+	private boolean isEof() {
+		return (current == -1);
+	}
+
+	/**
+	 * Creates unicode symbol from given code-point
+	 *
+	 * @param cp
+	 *            code-point to create unicode symbol from
+	 * @return unicode symbol
+	 */
+	private static String codePointToString(int cp) {
+		if (cp <= 0xFFFF) {
+			return Character.toString((char) cp);
+		}
+		return new String(new int[]{cp}, 0, 1);
+	}
+
+}
diff --git a/bundles/org.eclipse.wst.jsdt.core/src/org/eclipse/wst/jsdt/internal/esprima/DOMASTConverter.java b/bundles/org.eclipse.wst.jsdt.core/src/org/eclipse/wst/jsdt/internal/esprima/DOMASTConverter.java
index 5ce0ebc3e..a29afc91b 100644
--- a/bundles/org.eclipse.wst.jsdt.core/src/org/eclipse/wst/jsdt/internal/esprima/DOMASTConverter.java
+++ b/bundles/org.eclipse.wst.jsdt.core/src/org/eclipse/wst/jsdt/internal/esprima/DOMASTConverter.java
@@ -1,5 +1,5 @@
 /*******************************************************************************

- * Copyright (c) 2015 Red Hat, Inc. 

+ * Copyright (c) 2015-2016 Red Hat, Inc.

  * All rights reserved. This program and the accompanying materials

  * are made available under the terms of the Eclipse Public License v1.0

  * which accompanies this distribution, and is available at

@@ -94,7 +94,7 @@ import jdk.nashorn.api.scripting.ScriptObjectMirror;
 

 /**

  * Converts ESTree AST to DOM AST.

- * 

+ *

  * @author Gorkem Ercan

  *

  */

@@ -106,9 +106,9 @@ public class DOMASTConverter extends EStreeVisitor{
 	// Because switch also hosts all the statements in the case

 	// statement we need to pointer to hold the currently processed switch

 	private Stack<SwitchStatement> processingSwitchStatements = new Stack<SwitchStatement>();

- 	

+

 	/**

-	 * 

+	 *

 	 */

 	public DOMASTConverter(final JavaScriptUnit unit) {

 		if(unit == null ){

@@ -117,13 +117,13 @@ public class DOMASTConverter extends EStreeVisitor{
 		this.root = unit;

 		this.ast = unit.getAST();

 	}

-	

+

 	public JavaScriptUnit convert(ScriptObjectMirror jsobject){

 		this.traverse(jsobject);

 		Assert.isTrue(nodes.empty(),"Some nodes are not processed"); //$NON-NLS-1$

 		return root;

 	}

-	

+

 	/* (non-Javadoc)

 	 * @see org.eclipse.wst.jsdt.internal.esprima.EStreeVisitor#visit(jdk.nashorn.api.scripting.ScriptObjectMirror, org.eclipse.wst.jsdt.internal.esprima.ESTreeNodeTypes)

 	 */

@@ -136,8 +136,8 @@ public class DOMASTConverter extends EStreeVisitor{
 				return convertIdentifier(object);

 			case Literal:

 				return convertLiteral(object);

-			// Declarations	

-			case VariableDeclaration: 

+			// Declarations

+			case VariableDeclaration:

 				return convertVariableDeclaration(object);

 			case VariableDeclarator:

 				return convertVariableDeclarator(object);

@@ -186,16 +186,16 @@ public class DOMASTConverter extends EStreeVisitor{
 				return convertForOfStatement(object);

 			case ClassDeclaration:

 				return convertClassDeclaration(object);

-			case ClassExpression: 

+			case ClassExpression:

 				return convertClassExpression(object);

-			case ClassBody: 

-				// ClassBody is represented with a single TypeDeclaration 

+			case ClassBody:

+				// ClassBody is represented with a single TypeDeclaration

 				// pushing the TypeDeclaration created for ClassDeclaration

 				// twice to stack so that endVisit does not fail.

 				return VisitOptions.CONTINUE;

 			case MethodDefinition:

 				return convertMethodDefinition(object);

-			// Expressions	

+			// Expressions

 			case ThisExpression:

 				return convertThisExpression(object);

 			case ArrayExpression:

@@ -259,11 +259,11 @@ public class DOMASTConverter extends EStreeVisitor{
 				return convertExportDeclaration(object, false, false);

 			case ExportSpecifier:

 				return convertExportSpecifier(object);

-			case ExportAllDeclaration: 

+			case ExportAllDeclaration:

 				return convertExportDeclaration(object, false, true);

 			case ExportDefaultDeclaration:

 				return convertExportDeclaration(object, true, false);

-		

+

 			default:

 				throw new UnimplementedException(nodeType.getTypeString() + " conversion is not implemented"); //$NON-NLS-1$

 		}

@@ -277,7 +277,7 @@ public class DOMASTConverter extends EStreeVisitor{
 	 * @see org.eclipse.wst.jsdt.internal.esprima.EStreeVisitor#endVisit(jdk.nashorn.api.scripting.ScriptObjectMirror, org.eclipse.wst.jsdt.internal.esprima.ESTreeNodeTypes)

 	 */

 	public VisitOptions endVisit(final ScriptObjectMirror object, final ESTreeNodeTypes nodeType, final String key) {

-		

+

 		if(nodeType == ESTreeNodeTypes.Super){//Skip nothing is pushed for these nodes during visit

 			return VisitOptions.CONTINUE;

 		}

@@ -286,13 +286,13 @@ public class DOMASTConverter extends EStreeVisitor{
 		//Set source range for all the nodes.

 		//we can safely move this to visit

 		setRange(object, current);

-		

+

 		ASTNode parent = null;

 		if( nodeType != ESTreeNodeTypes.Program){ //Nodes can be empty only for Program

 			Assert.isTrue(!nodes.empty());

 			parent = nodes.peek();

 		}

-		

+

 		try{

 			//clean-up the switch statement

 			if(current.getNodeType() == SWITCH_STATEMENT){

@@ -315,14 +315,14 @@ public class DOMASTConverter extends EStreeVisitor{
 			}else

 			if(current.getNodeType() == TEMPLATE_ELEMENT){

 				assingTemplateElement(current, parent);

-			}else 

+			}else

 			if(current.getNodeType() == IMPORT_DECLARATION){

 				root.imports().add(current);

 			}else

 			if(current.getNodeType() == EXPORT_DECLARATION){

 				root.exports().add(current);

 			}

-			

+

 		}catch(Exception e){

 			StringBuilder sb = new StringBuilder(e.toString());

 			sb.append(" assigning "); //$NON-NLS-1$

@@ -358,10 +358,10 @@ public class DOMASTConverter extends EStreeVisitor{
 				exportDec.specifiers().add(module);

 				break;

 			default:

-				throw new UnimplementedException("Assigning "+ module + " to "+parent+ " is not handled");	

+				throw new UnimplementedException("Assigning "+ module + " to "+parent+ " is not handled");

 

 		}

-		

+

 	}

 

 	private void assingTemplateElement(ASTNode current, ASTNode parent) {

@@ -369,7 +369,7 @@ public class DOMASTConverter extends EStreeVisitor{
 		tl.elements().add(current);

 	}

 

-	

+

 	private void setRange(final ScriptObjectMirror object, final ASTNode node){

 		Object o = object.getMember("range");

 		if( !ScriptObjectMirror.isUndefined(o) ){

@@ -379,8 +379,8 @@ public class DOMASTConverter extends EStreeVisitor{
 			final int startPosition= x.intValue();

 			final int length =  y.intValue()-x.intValue();

 			node.setSourceRange(startPosition, length);

-			// FunctionDeclarationStatement and FunctionExpression are 

-			// wrappers around FunctionDeclaration and actual FunctionDeclaration 

+			// FunctionDeclarationStatement and FunctionExpression are

+			// wrappers around FunctionDeclaration and actual FunctionDeclaration

 			// is never pushed to stack so we propagate source range here.

 			if(node.getNodeType() == ASTNode.FUNCTION_DECLARATION_STATEMENT ){

 				FunctionDeclarationStatement fds = (FunctionDeclarationStatement)node;

@@ -392,7 +392,7 @@ public class DOMASTConverter extends EStreeVisitor{
 			}

 		}

 	}

-	

+

 	private void assignStatementToParent(final ProgramElement statement, final ASTNode parent, final String key){

 		switch(parent.getNodeType()){

 			case JAVASCRIPT_UNIT:

@@ -504,14 +504,14 @@ public class DOMASTConverter extends EStreeVisitor{
 				}

 				break;

 			default:

-				throw new UnimplementedException("Assigning "+statement + " to "+parent+ " is not handled");	

+				throw new UnimplementedException("Assigning "+statement + " to "+parent+ " is not handled");

 				}

 	}

-	

-	

+

+

 	private void assignVariableDeclarationToParent(VariableDeclaration declaration, ASTNode parent) {

 		switch(parent.getNodeType()){

-			

+

 			case VARIABLE_DECLARATION_STATEMENT:

 				VariableDeclarationStatement vd = (VariableDeclarationStatement)parent;

 				vd.fragments().add(declaration);

@@ -529,13 +529,13 @@ public class DOMASTConverter extends EStreeVisitor{
 				fd.parameters().add(declaration);

 				break;

 			default:

-				throw new UnimplementedException("Assigning "+ declaration + " to "+parent+ " is not handled");	

+				throw new UnimplementedException("Assigning "+ declaration + " to "+parent+ " is not handled");

 		}

-		

+

 	}

-	

+

 	private void assignExpressionToParent(final Expression expression, final ASTNode parent, final String key ){

-		

+

 		switch (parent.getNodeType()) {

 			case EXPRESSION_STATEMENT :

 				((ExpressionStatement)parent).setExpression(expression);

@@ -549,7 +549,7 @@ public class DOMASTConverter extends EStreeVisitor{
 				break;

 			case VARIABLE_DECLARATION_FRAGMENT:

 				if(expression.getNodeType() == SIMPLE_NAME){

-					((VariableDeclarationFragment)parent).setName((SimpleName)expression);	

+					((VariableDeclarationFragment)parent).setName((SimpleName)expression);

 				}else{

 					((VariableDeclarationFragment)parent).setInitializer(expression);

 				}

@@ -564,9 +564,9 @@ public class DOMASTConverter extends EStreeVisitor{
 				ObjectLiteralField olf = (ObjectLiteralField)parent;

 				if("key".equals(key)){

 					olf.setFieldName(expression);

-				}else 

+				}else

 				if("value".equals(key)){

-					olf.setInitializer(expression);					

+					olf.setInitializer(expression);

 				}

 				break;

 			case FUNCTION_EXPRESSION:

@@ -581,7 +581,7 @@ public class DOMASTConverter extends EStreeVisitor{
 				}

 				break;

 			case FUNCTION_DECLARATION_STATEMENT:

-				

+

 				FunctionDeclarationStatement fdecS = (FunctionDeclarationStatement)parent;

 				if("params".equals(key)){

 					SingleVariableDeclaration svd  = ast.newSingleVariableDeclaration();

@@ -825,7 +825,7 @@ public class DOMASTConverter extends EStreeVisitor{
 					sel.setArgument(expression);

 				}

 				break;

-			case IMPORT_DECLARATION: 

+			case IMPORT_DECLARATION:

 				ImportDeclaration idec = (ImportDeclaration) parent;

 				idec.setSource((StringLiteral)expression);

 				break;

@@ -853,7 +853,7 @@ public class DOMASTConverter extends EStreeVisitor{
 		TryStatement ts = (TryStatement) parent;

 		ts.catchClauses().add(current);

 	}

-	

+

 	private VisitOptions convertLiteral(final ScriptObjectMirror object) {

 		Object value = object.getMember("value");

 		String raw = (String)object.getMember("raw");

@@ -865,7 +865,8 @@ public class DOMASTConverter extends EStreeVisitor{
 			literal = ast.newBooleanLiteral(raw);

 		}else

 		if(value instanceof String){

-			literal = ast.newStringLiteral(raw);

+			literal = ast.newStringLiteral();

+			((StringLiteral) literal).setEscapedValue(raw);

 		}else

 		if(object.hasMember("regex")){

 			literal = ast.newRegularExpressionLiteral(raw);

@@ -873,7 +874,7 @@ public class DOMASTConverter extends EStreeVisitor{
 		if(value == null ){

 			literal = ast.newNullLiteral();

 		}

-		

+

 		if(literal == null ){

 			throw new UnimplementedException("Failed to translate Literal " + value); //$NON-NLS-1$

 		}

@@ -882,12 +883,12 @@ public class DOMASTConverter extends EStreeVisitor{
 		}

 		return VisitOptions.CONTINUE;

 	}

-	 

+

 	private VisitOptions convertProgram(final ScriptObjectMirror object){

-		nodes.push(root);	

+		nodes.push(root);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertVariableDeclaration(final ScriptObjectMirror object) {

 		String kind = (String) object.getMember("kind");

 		VariableKind variableKind = VariableKind.VAR;

@@ -898,7 +899,7 @@ public class DOMASTConverter extends EStreeVisitor{
 		}

 		int parentType = nodes.peek().getNodeType();

 		if(parentType == FOR_STATEMENT ){// For statements use expression

-			final VariableDeclarationExpression e = ast.newVariableDeclarationExpression();	

+			final VariableDeclarationExpression e = ast.newVariableDeclarationExpression();

 			e.setKind(variableKind);

 			nodes.push(e);

 		}else{

@@ -906,28 +907,28 @@ public class DOMASTConverter extends EStreeVisitor{
 			e.setKind(variableKind);

 			nodes.push(e);

 		}

-		return VisitOptions.CONTINUE;	

+		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertVariableDeclarator(final ScriptObjectMirror object) {

 		final VariableDeclarationFragment f  = ast.newVariableDeclarationFragment();

 		nodes.push(f);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertIdentifier(final ScriptObjectMirror object) {

 		final String s = (String)object.getMember("name");

 		SimpleName name = ast.newSimpleName(s);

 		nodes.push(name);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertExpressionStatement(final ScriptObjectMirror object) {

 		final ExpressionStatement es = ast.newExpressionStatement();

 		nodes.push(es);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertAssignmentExpression(final ScriptObjectMirror object) {

 		final Assignment a = ast.newAssignment();

 		final String op = (String) object.getMember("operator");

@@ -935,13 +936,13 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(a);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertThisExpression(final ScriptObjectMirror object) {

 		final ThisExpression t = ast.newThisExpression();

 		nodes.push(t);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertArrayExpression(final ScriptObjectMirror object) {

 		ArrayInitializer ai = ast.newArrayInitializer();

 		nodes.push(ai);

@@ -953,7 +954,7 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(o);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertPropertyExpression(final ScriptObjectMirror object) {

 		ObjectLiteralField of = ast.newObjectLiteralField();

 		String kind = (String) object.getMember("kind");

@@ -967,13 +968,13 @@ public class DOMASTConverter extends EStreeVisitor{
 		}

 		of.setKind(k);

 		nodes.push(of);

-		return VisitOptions.CONTINUE; 

+		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertFunctionExpression(final ScriptObjectMirror object) {

-		// We break from the usual pattern and create a functionDeclaration. 

-		// has a single object type for function expression while DOM AST 

-		// needs two objects for the same result. All further assignments needs 

+		// We break from the usual pattern and create a functionDeclaration.

+		// has a single object type for function expression while DOM AST

+		// needs two objects for the same result. All further assignments needs

 		// to handle this properly

 		FunctionExpression fe = ast.newFunctionExpression();

 		FunctionDeclaration d = ast.newFunctionDeclaration();

@@ -983,16 +984,16 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.add(fe);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertFunctionDeclaration(final ScriptObjectMirror object) {

 		FunctionDeclaration dec = ast.newFunctionDeclaration();

 		Boolean isGenerator = (Boolean)object.getMember("generator");

 		dec.setGenerator(isGenerator.booleanValue());

 		int parentType = nodes.peek().getNodeType();

-		if(parentType == ASTNode.EXPORT_DECLARATION 

+		if(parentType == ASTNode.EXPORT_DECLARATION

 				|| parentType == ASTNode.TYPE_DECLARATION

 				){

-			nodes.push(dec);			

+			nodes.push(dec);

 		}else

 		{

 			nodes.push(ast.newFunctionDeclarationStatement(dec));

@@ -1014,7 +1015,7 @@ public class DOMASTConverter extends EStreeVisitor{
 		}

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertBinaryExpression(final ScriptObjectMirror object) {

 		final String operator = (String) object.getMember("operator");

 		InfixExpression ie = ast.newInfixExpression();

@@ -1032,15 +1033,15 @@ public class DOMASTConverter extends EStreeVisitor{
 			FieldAccess fa = ast.newFieldAccess();

 			nodes.push(fa);

 		}

-		return VisitOptions.CONTINUE; 

+		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertConditionalExpression(final ScriptObjectMirror object) {

 		ConditionalExpression ce = ast.newConditionalExpression();

 		nodes.push(ce);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertCallExpression(final ScriptObjectMirror object) {

 		ScriptObjectMirror callee = (ScriptObjectMirror) object.getMember("callee");

 		String type = (String) callee.getMember("type");

@@ -1053,13 +1054,13 @@ public class DOMASTConverter extends EStreeVisitor{
 		}

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertSequenceExpression(final ScriptObjectMirror object) {

 		ListExpression le = ast.newListExpression();

 		nodes.push(le);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertYieldExpression(final ScriptObjectMirror object) {

 		Boolean isDelegate = (Boolean)object.getMember("delegate");

 		YieldExpression ye = ast.newYieldExpression();

@@ -1067,37 +1068,37 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(ye);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertNewExpression(final ScriptObjectMirror object) {

 		ClassInstanceCreation ci = ast.newClassInstanceCreation();

 		nodes.push(ci);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertArrowFunctionExpression(final ScriptObjectMirror object) {

 		ArrowFunctionExpression af = ast.newArrowFunctionExpression();

 		nodes.push(af);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertBlockStatement(final ScriptObjectMirror object) {

 		Block b = ast.newBlock();

 		nodes.push(b);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertEmptyStatement(final ScriptObjectMirror object) {

 		EmptyStatement es = ast.newEmptyStatement();

 		nodes.push(es);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertDebuggerStatememt(final ScriptObjectMirror object) {

 		DebuggerStatement ds = ast.newDebuggerStatement();

 		nodes.push(ds);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertWithStatement(final ScriptObjectMirror object) {

 		WithStatement ws = ast.newWithStatement();

 		nodes.push(ws);

@@ -1115,13 +1116,13 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(ls);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertBreakStatement(final ScriptObjectMirror object) {

 		BreakStatement bs = ast.newBreakStatement();

 		nodes.push(bs);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertContinueStatement(final ScriptObjectMirror object) {

 		ContinueStatement cs = ast.newContinueStatement();

 		nodes.push(cs);

@@ -1133,14 +1134,14 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(is);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertSwitchStatement(final ScriptObjectMirror object) {

 		SwitchStatement ss = ast.newSwitchStatement();

 		this.processingSwitchStatements.push(ss);

 		nodes.push(ss);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertSwitchCaseStatement(final ScriptObjectMirror object) {

 		SwitchCase sc = ast.newSwitchCase();

 		nodes.push(sc);

@@ -1169,7 +1170,7 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(ts);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertWhileStatement(final ScriptObjectMirror object) {

 		WhileStatement ws = ast.newWhileStatement();

 		nodes.push(ws);

@@ -1199,7 +1200,7 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(fo);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertClassDeclaration(final ScriptObjectMirror object) {

 		AbstractTypeDeclaration td = ast.newTypeDeclaration();

 		TypeDeclarationStatement tds = ast.newTypeDeclarationStatement(td);

@@ -1207,7 +1208,7 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(td);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 

 	/**

 	 * @param object

@@ -1220,7 +1221,7 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(td);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertMethodDefinition(final ScriptObjectMirror object) {

 		FunctionDeclaration fd = ast.newFunctionDeclaration();

 		Boolean isStatic = (Boolean)object.getMember("static");

@@ -1243,28 +1244,28 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(fd);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertRestElement(final ScriptObjectMirror object) {

 		RestElementName ren = ast.newRestElementName();

 		nodes.push(ren);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertArrayPattern(final ScriptObjectMirror object) {

 		ArrayName an = ast.newArrayName();

 		nodes.push(an);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertObjectPattern(final ScriptObjectMirror object) {

 		ObjectName on = ast.newObjectName();

 		nodes.push(on);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertTemplateLiteral(final ScriptObjectMirror object, String key) {

 		if("quasi".equals(key)){

-			// reuse the tempalateLiteral created with parent TaggedTemplateLiteral so 

+			// reuse the tempalateLiteral created with parent TaggedTemplateLiteral so

 			//that it eventually cascades to one TemplateLiteral

 			nodes.push(nodes.peek());

 			return VisitOptions.CONTINUE;

@@ -1273,7 +1274,7 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(literal);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertTemplateElement(final ScriptObjectMirror object) {

 		TemplateElement te = ast.newTemplateElement();

 		ScriptObjectMirror val = (ScriptObjectMirror)object.getMember("value");

@@ -1284,7 +1285,7 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(te);

 		return VisitOptions.SKIP;

 	}

-	

+

 	private VisitOptions convertAssignmentPattern(final ScriptObjectMirror object) {

 		AssignmentName an = ast.newAssignmentName();

 		nodes.push(an);

@@ -1312,8 +1313,8 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(importDecl);

 		return VisitOptions.CONTINUE;

 	}

-	

-	

+

+

 	private VisitOptions convertImportSpecifer(ScriptObjectMirror object, boolean isDefault, boolean isNamespace) {

 		ModuleSpecifier specifier = ast.newModuleSpecifier();

 		specifier.setDefault(isDefault);

@@ -1321,18 +1322,18 @@ public class DOMASTConverter extends EStreeVisitor{
 		nodes.push(specifier);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertExportSpecifier(ScriptObjectMirror object) {

 		ModuleSpecifier specifier = ast.newModuleSpecifier();

 		nodes.push(specifier);

 		return VisitOptions.CONTINUE;

 	}

-	

+

 	private VisitOptions convertExportDeclaration(ScriptObjectMirror object, boolean isDefault, boolean isAll) {

 		ExportDeclaration declaration = ast.newExportDeclaration();

 		declaration.setDefault(isDefault);

 		declaration.setAll(isAll);

 		nodes.push(declaration);

 		return VisitOptions.CONTINUE;

-	}	

+	}

 }

diff --git a/tests/org.eclipse.wst.jsdt.core.tests.model/.settings/org.eclipse.core.resources.prefs b/tests/org.eclipse.wst.jsdt.core.tests.model/.settings/org.eclipse.core.resources.prefs
index a6ed35207..cd187cc56 100644
--- a/tests/org.eclipse.wst.jsdt.core.tests.model/.settings/org.eclipse.core.resources.prefs
+++ b/tests/org.eclipse.wst.jsdt.core.tests.model/.settings/org.eclipse.core.resources.prefs
@@ -1,5 +1,5 @@
-#Thu Jul 14 08:12:53 EDT 2011
 eclipse.preferences.version=1
+encoding//src/org/eclipse/wst/jsdt/core/tests/dom/StringLiteralTest.java=UTF-8
 encoding//src/org/eclipse/wst/jsdt/core/tests/model/JSDTModelTests.java=US-ASCII
 encoding/<project>=ISO-8859-1
 instance/org.eclipse.core.net/org.eclipse.core.net.hasMigrated=true
diff --git a/tests/org.eclipse.wst.jsdt.core.tests.model/src/org/eclipse/wst/jsdt/core/tests/dom/StringLiteralTest.java b/tests/org.eclipse.wst.jsdt.core.tests.model/src/org/eclipse/wst/jsdt/core/tests/dom/StringLiteralTest.java
new file mode 100644
index 000000000..ecafdeaa4
--- /dev/null
+++ b/tests/org.eclipse.wst.jsdt.core.tests.model/src/org/eclipse/wst/jsdt/core/tests/dom/StringLiteralTest.java
@@ -0,0 +1,226 @@
+/*******************************************************************************
+ * Copyright (c) 2016 Eugene Melekhov and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Distribution License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * Contributors:
+ *   Eugene Melekhov - initial implementation
+ *******************************************************************************/
+
+package org.eclipse.wst.jsdt.core.tests.dom;
+
+import static org.junit.Assert.assertEquals;
+
+import org.eclipse.wst.jsdt.core.dom.AST;
+import org.eclipse.wst.jsdt.core.dom.StringLiteral;
+import org.junit.Test;
+
+@SuppressWarnings("nls")
+public class StringLiteralTest {
+
+	// ---------------------- Basic simple cases -------------------------------
+	@Test
+	public void testBasic1() {
+		testFromEscape("", "\"\"");
+	}
+
+	@Test
+	public void testBasic2() {
+		testFromEscape("", "''");
+	}
+
+	@Test
+	public void testBasic3() {
+		testFromEscape("a", "'a'");
+	}
+
+	@Test
+	public void testBasic4() {
+		testFromEscape("\"", "\"\\\"\"");
+	}
+
+	@Test
+	public void testBasic5() {
+		testFromEscape("'", "'\\''");
+	}
+
+	@Test
+	public void testBasic6() {
+		testFromEscape("a", "\"a\"");
+	}
+
+	@Test
+	public void testBasic7() {
+		testFromEscape("\"", "'\"'");
+	}
+
+	// ----------------------\x 2 hex digits -----------------------------------
+	@Test
+	public void testHexEscape1() {
+		testFromEscape("X", "\"\\x58\"");
+	}
+
+	@Test
+	public void testHexEscape2() {
+		testFromEscape("XY", "\"\\x58Y\"");
+	}
+
+	@Test
+	public void testHexEscape3() {
+		testFromEscape("YX", "\"Y\\x58\"");
+	}
+
+	public void testHexEscape4() {
+		testFromEscape("X9", "\"\\x589\"");
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testHexEscape5() {
+		testFromEscape("X", "\"\\x5\"");
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testHexEscape6() {
+		testFromEscape("X", "\"\\x\"");
+	}
+
+	// ------------------- \ u {..} Unicode hex digits ---------------------------
+
+	@Test
+	public void testUnicodeNew1() {
+		testFromEscape("X", "\"\\u{58}\"");
+	}
+
+	@Test
+	public void testUnicodeNew2() {
+		testFromEscape("X", "\"\\u{058}\"");
+	}
+
+	@Test
+	public void testUnicodeNew3() {
+		testFromEscape("X", "\"\\u{0058}\"");
+	}
+
+	@Test
+	public void testUnicodeNew4() {
+		testFromEscape("XX", "\"\\u{0058}X\"");
+	}
+
+	@Test
+	public void testUnicodeNew5() {
+		testFromEscape("XX", "\"X\\u{0058}\"");
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testUnicodeNew6() {
+		testFromEscape("X", "\"\\u{0058\"");
+	}
+
+	// ------------------- \ u... Unicode 4 hex digits ---------------------------
+	@Test
+	public void testUnicode1() {
+		testFromEscape("X", "\"\\u0058\"");
+	}
+
+	@Test
+	public void testUnicode2() {
+		testFromEscape("XX", "\"\\u0058X\"");
+	}
+
+	@Test
+	public void testUnicode3() {
+		testFromEscape("XX", "\"X\\u0058\"");
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testUnicode4() {
+		testFromEscape("X", "\"\\u058\"");
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testUnicode5() {
+		testFromEscape("X", "\"\\u58\"");
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testUnicode6() {
+		testFromEscape("X", "\"\\u8\"");
+	}
+
+	// ------------------- \xxx Octal Digits ------------------------------------
+
+	@Test
+	public void testOctal1() {
+		testFromEscape("X", "\"\\130\"");
+	}
+
+	@Test
+	public void testOctal2() {
+		testFromEscape("YX", "\"Y\\130\"");
+	}
+
+	@Test
+	public void testOctal3() {
+		testFromEscape("XY", "\"\\130Y\"");
+	}
+
+	@Test
+	public void testOctal4() {
+		testFromEscape("\u00FF", "\"\\377\"");
+	}
+
+	@Test
+	public void testOctal5() {
+		testFromEscape("\u001F8", "\"\\378\"");
+	}
+
+	@Test
+	public void testOctal6() {
+		testFromEscape(" 1", "\"\\401\"");
+	}
+
+	@Test
+	public void testOctal7() {
+		testFromEscape("1 1", "\"1\\401\"");
+	}
+
+	@Test
+	public void testOctal8() {
+		testFromEscape("W", "\"\\W\"");
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testOctal9() {
+		testFromEscape("\\", "\"\\\"");
+	}
+
+	// ------------------------------- From Literal ----------------------------
+
+	@Test
+	public void testFromLiteral1() {
+		testFromLiteral("\0\1\2\3\4\5\6\7\b\n\r\t\u000B\f\u2028\u2029",
+				"\"\\0\\1\\2\\3\\4\\5\\6\\7\\b\\n\\r\\t\\v\\f\\u2028\\u2029\"");
+	}
+
+	// ------------------------------- All in one ------------------------------
+	@Test
+	public void testAllInOneFromEscape() {
+		testFromEscape("\0\b\n\r\t\u000B\f\\\"'\u2028\u2029日本", "\"\\0\\b\\n\\r\\t\\v\\f\\\\\\\"'\\u2028\\u2029日本\"");
+	}
+
+	// ----------------------------- Helper function ---------------------------
+	private void testFromEscape(String literalValue, String escapedValue) {
+		StringLiteral sl = AST.newAST(AST.JLS3).newStringLiteral();
+		sl.setEscapedValue(escapedValue);
+		assertEquals(literalValue, sl.getLiteralValue());
+	}
+
+	private void testFromLiteral(String literalValue, String escapedValue) {
+		StringLiteral sl = AST.newAST(AST.JLS3).newStringLiteral();
+		sl.setLiteralValue(literalValue);
+		assertEquals(escapedValue, sl.getEscapedValue());
+	}
+
+}