Bug 534211 - unify Detail Code parsing

new scanner, parser and linker for detail code including tests
* the scanner is tokenizing the text
* the parser creates an AST
* the linker associates Room objects to the AST nodes in a context
dependent way
* unit tests for scanner, parser and linker
* a detail code translator is provided with a slightly changed interface
to the translation provider

Change-Id: Id31abfc0913353aeeb86d0070b1cec8a76d7a82e

12 files changed, 31 insertions, 28 deletions

diff --git a/plugins/org.eclipse.etrice.dctools.fsm/META-INF/MANIFEST.MF b/plugins/org.eclipse.etrice.dctools.fsm/META-INF/MANIFEST.MF
index 88e3030b2..b364b6786 100644
--- a/plugins/org.eclipse.etrice.dctools.fsm/META-INF/MANIFEST.MF
+++ b/plugins/org.eclipse.etrice.dctools.fsm/META-INF/MANIFEST.MF
@@ -16,5 +16,6 @@ Require-Bundle: org.eclipse.jface.text,
 Export-Package: org.eclipse.etrice.dctools.fsm.ast,
  org.eclipse.etrice.dctools.fsm.ast.internal;x-friends:="org.eclipse.etrice.dctools.tests",
  org.eclipse.etrice.dctools.fsm.ast.nodes,
+ org.eclipse.etrice.dctools.fsm.ast.tokens,
  org.eclipse.etrice.dctools.fsm.ast.util
 
diff --git a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/DCParser.xtend b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/DCParser.xtend
index d2c554a4d..4eaa04710 100644
--- a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/DCParser.xtend
+++ b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/DCParser.xtend
@@ -13,10 +13,6 @@
 package org.eclipse.etrice.dctools.fsm.ast
 
 import java.util.ArrayList
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCBracketToken
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCBracketToken.BracketKind
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCTextToken
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCToken.Kind
 import org.eclipse.etrice.dctools.fsm.ast.nodes.DCAstArrayAccessNode
 import org.eclipse.etrice.dctools.fsm.ast.nodes.DCAstBracketNode
 import org.eclipse.etrice.dctools.fsm.ast.nodes.DCAstBracketNode.BracketType
@@ -29,6 +25,10 @@ import org.eclipse.etrice.dctools.fsm.ast.nodes.DCAstOperationCallNode
 import org.eclipse.etrice.dctools.fsm.ast.nodes.DCAstOtherNode
 import org.eclipse.etrice.dctools.fsm.ast.nodes.DCAstPeriodNode
 import org.eclipse.etrice.dctools.fsm.ast.nodes.DCAstWhitespaceNode
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCBracketToken
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCBracketToken.BracketKind
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCTextToken
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCToken.Kind
 
 /**
  * 
diff --git a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/DCScanner.xtend b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/DCScanner.xtend
index d228afe9a..1ae4813f7 100644
--- a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/DCScanner.xtend
+++ b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/DCScanner.xtend
@@ -13,24 +13,24 @@
 package org.eclipse.etrice.dctools.fsm.ast
 
 import com.google.common.collect.Iterables
+import org.eclipse.etrice.dctools.fsm.ast.internal.DCBracketRule
+import org.eclipse.etrice.dctools.fsm.ast.internal.DCIdentifierRule
 import org.eclipse.etrice.dctools.fsm.ast.internal.DCKeywordRule
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCToken
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCToken.Kind
+import org.eclipse.etrice.dctools.fsm.ast.internal.DCSpecialCharRule
 import org.eclipse.etrice.dctools.fsm.ast.internal.DCWhitespaceDetector
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCTextToken
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCToken
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCToken.Kind
+import org.eclipse.jface.text.Document
 import org.eclipse.jface.text.rules.EndOfLineRule
 import org.eclipse.jface.text.rules.IRule
+import org.eclipse.jface.text.rules.IToken
 import org.eclipse.jface.text.rules.MultiLineRule
 import org.eclipse.jface.text.rules.NumberRule
 import org.eclipse.jface.text.rules.RuleBasedScanner
 import org.eclipse.jface.text.rules.SingleLineRule
-import org.eclipse.jface.text.rules.WhitespaceRule
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCIdentifierRule
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCBracketRule
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCSpecialCharRule
-import org.eclipse.jface.text.rules.IToken
 import org.eclipse.jface.text.rules.Token
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCTextToken
-import org.eclipse.jface.text.Document
+import org.eclipse.jface.text.rules.WhitespaceRule
 
 class DCScanner {
 	
diff --git a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCBracketRule.xtend b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCBracketRule.xtend
index 8faf2654f..6035f0d5a 100644
--- a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCBracketRule.xtend
+++ b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCBracketRule.xtend
@@ -15,6 +15,7 @@ package org.eclipse.etrice.dctools.fsm.ast.internal
 import org.eclipse.jface.text.rules.IRule
 import org.eclipse.jface.text.rules.ICharacterScanner
 import org.eclipse.jface.text.rules.Token
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCBracketToken
 
 class DCBracketRule implements IRule {
 	
diff --git a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCIdentifierRule.xtend b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCIdentifierRule.xtend
index 936af2ddb..a81db6e80 100644
--- a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCIdentifierRule.xtend
+++ b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCIdentifierRule.xtend
@@ -12,7 +12,8 @@
 
 package org.eclipse.etrice.dctools.fsm.ast.internal
 
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCToken.Kind
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCToken
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCToken.Kind
 import org.eclipse.jface.text.rules.ICharacterScanner
 import org.eclipse.jface.text.rules.IRule
 import org.eclipse.jface.text.rules.Token
diff --git a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCKeywordRule.xtend b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCKeywordRule.xtend
index aba6941dc..9dbef0a20 100644
--- a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCKeywordRule.xtend
+++ b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCKeywordRule.xtend
@@ -12,16 +12,16 @@
 
 package org.eclipse.etrice.dctools.fsm.ast.internal
 
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCToken.Kind
-import org.eclipse.jface.text.rules.WordRule
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCKeywords
 import org.eclipse.etrice.dctools.fsm.ast.DCLanguage
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCToken
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCToken.Kind
+import org.eclipse.jface.text.rules.WordRule
 
 class DCKeywordRule extends WordRule {
 	
 	val keywordToken = new DCToken(Kind.KEYWORD)
 	
-	new(org.eclipse.etrice.dctools.fsm.ast.DCLanguage language) {
+	new(DCLanguage language) {
 		super(new DCWordDetector(language.isCFamily))
 		
 		switch language {
@@ -34,7 +34,7 @@ class DCKeywordRule extends WordRule {
 		}
 	}
 	
-	static def isCFamily(org.eclipse.etrice.dctools.fsm.ast.DCLanguage language) {
+	static def isCFamily(DCLanguage language) {
 		language==DCLanguage.C_LANGUAGE || language==DCLanguage.CPP_LANGUAGE
 	}
 }
diff --git a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCSpecialCharRule.xtend b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCSpecialCharRule.xtend
index 4a5dd1418..09fa428c9 100644
--- a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCSpecialCharRule.xtend
+++ b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCSpecialCharRule.xtend
@@ -12,10 +12,11 @@
 
 package org.eclipse.etrice.dctools.fsm.ast.internal
 
-import org.eclipse.jface.text.rules.IRule
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCToken
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCToken.Kind
 import org.eclipse.jface.text.rules.ICharacterScanner
+import org.eclipse.jface.text.rules.IRule
 import org.eclipse.jface.text.rules.Token
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCToken.Kind
 
 class DCSpecialCharRule implements IRule {
 	
diff --git a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/nodes/DCAstIdentifierNode.xtend b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/nodes/DCAstIdentifierNode.xtend
index 24bfe2177..84c932de9 100644
--- a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/nodes/DCAstIdentifierNode.xtend
+++ b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/nodes/DCAstIdentifierNode.xtend
@@ -12,9 +12,8 @@
 
 package org.eclipse.etrice.dctools.fsm.ast.nodes
 
-import org.eclipse.etrice.dctools.fsm.ast.nodes.DCAstNode
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCTextToken
 import org.eclipse.xtend.lib.annotations.Accessors
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCTextToken
 
 @Accessors
 class DCAstIdentifierNode extends DCAstNode {
diff --git a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/nodes/DCAstOtherNode.xtend b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/nodes/DCAstOtherNode.xtend
index 5f6e330fb..1ebd19329 100644
--- a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/nodes/DCAstOtherNode.xtend
+++ b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/nodes/DCAstOtherNode.xtend
@@ -12,7 +12,7 @@
 
 package org.eclipse.etrice.dctools.fsm.ast.nodes
 
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCTextToken
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCTextToken
 import org.eclipse.xtend.lib.annotations.Accessors
 
 @Accessors
diff --git a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCBracketToken.xtend b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/tokens/DCBracketToken.xtend
index 03341f329..073388b5f 100644
--- a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCBracketToken.xtend
+++ b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/tokens/DCBracketToken.xtend
@@ -10,9 +10,9 @@
  * 
  *******************************************************************************/
 
-package org.eclipse.etrice.dctools.fsm.ast.internal
+package org.eclipse.etrice.dctools.fsm.ast.tokens
 
-import org.eclipse.etrice.dctools.fsm.ast.internal.DCBracketToken.BracketKind
+import org.eclipse.etrice.dctools.fsm.ast.tokens.DCBracketToken.BracketKind
 import org.eclipse.xtend.lib.annotations.Data
 
 @Data
diff --git a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCTextToken.xtend b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/tokens/DCTextToken.xtend
index 02f71becc..0a119c6ca 100644
--- a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCTextToken.xtend
+++ b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/tokens/DCTextToken.xtend
@@ -10,7 +10,7 @@
  * 
  *******************************************************************************/
 
-package org.eclipse.etrice.dctools.fsm.ast.internal
+package org.eclipse.etrice.dctools.fsm.ast.tokens
 
 import org.eclipse.xtend.lib.annotations.Data
 
diff --git a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCToken.xtend b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/tokens/DCToken.xtend
index d54abfd4b..c6dc4ec80 100644
--- a/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/internal/DCToken.xtend
+++ b/plugins/org.eclipse.etrice.dctools.fsm/src/org/eclipse/etrice/dctools/fsm/ast/tokens/DCToken.xtend
@@ -10,7 +10,7 @@
  * 
  *******************************************************************************/
 
-package org.eclipse.etrice.dctools.fsm.ast.internal
+package org.eclipse.etrice.dctools.fsm.ast.tokens
 
 import org.eclipse.jface.text.rules.IToken
 import org.eclipse.xtend.lib.annotations.Data