ILookaheadLineReader: use CharSequence, not String

9 files changed, 338 insertions, 66 deletions

diff --git a/org.eclipse.photran.core.vpg.tests/src/org/eclipse/photran/internal/tests/lang/FortranLineScannerBenchmark.java b/org.eclipse.photran.core.vpg.tests/src/org/eclipse/photran/internal/tests/lang/FortranLineScannerBenchmark.java
new file mode 100644
index 00000000..dfed357a
--- /dev/null
+++ b/org.eclipse.photran.core.vpg.tests/src/org/eclipse/photran/internal/tests/lang/FortranLineScannerBenchmark.java
@@ -0,0 +1,123 @@
+/*******************************************************************************
+ * Copyright (c) 2014 Auburn University and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *    Jeff Overbey (Auburn University) - Initial API and implementation
+ *******************************************************************************/
+package org.eclipse.photran.internal.tests.lang;
+
+import java.io.IOException;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.eclipse.photran.internal.core.lang.linescanner.CharArraySequence;
+import org.eclipse.photran.internal.core.lang.linescanner.CharSeqLookaheadLineReader;
+import org.eclipse.photran.internal.core.lang.linescanner.FortranLineScanner;
+
+/**
+ * Benchmark for the {@link FortranLineScanner}.
+ * 
+ * @author Jeff Overbey
+ */
+public class FortranLineScannerBenchmark extends TestCase
+{
+    private static final int N = 20;
+    private static final int LINES = 500000;
+
+    private static final char[][] lines = new char[][] {
+        "! This is an example of a line that is a comment\n".toCharArray(),
+        "                      ! This is also a comment  \n".toCharArray(),
+        "        357  print *, 3  ! This is a statement  \n".toCharArray(),
+        "    include \"windows.h\"                         \n".toCharArray()
+    };
+
+    public void testBenchmark() throws IOException
+    {
+        int maxLineLen = maxLen(lines);
+        Random rand = new Random();
+        char[] string = new char[LINES*maxLineLen];
+        int idx = 0;
+        for (int i = 0; i < LINES; i++) {
+            char[] line = lines[rand.nextInt(lines.length)];
+            System.arraycopy(line, 0, string, idx, line.length);
+            idx += line.length;
+        }
+
+        long start = System.currentTimeMillis();
+        for (int i = 0; i < N; i++) {
+            CharSeqLookaheadLineReader reader = new CharSeqLookaheadLineReader(string);
+            FortranLineScanner scanner = new FortranLineScanner(false, true);
+            int len = 0;
+            for (int j = 0; j < LINES; j++) {
+                len += scanner.scan(reader).length();
+            }
+            assertEquals(string.length, len);
+        }
+        long end = System.currentTimeMillis();
+        long elapsedTimeMS = (end-start);
+        System.out.printf("%s: %d ms/%d KLOC\n", getClass().getSimpleName(), elapsedTimeMS, N*LINES);
+
+        start = System.currentTimeMillis();
+        CharSequence charSeq = new CharArraySequence(string);
+        for (int i = 0; i < N; i++) {
+            idx = 0;
+            for (int j = 0; j < LINES; j++) {
+                //while (charSeq.charAt(idx++) != '\n') {
+                //}
+                while (charSeq.charAt(idx) != '\n') {
+                    switch (charSeq.charAt(idx)) {
+                        case '!':
+                            idx++;
+                        case '#':
+                            idx++;
+                        default:
+                            idx++;
+                    }
+                }
+                idx++;
+            }
+            assertEquals(string.length, idx);
+        }
+        end = System.currentTimeMillis();
+        elapsedTimeMS = (end-start);
+        System.out.printf("    Line reading via CharSequence: %d ms/%d KLOC\n", elapsedTimeMS, N*LINES);
+
+        start = System.currentTimeMillis();
+        for (int i = 0; i < N; i++) {
+            idx = 0;
+            for (int j = 0; j < LINES; j++) {
+//                while (chars[idx++] != '\n') {
+//                }
+                while (string[idx] != '\n') {
+                    switch (string[idx]) {
+                        case '!':
+                            idx++;
+                        case '#':
+                            idx++;
+                        default:
+                            idx++;
+                    }
+                }
+                idx++;
+            }
+            assertEquals(string.length, idx);
+        }
+        end = System.currentTimeMillis();
+        elapsedTimeMS = (end-start);
+        System.out.printf("    Line reading via char[]: %d ms/%d KLOC\n", elapsedTimeMS, N*LINES);
+    }
+
+    private int maxLen(char[][] lines)
+    {
+        int max = 0;
+        for (char[] line : lines) {
+            max = Math.max(max, line.length);
+        }
+        return max;
+    }
+}
diff --git a/org.eclipse.photran.core.vpg.tests/src/org/eclipse/photran/internal/tests/lang/FortranLineScannerTests.java b/org.eclipse.photran.core.vpg.tests/src/org/eclipse/photran/internal/tests/lang/FortranLineScannerTests.java
index e41803a8..bf4ab292 100644
--- a/org.eclipse.photran.core.vpg.tests/src/org/eclipse/photran/internal/tests/lang/FortranLineScannerTests.java
+++ b/org.eclipse.photran.core.vpg.tests/src/org/eclipse/photran/internal/tests/lang/FortranLineScannerTests.java
@@ -10,12 +10,18 @@
  *******************************************************************************/
 package org.eclipse.photran.internal.tests.lang;
 
-import static org.eclipse.photran.internal.core.lang.linescanner.FortranLineType.*;
+import static org.eclipse.photran.internal.core.lang.linescanner.FortranLineType.COMMENT;
+import static org.eclipse.photran.internal.core.lang.linescanner.FortranLineType.INCLUDE_LINE;
+import static org.eclipse.photran.internal.core.lang.linescanner.FortranLineType.PREPROCESSOR_DIRECTIVE;
+import static org.eclipse.photran.internal.core.lang.linescanner.FortranLineType.STATEMENT;
+
+import java.io.IOException;
+
 import junit.framework.TestCase;
 
+import org.eclipse.photran.internal.core.lang.linescanner.CharSeqLookaheadLineReader;
 import org.eclipse.photran.internal.core.lang.linescanner.FortranLineScanner;
 import org.eclipse.photran.internal.core.lang.linescanner.FortranLineType;
-import org.eclipse.photran.internal.core.lang.linescanner.StringLookaheadLineReader;
 
 /**
  * Unit tests for {@link FortranLineScanner}.
@@ -28,7 +34,7 @@ public class FortranLineScannerTests extends TestCase
 
     private boolean preprocessed;
 
-    private void freeFormTests()
+    private void freeFormTests() throws IOException
     {
         //check(COMMENT, "");
         check(COMMENT, " ");
@@ -59,7 +65,7 @@ public class FortranLineScannerTests extends TestCase
         check(STATEMENT, "print *, \"Hello\"");
     }
 
-    public void testFreeFormPreprocessed()
+    public void testFreeFormPreprocessed() throws IOException
     {
         fixedForm = false;
         preprocessed = true;
@@ -69,9 +75,11 @@ public class FortranLineScannerTests extends TestCase
         check(PREPROCESSOR_DIRECTIVE, "#error");
         check(PREPROCESSOR_DIRECTIVE, " #  include  \"mpif.h\"");
         check(PREPROCESSOR_DIRECTIVE, "  ??= include  \"mpif.h\"");
+        check(PREPROCESSOR_DIRECTIVE, " #  include  \"C:\\Windows\\Windows.h\"");
+        check(PREPROCESSOR_DIRECTIVE, " #  include  \"C:\\Windows\\Windows.h\"\n!OK", " #  include  \"C:\\Windows\\Windows.h\"\n");
     }
 
-    public void testFreeFormUnpreprocessed()
+    public void testFreeFormUnpreprocessed() throws IOException
     {
         fixedForm = false;
         preprocessed = false;
@@ -83,7 +91,7 @@ public class FortranLineScannerTests extends TestCase
         check(STATEMENT, "  ??= include  \"mpif.h\"");
     }
 
-    private void fixedFormTests()
+    private void fixedFormTests() throws IOException
     {
         //check(COMMENT, "");
         check(COMMENT, " ");
@@ -142,7 +150,7 @@ public class FortranLineScannerTests extends TestCase
         check(STATEMENT, "print *, \"Hello\"");
     }
 
-    public void testFixedFormPreprocessed()
+    public void testFixedFormPreprocessed() throws IOException
     {
         fixedForm = true;
         preprocessed = true;
@@ -154,7 +162,7 @@ public class FortranLineScannerTests extends TestCase
         check(PREPROCESSOR_DIRECTIVE, "  ??= include  \"mpif.h\"");
     }
 
-    public void testFixedFormUnpreprocessed()
+    public void testFixedFormUnpreprocessed() throws IOException
     {
         fixedForm = true;
         preprocessed = false;
@@ -166,7 +174,7 @@ public class FortranLineScannerTests extends TestCase
         check(STATEMENT, "  ??= include  \"mpif.h\"");
     }
 
-    public void testPreprocessorContinuations()
+    public void testPreprocessorContinuations() throws IOException
     {
         fixedForm = false;
         preprocessed = true;
@@ -179,7 +187,7 @@ public class FortranLineScannerTests extends TestCase
             "print \"Hel\\\n");
     }
 
-    public void testFreeFormContinuations()
+    public void testFreeFormContinuations() throws IOException
     {
         fixedForm = false;
         preprocessed = false;
@@ -214,7 +222,7 @@ public class FortranLineScannerTests extends TestCase
         check(STATEMENT, "print *, & \n"); // No continuation line available
     }
 
-    public void testFixedFormContinuations()
+    public void testFixedFormContinuations() throws IOException
     {
         fixedForm = true;
         preprocessed = false;
@@ -236,21 +244,16 @@ public class FortranLineScannerTests extends TestCase
             "      if (a < b)\n"); // No continuation after comment
     }
 
-    private void check(FortranLineType expectedType, String string)
+    private void check(FortranLineType expectedType, String string) throws IOException
     {
         check(expectedType, string, string);
     }
 
     private void check(FortranLineType expectedType, String string, String expectedStmt)
     {
-        FortranLineScanner scanner = createScanner();
-        String actualStmt = scanner.scan(new StringLookaheadLineReader(string));
+        FortranLineScanner scanner = new FortranLineScanner(fixedForm, preprocessed);
+        CharSequence actualStmt = scanner.scan(new CharSeqLookaheadLineReader(string));
         assertEquals(expectedType, scanner.getLineType());
         assertEquals(expectedStmt, actualStmt);
     }
-
-    private FortranLineScanner createScanner()
-    {
-        return new FortranLineScanner(fixedForm, preprocessed);
-    }
 }
diff --git a/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/CharArraySequence.java b/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/CharArraySequence.java
new file mode 100644
index 00000000..b0ba7c47
--- /dev/null
+++ b/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/CharArraySequence.java
@@ -0,0 +1,52 @@
+/*******************************************************************************
+ * Copyright (c) Auburn University and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *    Jeff Overbey (Auburn University) - Initial API and implementation
+ *******************************************************************************/
+package org.eclipse.photran.internal.core.lang.linescanner;
+
+/**
+ * A {@link CharSequence} that wraps a char[] array.
+ * 
+ * @author Jeff Overbey
+ */
+public final class CharArraySequence implements CharSequence
+{
+    private final char[] chars;
+
+    private final int start;
+
+    private final int end;
+
+    public CharArraySequence(char[] chars)
+    {
+        this(chars, 0, chars.length);
+    }
+
+    public CharArraySequence(char[] chars, int start, int end)
+    {
+        this.chars = chars;
+        this.start = start;
+        this.end = end;
+    }
+
+    public char charAt(int i)
+    {
+        return chars[start + i];
+    }
+
+    public int length()
+    {
+        return end - start;
+    }
+
+    public CharSequence subSequence(int start, int end)
+    {
+        return new CharArraySequence(chars, start, end);
+    }
+}
\ No newline at end of file
diff --git a/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/StringLookaheadLineReader.java b/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/CharSeqLookaheadLineReader.java
index ce89a588..b1b54415 100644
--- a/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/StringLookaheadLineReader.java
+++ b/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/CharSeqLookaheadLineReader.java
@@ -11,26 +11,33 @@
 package org.eclipse.photran.internal.core.lang.linescanner;
 
 /**
- * An implementation of {@link ILookaheadLineReader} which reads lines from a String.
+ * An implementation of {@link ILookaheadLineReader} that reads lines from a String, char[] or other CharSequence.
  * 
  * @author Jeff Overbey
  */
-public final class StringLookaheadLineReader implements ILookaheadLineReader<Error>
+public final class CharSeqLookaheadLineReader implements ILookaheadLineReader<Error>
 {
-    private final String string;
+    private final CharSequence string;
 
     private int start;
 
     private int index;
 
-    public StringLookaheadLineReader(String string)
+    public CharSeqLookaheadLineReader(CharSequence string)
     {
         this.string = string;
         this.start = 0;
         this.index = 0;
     }
 
-    public String readNextLine()
+    public CharSeqLookaheadLineReader(char[] chars)
+    {
+        this.string = new CharArraySequence(chars);
+        this.start = 0;
+        this.index = 0;
+    }
+
+    public CharSequence readNextLine()
     {
         if (index >= string.length())
         {
@@ -38,17 +45,25 @@ public final class StringLookaheadLineReader implements ILookaheadLineReader<Err
         }
         else
         {
-            int nextLF = string.indexOf('\n', index) + 1;
+            int nextLF = indexOf('\n', string, index) + 1;
             if (nextLF <= 0) nextLF = string.length();
-            String result = string.substring(index, nextLF);
+            CharSequence result = string.subSequence(index, nextLF);
             index = nextLF;
             return result;
         }
     }
 
-    public String advanceAndRestart(int numChars)
+    private int indexOf(char c, CharSequence seq, int start)
+    {
+        for (int i = start, len = seq.length(); i < len; i++)
+            if (seq.charAt(i) == c)
+                return i;
+        return -1;
+    }
+
+    public CharSequence advanceAndRestart(int numChars)
     {
-        String result = string.substring(start, numChars);
+        CharSequence result = string.subSequence(start, start + numChars);
         this.start += numChars;
         this.index = start;
         return result;
diff --git a/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/FortranLineScanner.java b/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/FortranLineScanner.java
index bf3fa8e8..26fa6c73 100644
--- a/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/FortranLineScanner.java
+++ b/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/FortranLineScanner.java
@@ -27,6 +27,7 @@ public final class FortranLineScanner
         .compile("[ \t]*[Ii][ \t]*[Nn][ \t]*[Cc][ \t]*[Ll][ \t]*[Uu][ \t]*[Dd][ \t]*[Ee][ \t]*[\"']([^\r\n\"]*)[\"'][ \t]*(![^\r\n]*)?[\r\n]*"); //$NON-NLS-1$
     private static final Pattern FREE_FORM_INCLUDE_LINE = Pattern
         .compile("[ \t]*[Ii][Nn][Cc][Ll][Uu][Dd][Ee][ \t]+[\"']([^\r\n\"]*)[\"'][ \t]*(![^\r\n]*)?[\r\n]*"); //$NON-NLS-1$
+    private static final Pattern TRIGRAPH_DIRECTIVE = Pattern.compile("[ \t]*\\?\\?=.*"); //$NON-NLS-1$
 
     @SuppressWarnings("unused")
     private static final int INCLUDE_LINE_CAPTURING_GROUP_OF_FILENAME = 1;
@@ -53,12 +54,12 @@ public final class FortranLineScanner
      * statement. This is necessary to detect (the absence of) continuation lines in fixed form
      * source code.
      */
-    public final <X extends Throwable> String scan(ILookaheadLineReader<X> reader) throws X
+    public final <X extends Throwable> CharSequence scan(ILookaheadLineReader<X> reader) throws X
     {
         this.lineLength = -1;
         this.lineType = null;
 
-        String line = reader.readNextLine();
+        CharSequence line = reader.readNextLine();
         if (line == null)
         {
             this.lineLength = 0;
@@ -74,7 +75,7 @@ public final class FortranLineScanner
         }
     }
 
-    private <X extends Throwable> int checkForContinuationLines(String line, ILookaheadLineReader<X> reader) throws X
+    private <X extends Throwable> int checkForContinuationLines(CharSequence line, ILookaheadLineReader<X> reader) throws X
     {
         if (this.lineType == FortranLineType.PREPROCESSOR_DIRECTIVE)
             return checkForPreprocessorDirectiveContinuation(line, reader);
@@ -84,7 +85,7 @@ public final class FortranLineScanner
             return 0;
     }
 
-    private <X extends Throwable> int checkForPreprocessorDirectiveContinuation(String line, ILookaheadLineReader<X> reader) throws X
+    private <X extends Throwable> int checkForPreprocessorDirectiveContinuation(CharSequence line, ILookaheadLineReader<X> reader) throws X
     {
         int numberOfAdditionalCharacters = 0;
 
@@ -97,7 +98,7 @@ public final class FortranLineScanner
         return numberOfAdditionalCharacters;
     }
 
-    private <X extends Throwable> int checkForFreeFormContinuation(String line, ILookaheadLineReader<X> reader) throws X
+    private <X extends Throwable> int checkForFreeFormContinuation(CharSequence line, ILookaheadLineReader<X> reader) throws X
     {
         int numberOfAdditionalCharacters = 0;
 
@@ -120,7 +121,7 @@ public final class FortranLineScanner
         {
             // Skip comment lines
             int lengthOfComments = 0;
-            String line = reader.readNextLine();
+            CharSequence line = reader.readNextLine();
             while (line != null
                 && classify(line) == FortranLineType.COMMENT
                 && findFirstNonWhitespaceCharacter(line) != '\0')
@@ -132,7 +133,7 @@ public final class FortranLineScanner
             // Then check for continuation line
             if (line != null
                 && line.length() >= 7
-                && line.startsWith("     ") && line.charAt(5) != ' ' && line.charAt(5) != '0') //$NON-NLS-1$
+                && startsWith(line, "     ") && line.charAt(5) != ' ' && line.charAt(5) != '0') //$NON-NLS-1$
             {
                 numberOfAdditionalCharacters += lengthOfComments + line.length();
                 moreContinuationLinesPossible = true;
@@ -146,9 +147,9 @@ public final class FortranLineScanner
         return numberOfAdditionalCharacters;
     }
 
-    private FortranLineType classify(String line)
+    private FortranLineType classify(CharSequence line)
     {
-        final char firstChar = line.isEmpty() ? '\0' : line.charAt(0);
+        final char firstChar = line.length() == 0 ? '\0' : line.charAt(0);
         final char firstNonSpaceChar = findFirstNonWhitespaceCharacter(line);
 
         if (firstNonSpaceChar == '\0') // Line contains only blanks
@@ -157,21 +158,20 @@ public final class FortranLineScanner
         }
         else if (fixedForm && (firstChar == 'C' || firstChar == 'c' || firstChar == '*'))
         {
-            if (line.startsWith("      C") || line.startsWith("      c") || line.startsWith("      *")) //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+            if (startsWith(line, "      C") || startsWith(line, "      c") || startsWith(line, "      *")) //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
                 return FortranLineType.STATEMENT; // Actually a continuation of a previous line
             else
                 return isCommentDirective(line) ? FortranLineType.COMMENT_DIRECTIVE : FortranLineType.COMMENT;
         }
         else if (firstNonSpaceChar == '!')
         {
-            if (fixedForm && line.startsWith("      !")) //$NON-NLS-1$
+            if (fixedForm && startsWith(line, "      !")) //$NON-NLS-1$
                 return FortranLineType.STATEMENT; // Actually a continuation of a previous line
             else
                 return isCommentDirective(line) ? FortranLineType.COMMENT_DIRECTIVE : FortranLineType.COMMENT;
         }
         else if (preprocessed
-            && (firstNonSpaceChar == '#' || (firstNonSpaceChar == '?' && line.trim().startsWith(
-                "??=")))) //$NON-NLS-1$
+            && (firstNonSpaceChar == '#' || (firstNonSpaceChar == '?' && TRIGRAPH_DIRECTIVE.matcher(line).matches())))
         {
             return FortranLineType.PREPROCESSOR_DIRECTIVE;
         }
@@ -184,7 +184,17 @@ public final class FortranLineScanner
         }
     }
 
-    private boolean isIncludeLine(String line)
+    private boolean startsWith(CharSequence line, CharSequence prefix)
+    {
+        if (line.length() < prefix.length()) return false;
+
+        for (int i = 0, len = prefix.length(); i < len; i++)
+            if (line.charAt(i) != prefix.charAt(i))
+                return false;
+        return true;
+    }
+
+    private boolean isIncludeLine(CharSequence line)
     {
         if (fixedForm)
             return FIXED_FORM_INCLUDE_LINE.matcher(line).matches();
@@ -192,7 +202,7 @@ public final class FortranLineScanner
             return FREE_FORM_INCLUDE_LINE.matcher(line).matches();
     }
 
-    private static final char findFirstNonWhitespaceCharacter(String string)
+    private static final char findFirstNonWhitespaceCharacter(CharSequence string)
     {
         if (string != null)
         {
@@ -205,12 +215,12 @@ public final class FortranLineScanner
         return '\0';
     }
 
-    private boolean isCommentDirective(String line)
+    private boolean isCommentDirective(CharSequence line)
     {
         return COMMENT_DIRECTIVE.matcher(line).find();
     }
 
-    private static final char findLastNonWhitespaceCharacter(String string)
+    private static final char findLastNonWhitespaceCharacter(CharSequence string)
     {
         if (string != null)
         {
@@ -223,7 +233,7 @@ public final class FortranLineScanner
         return '\0';
     }
 
-    private boolean isContinued(String line)
+    private boolean isContinued(CharSequence line)
     {
         if (line == null) return false;
 
@@ -266,7 +276,7 @@ public final class FortranLineScanner
     public final int getLineLength()
     {
         if (lineLength < 0)
-            throw new IllegalStateException("Must call #readNextLine before calling #getLineLength"); //$NON-NLS-1$
+            throw new IllegalStateException("Must call #scan before calling #getLineLength"); //$NON-NLS-1$
         else
             return lineLength;
     }
@@ -274,7 +284,7 @@ public final class FortranLineScanner
     public final FortranLineType getLineType()
     {
         if (lineType == null)
-            throw new IllegalStateException("Must call #readNextLine before calling #getLineType"); //$NON-NLS-1$
+            throw new IllegalStateException("Must call #scan before calling #getLineType"); //$NON-NLS-1$
         else
             return lineType;
     }
diff --git a/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/FortranLineType.java b/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/FortranLineType.java
index 82436449..089e70a3 100644
--- a/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/FortranLineType.java
+++ b/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/FortranLineType.java
@@ -24,6 +24,13 @@ package org.eclipse.photran.internal.core.lang.linescanner;
  */
 public enum FortranLineType {
     /**
+     * A blank line.
+     * <p>
+     * A blank line consists of zero or more whitespace characters.
+     */
+    BLANK,
+
+    /**
      * A Fortran comment.
      * <p>
      * Comments usually start with !, although in fixed form they may be indicated by a C, c, or *
diff --git a/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/ILookaheadLineReader.java b/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/ILookaheadLineReader.java
index 3d18d3b3..58a669b1 100644
--- a/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/ILookaheadLineReader.java
+++ b/org.eclipse.photran.core/src/org/eclipse/photran/internal/core/lang/linescanner/ILookaheadLineReader.java
@@ -16,7 +16,7 @@ package org.eclipse.photran.internal.core.lang.linescanner;
  * @author Jeff Overbey
  * 
  * @see LookaheadLineReader
- * @see StringLookaheadLineReader
+ * @see CharSeqLookaheadLineReader
  * @see FortranLineScanner#scan(ILookaheadLineReader)
  */
 public interface ILookaheadLineReader<X extends Throwable>
@@ -26,9 +26,9 @@ public interface ILookaheadLineReader<X extends Throwable>
      * @return or <code>null</code>
      * @throws X
      */
-    String readNextLine() throws X;
+    CharSequence readNextLine() throws X;
 
-    String advanceAndRestart(int numChars);
+    CharSequence advanceAndRestart(int numChars);
 
     void close() throws X;
 }
diff --git a/org.eclipse.photran.ui/src/org/eclipse/photran/internal/ui/editor/FortranStmtPartitionScanner.java b/org.eclipse.photran.ui/src/org/eclipse/photran/internal/ui/editor/FortranStmtPartitionScanner.java
index 3e07a1c3..055d598f 100644
--- a/org.eclipse.photran.ui/src/org/eclipse/photran/internal/ui/editor/FortranStmtPartitionScanner.java
+++ b/org.eclipse.photran.ui/src/org/eclipse/photran/internal/ui/editor/FortranStmtPartitionScanner.java
@@ -130,11 +130,23 @@ public class FortranStmtPartitionScanner implements IPartitionTokenScanner
         return editor == null ? true : editor.isCPreprocessed();
     }
 
+    @Override
+    public int getTokenOffset()
+    {
+        return tokenOffset;
+    }
+
+    @Override
+    public int getTokenLength()
+    {
+        return tokenLength;
+    }
+
     private final class DocumentLookaheadLineReader implements ILookaheadLineReader<BadLocationException>
     {
         private int offset = startOffset;
 
-        public String readNextLine() throws BadLocationException
+        public CharSequence readNextLine() throws BadLocationException
         {
             if (offset >= document.getLength()) return null;
 
@@ -142,12 +154,12 @@ public class FortranStmtPartitionScanner implements IPartitionTokenScanner
             IRegion line = document.getLineInformation(lineNumber);
             String delimiter = document.getLineDelimiter(lineNumber);
             if (delimiter == null) delimiter = ""; //$NON-NLS-1$
-            String result = document.get(line.getOffset(), line.getLength()) + delimiter;
+            CharSequence result = new IDocumentCharSequence(document, line.getOffset(), line.getOffset() + line.getLength() + delimiter.length());
             offset += result.length();
             return result;
         }
 
-        public String advanceAndRestart(int numChars)
+        public CharSequence advanceAndRestart(int numChars)
         {
             offset = startOffset + numChars; // Pointless, actually, since this is never reused
             return null; // Return value (passed back via FortranLineScanner#scan) not used above
@@ -157,16 +169,4 @@ public class FortranStmtPartitionScanner implements IPartitionTokenScanner
         {
         }
     }
-
-    @Override
-    public int getTokenOffset()
-    {
-        return tokenOffset;
-    }
-
-    @Override
-    public int getTokenLength()
-    {
-        return tokenLength;
-    }
 }
diff --git a/org.eclipse.photran.ui/src/org/eclipse/photran/internal/ui/editor/IDocumentCharSequence.java b/org.eclipse.photran.ui/src/org/eclipse/photran/internal/ui/editor/IDocumentCharSequence.java
new file mode 100644
index 00000000..933f851d
--- /dev/null
+++ b/org.eclipse.photran.ui/src/org/eclipse/photran/internal/ui/editor/IDocumentCharSequence.java
@@ -0,0 +1,62 @@
+/*******************************************************************************
+ * Copyright (c) Auburn University and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *    Jeff Overbey (Auburn University) - Initial API and implementation
+ *******************************************************************************/
+package org.eclipse.photran.internal.ui.editor;
+
+import org.eclipse.jface.text.BadLocationException;
+import org.eclipse.jface.text.IDocument;
+
+/**
+ * A {@link CharSequence} that provides an interface to an {@link IDocument}.
+ * 
+ * @author Jeff Overbey
+ */
+public final class IDocumentCharSequence implements CharSequence
+{
+    private final IDocument document;
+
+    private final int start;
+
+    private final int end;
+
+    public IDocumentCharSequence(IDocument document)
+    {
+        this(document, 0, document.getLength());
+    }
+
+    public IDocumentCharSequence(IDocument document, int start, int end)
+    {
+        this.document = document;
+        this.start = start;
+        this.end = end;
+    }
+
+    public char charAt(int i)
+    {
+        try
+        {
+            return document.getChar(start + i);
+        }
+        catch (BadLocationException e)
+        {
+            throw new Error(e);
+        }
+    }
+
+    public int length()
+    {
+        return end - start;
+    }
+
+    public CharSequence subSequence(int start, int end)
+    {
+        return new IDocumentCharSequence(document, start, end);
+    }
+}
\ No newline at end of file