getDocument().getTokenMarker().
*/
- public final TokenMarker getTokenMarker() {
+ public final TokenMarkerState getTokenMarker() {
return document.getTokenMarker();
}
@@ -1680,7 +1680,7 @@ public class JEditTextArea extends JComponent
int segmentOffset = segment.offset;
int segmentCount = segment.count;
- TokenMarker tokenMarker = doc.getTokenMarker();
+ TokenMarkerState tokenMarker = doc.getTokenMarker();
// If syntax coloring is disabled, do simple translation
if (tokenMarker == null) {
for (int j = 0; j < segmentCount; j++) {
diff --git a/app/src/processing/app/syntax/PdeTextAreaPainter.java b/app/src/processing/app/syntax/PdeTextAreaPainter.java
index 290cdfc41..24232f23e 100644
--- a/app/src/processing/app/syntax/PdeTextAreaPainter.java
+++ b/app/src/processing/app/syntax/PdeTextAreaPainter.java
@@ -113,7 +113,7 @@ public class PdeTextAreaPainter extends TextAreaPainter {
* @param x horizontal position
*/
@Override
- protected void paintLine(Graphics gfx, int line, int x, TokenMarker marker) {
+ protected void paintLine(Graphics gfx, int line, int x, TokenMarkerState marker) {
try {
// TODO This line is causing NPEs randomly ever since I added the
// toggle for Java Mode/Debugger toolbar. [Manindra]
@@ -346,4 +346,4 @@ public class PdeTextAreaPainter extends TextAreaPainter {
public PdeTextArea getPdeTextArea() {
return (PdeTextArea) textArea;
}
-}
\ No newline at end of file
+}
diff --git a/app/src/processing/app/syntax/SyntaxDocument.java b/app/src/processing/app/syntax/SyntaxDocument.java
index 25b50d69a..cead84439 100644
--- a/app/src/processing/app/syntax/SyntaxDocument.java
+++ b/app/src/processing/app/syntax/SyntaxDocument.java
@@ -27,7 +27,7 @@ public class SyntaxDocument extends PlainDocument
* of this document up into tokens. May return null if this
* document is not to be colorized.
*/
- public TokenMarker getTokenMarker()
+ public TokenMarkerState getTokenMarker()
{
return tokenMarker;
}
@@ -40,9 +40,11 @@ public class SyntaxDocument extends PlainDocument
*/
public void setTokenMarker(TokenMarker tm)
{
- tokenMarker = tm;
- if(tm == null)
+ if (tm == null) {
+ tokenMarker = null;
return;
+ }
+ tokenMarker = tm.createStateInstance();
tokenMarker.insertLines(0,getDefaultRootElement()
.getElementCount());
tokenizeLines();
@@ -67,7 +69,7 @@ public class SyntaxDocument extends PlainDocument
*/
public void tokenizeLines(int start, int len)
{
- if(tokenMarker == null || !tokenMarker.supportsMultilineTokens())
+ if(tokenMarker == null || !tokenMarker.marker.supportsMultilineTokens())
return;
Segment lineSegment = new Segment();
@@ -118,7 +120,7 @@ public class SyntaxDocument extends PlainDocument
public void addUndoableEdit(UndoableEdit edit) {}
// protected members
- protected TokenMarker tokenMarker;
+ protected TokenMarkerState tokenMarker;
/**
* We overwrite this method to update the token marker
diff --git a/app/src/processing/app/syntax/TextAreaPainter.java b/app/src/processing/app/syntax/TextAreaPainter.java
index 251e6cb8b..fd7c23da2 100644
--- a/app/src/processing/app/syntax/TextAreaPainter.java
+++ b/app/src/processing/app/syntax/TextAreaPainter.java
@@ -488,7 +488,7 @@ public class TextAreaPainter extends JComponent implements TabExpander {
int lastInvalid = firstLine + (clipRect.y + clipRect.height - 1) / height;
try {
- TokenMarker tokenMarker = textArea.getDocument().getTokenMarker();
+ TokenMarkerState tokenMarker = textArea.getDocument().getTokenMarker();
int x = textArea.getHorizontalOffset();
for (int line = firstInvalid; line <= lastInvalid; line++) {
@@ -611,7 +611,7 @@ public class TextAreaPainter extends JComponent implements TabExpander {
// Font defaultFont = getFont();
// Color defaultColor = getForeground();
protected void paintLine(Graphics gfx, int line, int x,
- TokenMarker tokenMarker) {
+ TokenMarkerState tokenMarker) {
currentLineIndex = line;
int y = textArea.lineToY(line);
@@ -671,7 +671,7 @@ public class TextAreaPainter extends JComponent implements TabExpander {
// int line, Font defaultFont,
// Color defaultColor, int x, int y) {
protected void paintSyntaxLine(Graphics gfx, int line, int x, int y,
- TokenMarker tokenMarker) {
+ TokenMarkerState tokenMarker) {
textArea.getLineText(currentLineIndex, currentLine);
currentLineTokens = tokenMarker.markTokens(currentLine, currentLineIndex);
diff --git a/app/src/processing/app/syntax/TokenMarker.java b/app/src/processing/app/syntax/TokenMarker.java
index 0e414e901..57d76e6b7 100644
--- a/app/src/processing/app/syntax/TokenMarker.java
+++ b/app/src/processing/app/syntax/TokenMarker.java
@@ -15,7 +15,7 @@ import javax.swing.text.Segment;
* A token marker that splits lines of text into tokens. Each token carries
* a length field and an indentification tag that can be mapped to a color
* for painting that token.- * + *
* For performance reasons, the linked list of tokens is reused after each
* line is tokenized. Therefore, the return value of markTokens
* should only be used for immediate painting. Notably, it cannot be
@@ -23,321 +23,68 @@ import javax.swing.text.Segment;
*
* @author Slava Pestov
*/
-public abstract class TokenMarker
-{
- abstract public void addColoring(String keyword, String coloring);
+public abstract class TokenMarker {
- /**
- * A wrapper for the lower-level markTokensImpl method
- * that is called to split a line up into tokens.
- * @param line The line
- * @param lineIndex The line number
- */
- public Token markTokens(Segment line, int lineIndex)
- {
- if(lineIndex >= length)
- {
- throw new IllegalArgumentException("Tokenizing invalid line: "
- + lineIndex);
- }
+ public interface TokenListener {
+ void addToken(int length, byte id);
+ }
- lastToken = null;
+ // Only needed during markTokensImpl() call so addToken() can be forwarded
+ private TokenListener tokenListener = null;
- LineInfo info = lineInfo[lineIndex];
- LineInfo prev;
- if(lineIndex == 0)
- prev = null;
- else
- prev = lineInfo[lineIndex - 1];
+ public final void setTokenListener(TokenListener listener) {
+ this.tokenListener = listener;
+ }
- byte oldToken = info.token;
- byte token = markTokensImpl(prev == null ?
- Token.NULL : prev.token,line,lineIndex);
+ public final TokenMarkerState createStateInstance() {
+ return new TokenMarkerState(this);
+ }
- info.token = token;
+ /**
+ * Creates a new TokenMarker. This DOES NOT create
+ * a lineInfo array; an initial call to insertLines()
+ * does that.
+ */
+ protected TokenMarker() { }
- /*
- * This is a foul hack. It stops nextLineRequested
- * from being cleared if the same line is marked twice.
- *
- * Why is this necessary? It's all JEditTextArea's fault.
- * When something is inserted into the text, firing a
- * document event, the insertUpdate() method shifts the
- * caret (if necessary) by the amount inserted.
- *
- * All caret movement is handled by the select() method,
- * which eventually pipes the new position to scrollTo()
- * and calls repaint().
- *
- * Note that at this point in time, the new line hasn't
- * yet been painted; the caret is moved first.
- *
- * scrollTo() calls offsetToX(), which tokenizes the line
- * unless it is being called on the last line painted
- * (in which case it uses the text area's painter cached
- * token list). What scrollTo() does next is irrelevant.
- *
- * After scrollTo() has done it's job, repaint() is
- * called, and eventually we end up in paintLine(), whose
- * job is to paint the changed line. It, too, calls
- * markTokens().
- *
- * The problem was that if the line started a multiline
- * token, the first markTokens() (done in offsetToX())
- * would set nextLineRequested (because the line end
- * token had changed) but the second would clear it
- * (because the line was the same that time) and therefore
- * paintLine() would never know that it needed to repaint
- * subsequent lines.
- *
- * This bug took me ages to track down, that's why I wrote
- * all the relevant info down so that others wouldn't
- * duplicate it.
- */
- if(!(lastLine == lineIndex && nextLineRequested))
- nextLineRequested = (oldToken != token);
+ abstract public void addColoring(String keyword, String coloring);
- lastLine = lineIndex;
+ /**
+ * An abstract method that splits a line up into tokens. It
+ * should parse the line, and call addToken() to
+ * add syntax tokens to the token list. Then, it should return
+ * the initial token type for the next line.
+ *
+ * For example if the current line contains the start of a
+ * multiline comment that doesn't end on that line, this method
+ * should return the comment token type so that it continues on
+ * the next line.
+ *
+ * @param token The initial token type for this line
+ * @param line The line to be tokenized
+ * @param lineIndex The index of the line in the document,
+ * starting at 0
+ * @return The initial token type for the next line
+ */
+ protected abstract byte markTokensImpl(byte token, Segment line,
+ int lineIndex);
- addToken(0,Token.END);
+ protected final void addToken(int length, byte id) {
+ if (tokenListener != null) {
+ tokenListener.addToken(length, id);
+ }
+ }
- return firstToken;
- }
-
- /**
- * An abstract method that splits a line up into tokens. It
- * should parse the line, and call addToken() to
- * add syntax tokens to the token list. Then, it should return
- * the initial token type for the next line.
- *
- * For example if the current line contains the start of a
- * multiline comment that doesn't end on that line, this method
- * should return the comment token type so that it continues on
- * the next line.
- *
- * @param token The initial token type for this line
- * @param line The line to be tokenized
- * @param lineIndex The index of the line in the document,
- * starting at 0
- * @return The initial token type for the next line
- */
- protected abstract byte markTokensImpl(byte token, Segment line,
- int lineIndex);
-
- /**
- * Returns if the token marker supports tokens that span multiple
- * lines. If this is true, the object using this token marker is
- * required to pass all lines in the document to the
- * markTokens() method (in turn).
- *
- * The default implementation returns true; it should be overridden
- * to return false on simpler token markers for increased speed.
- */
- public boolean supportsMultilineTokens()
- {
- return true;
- }
-
- /**
- * Informs the token marker that lines have been inserted into
- * the document. This inserts a gap in the lineInfo
- * array.
- * @param index The first line number
- * @param lines The number of lines
- */
- public void insertLines(int index, int lines)
- {
- if(lines <= 0)
- return;
- length += lines;
- ensureCapacity(length);
- int len = index + lines;
- System.arraycopy(lineInfo,index,lineInfo,len,
- lineInfo.length - len);
-
- for(int i = index + lines - 1; i >= index; i--)
- {
- lineInfo[i] = new LineInfo();
- }
- }
-
- /**
- * Informs the token marker that line have been deleted from
- * the document. This removes the lines in question from the
- * lineInfo array.
- * @param index The first line number
- * @param lines The number of lines
- */
- public void deleteLines(int index, int lines)
- {
- if (lines <= 0)
- return;
- int len = index + lines;
- length -= lines;
- System.arraycopy(lineInfo,len,lineInfo,
- index,lineInfo.length - len);
- }
-
- /**
- * Returns the number of lines in this token marker.
- */
- public int getLineCount()
- {
- return length;
- }
-
- /**
- * Returns true if the next line should be repainted. This
- * will return true after a line has been tokenized that starts
- * a multiline token that continues onto the next line.
- */
- public boolean isNextLineRequested()
- {
- return nextLineRequested;
- }
-
- // protected members
-
- /**
- * The first token in the list. This should be used as the return
- * value from markTokens().
- */
- protected Token firstToken;
-
- /**
- * The last token in the list. New tokens are added here.
- * This should be set to null before a new line is to be tokenized.
- */
- protected Token lastToken;
-
- /**
- * An array for storing information about lines. It is enlarged and
- * shrunk automatically by the insertLines() and
- * deleteLines() methods.
- */
- protected LineInfo[] lineInfo;
-
- /**
- * The number of lines in the model being tokenized. This can be
- * less than the length of the lineInfo array.
- */
- protected int length;
-
- /**
- * The last tokenized line.
- */
- protected int lastLine;
-
- /**
- * True if the next line should be painted.
- */
- protected boolean nextLineRequested;
-
- /**
- * Creates a new TokenMarker. This DOES NOT create
- * a lineInfo array; an initial call to insertLines()
- * does that.
- */
- protected TokenMarker()
- {
- lastLine = -1;
- }
-
- /**
- * Ensures that the lineInfo array can contain the
- * specified index. This enlarges it if necessary. No action is
- * taken if the array is large enough already.
- *
- * It should be unnecessary to call this under normal
- * circumstances; insertLine() should take care of
- * enlarging the line info array automatically.
- *
- * @param index The array index
- */
- protected void ensureCapacity(int index)
- {
- if(lineInfo == null)
- lineInfo = new LineInfo[index + 1];
- else if(lineInfo.length <= index)
- {
- LineInfo[] lineInfoN = new LineInfo[(index + 1) * 2];
- System.arraycopy(lineInfo,0,lineInfoN,0,
- lineInfo.length);
- lineInfo = lineInfoN;
- }
- }
-
- /**
- * Adds a token to the token list.
- * @param length The length of the token
- * @param id The id of the token
- */
- protected void addToken(int length, byte id)
- {
- if(id >= Token.INTERNAL_FIRST && id <= Token.INTERNAL_LAST)
- throw new InternalError("Invalid id: " + id);
-
- if(length == 0 && id != Token.END)
- return;
-
- if(firstToken == null)
- {
- firstToken = new Token(length,id);
- lastToken = firstToken;
- }
- else if(lastToken == null)
- {
- lastToken = firstToken;
- firstToken.length = length;
- firstToken.id = id;
- }
- else if(lastToken.next == null)
- {
- lastToken.next = new Token(length,id);
- lastToken = lastToken.next;
- }
- else
- {
- lastToken = lastToken.next;
- lastToken.length = length;
- lastToken.id = id;
- }
- }
-
- /**
- * Inner class for storing information about tokenized lines.
- */
- private static class LineInfo
- {
- /**
- * Creates a new LineInfo object with token = Token.NULL
- * and obj = null.
- */
- public LineInfo()
- {
- }
-
- /**
- * Creates a new LineInfo object with the specified
- * parameters.
- */
-// public LineInfo(byte token, Object obj)
-// {
-// this.token = token;
-// this.obj = obj;
-// }
-
- /**
- * The id of the last token of the line.
- */
- public byte token;
-
- /**
- * This is for use by the token marker implementations
- * themselves. It can be used to store anything that
- * is an object and that needs to exist on a per-line
- * basis.
- */
-// public Object obj;
- }
+ /**
+ * Returns if the token marker supports tokens that span multiple
+ * lines. If this is true, the object using this token marker is
+ * required to pass all lines in the document to the
+ * markTokens() method (in turn).
+ *
+ * The default implementation returns true; it should be overridden
+ * to return false on simpler token markers for increased speed.
+ */
+ public boolean supportsMultilineTokens() {
+ return true;
+ }
}
diff --git a/app/src/processing/app/syntax/TokenMarkerState.java b/app/src/processing/app/syntax/TokenMarkerState.java
new file mode 100644
index 000000000..8b20b2603
--- /dev/null
+++ b/app/src/processing/app/syntax/TokenMarkerState.java
@@ -0,0 +1,266 @@
+package processing.app.syntax;
+
+import javax.swing.text.Segment;
+
+/**
+ * This class serves only as a workaround to preserve API and should be removed
+ * in the next major version. Base TokenMarker which serves as superclass for
+ * token markes for various modes is stateful, but single instance was shared
+ * between all tabs and Editors. This caused inherent bugs by leaking state
+ * between contexts.
+ *
+ * TokenMarker subclasses now serve two purposes: they keep keyword list and
+ * they override markTokensImpl to provide the marking logic.
+ *
+ * Since each tab and Editor should have it's own token marker state, I extracted
+ * most of the fields and associated metods working with them from TokenMarker
+ * into this class, and allowed TokenMarker to create instances of this class
+ * when requested.
+ *
+ * The way marking logic is handled is unfortunate, since markTokensImpl is
+ * expected to call addToken() of TokenMarker superclass instead of – for
+ * example - returning a List of tokens. I worked around this by plugging in
+ * TokenMarkerState instance as listener before markTokensImpl is called.
+ * This behavior is safe since TokenMarker is always operated from Event
+ * Dispatch Thread and no multithreading is involved.
+ *
+ * This allows having only single instance of TokenMarker in a way it was
+ * intended before while keeping state separate for each tab.
+ *
+ * In the next major version TokenMarker shound be redesigned with following
+ * requirements in mind:
+ * - Single instance of keyword list and other common data, initialized by Mode
+ * - Each tab should have its own instance of TokenMarker containing its state
+ * - Support multiple flavors for different doc types
+ * - Other modes should provide logic in a way which is compatible with
+ * multiple states (pure function? Function object?).
+ * Currently state, logic and keywords list are tied together into one
+ * TokenMarker instance, which leads to need for this workaround.
+ */
+public class TokenMarkerState {
+
+ protected TokenMarker marker;
+
+ protected TokenMarkerState(TokenMarker marker) {
+ this.marker = marker;
+ }
+
+ /**
+ * The first token in the list. This should be used as the return
+ * value from markTokens().
+ */
+ protected Token firstToken;
+
+ /**
+ * The last token in the list. New tokens are added here.
+ * This should be set to null before a new line is to be tokenized.
+ */
+ protected Token lastToken;
+
+ /**
+ * An array for storing information about lines. It is enlarged and
+ * shrunk automatically by the insertLines() and
+ * deleteLines() methods.
+ */
+ protected byte[] lineInfo;
+
+ /**
+ * The number of lines in the model being tokenized. This can be
+ * less than the length of the lineInfo array.
+ */
+ protected int length;
+
+ /**
+ * The last tokenized line.
+ */
+ protected int lastLine = -1;
+
+ /**
+ * True if the next line should be painted.
+ */
+ protected boolean nextLineRequested;
+
+ /**
+ * A wrapper for the lower-level markTokensImpl method
+ * that is called to split a line up into tokens.
+ *
+ * @param line The line
+ * @param lineIndex The line number
+ */
+ public Token markTokens(Segment line, int lineIndex) {
+ if (lineIndex >= length) {
+ throw new IllegalArgumentException("Tokenizing invalid line: "
+ + lineIndex);
+ }
+
+ marker.setTokenListener(this::addToken);
+
+ lastToken = null;
+
+ byte prev = (lineIndex == 0) ? Token.NULL : lineInfo[lineIndex - 1];
+
+ byte oldToken = lineInfo[lineIndex];
+ byte token = marker.markTokensImpl(prev, line, lineIndex);
+
+ marker.setTokenListener(null);
+
+ lineInfo[lineIndex] = token;
+
+ /*
+ * This is a foul hack. It stops nextLineRequested
+ * from being cleared if the same line is marked twice.
+ *
+ * Why is this necessary? It's all JEditTextArea's fault.
+ * When something is inserted into the text, firing a
+ * document event, the insertUpdate() method shifts the
+ * caret (if necessary) by the amount inserted.
+ *
+ * All caret movement is handled by the select() method,
+ * which eventually pipes the new position to scrollTo()
+ * and calls repaint().
+ *
+ * Note that at this point in time, the new line hasn't
+ * yet been painted; the caret is moved first.
+ *
+ * scrollTo() calls offsetToX(), which tokenizes the line
+ * unless it is being called on the last line painted
+ * (in which case it uses the text area's painter cached
+ * token list). What scrollTo() does next is irrelevant.
+ *
+ * After scrollTo() has done it's job, repaint() is
+ * called, and eventually we end up in paintLine(), whose
+ * job is to paint the changed line. It, too, calls
+ * markTokens().
+ *
+ * The problem was that if the line started a multiline
+ * token, the first markTokens() (done in offsetToX())
+ * would set nextLineRequested (because the line end
+ * token had changed) but the second would clear it
+ * (because the line was the same that time) and therefore
+ * paintLine() would never know that it needed to repaint
+ * subsequent lines.
+ *
+ * This bug took me ages to track down, that's why I wrote
+ * all the relevant info down so that others wouldn't
+ * duplicate it.
+ */
+ if (!(lastLine == lineIndex && nextLineRequested)) {
+ nextLineRequested = (oldToken != token);
+ }
+
+ lastLine = lineIndex;
+
+ addToken(0, Token.END);
+
+ return firstToken;
+ }
+
+ /**
+ * Informs the token marker that lines have been inserted into
+ * the document. This inserts a gap in the lineInfo
+ * array.
+ *
+ * @param index The first line number
+ * @param lines The number of lines
+ */
+ public void insertLines(int index, int lines) {
+ if (lines <= 0)
+ return;
+ length += lines;
+ ensureCapacity(length);
+ int len = index + lines;
+ System.arraycopy(lineInfo, index, lineInfo, len,
+ lineInfo.length - len);
+
+ for (int i = index + lines - 1; i >= index; i--) {
+ lineInfo[i] = Token.NULL;
+ }
+ }
+
+ /**
+ * Informs the token marker that line have been deleted from
+ * the document. This removes the lines in question from the
+ * lineInfo array.
+ *
+ * @param index The first line number
+ * @param lines The number of lines
+ */
+ public void deleteLines(int index, int lines) {
+ if (lines <= 0)
+ return;
+ int len = index + lines;
+ length -= lines;
+ System.arraycopy(lineInfo, len, lineInfo,
+ index, lineInfo.length - len);
+ }
+
+ /**
+ * Returns the number of lines in this token marker.
+ */
+ public int getLineCount() {
+ return length;
+ }
+
+ /**
+ * Returns true if the next line should be repainted. This
+ * will return true after a line has been tokenized that starts
+ * a multiline token that continues onto the next line.
+ */
+ public boolean isNextLineRequested() {
+ return nextLineRequested;
+ }
+
+ /**
+ * Ensures that the lineInfo array can contain the
+ * specified index. This enlarges it if necessary. No action is
+ * taken if the array is large enough already.
+ *
+ * It should be unnecessary to call this under normal
+ * circumstances; insertLine() should take care of
+ * enlarging the line info array automatically.
+ *
+ * @param index The array index
+ */
+ protected void ensureCapacity(int index) {
+ if (lineInfo == null) {
+ lineInfo = new byte[index + 1];
+ } else if (lineInfo.length <= index) {
+ byte[] lineInfoN = new byte[(index + 1) * 2];
+ System.arraycopy(lineInfo, 0, lineInfoN, 0,
+ lineInfo.length);
+ lineInfo = lineInfoN;
+ }
+ }
+
+ /**
+ * Adds a token to the token list.
+ *
+ * @param length The length of the token
+ * @param id The id of the token
+ */
+ protected void addToken(int length, byte id) {
+ if (id >= Token.INTERNAL_FIRST && id <= Token.INTERNAL_LAST) {
+ throw new InternalError("Invalid id: " + id);
+ }
+
+ if (length == 0 && id != Token.END) {
+ return;
+ }
+
+ if (firstToken == null) {
+ firstToken = new Token(length, id);
+ lastToken = firstToken;
+ } else if (lastToken == null) {
+ lastToken = firstToken;
+ firstToken.length = length;
+ firstToken.id = id;
+ } else if (lastToken.next == null) {
+ lastToken.next = new Token(length, id);
+ lastToken = lastToken.next;
+ } else {
+ lastToken = lastToken.next;
+ lastToken.length = length;
+ lastToken.id = id;
+ }
+ }
+}