From 42f48475b738864145c92bf7c88c43d50dd2171a Mon Sep 17 00:00:00 2001 From: George Bateman Date: Thu, 22 Jun 2017 12:37:43 +0100 Subject: [PATCH 1/8] Autoformatter: fix curly quotes. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also it now doesn't destroy the rest of the file if a string is left unterminated. Will never fix "content” since curly quotes are valid string content. --- java/src/processing/mode/java/AutoFormat.java | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/java/src/processing/mode/java/AutoFormat.java b/java/src/processing/mode/java/AutoFormat.java index ec61540dc..659ae4f5c 100644 --- a/java/src/processing/mode/java/AutoFormat.java +++ b/java/src/processing/mode/java/AutoFormat.java @@ -639,25 +639,48 @@ public class AutoFormat implements Formatter { break; case '"': + case '“': + case '”': case '\'': + case '‘': + case '’': inStatementFlag = true; - buf.append(c); + char realQuote = c; + if (c == '“' || c == '”') realQuote = '"'; + if (c == '‘' || c == '’') realQuote = '\''; + buf.append(realQuote); + + char otherQuote = c; + if (c == '“') otherQuote = '”'; + if (c == '”') otherQuote = '“'; + if (c == '‘') otherQuote = '’'; + if (c == '’') otherQuote = '‘'; + char cc = nextChar(); - while (!EOF && cc != c) { + // In a proper string, all the quotes tested are c. In a curly-quoted + // string, there are three possible end quotes: c, its reverse, and + // the correct straight quote. + while (!EOF && cc != otherQuote && cc != realQuote && cc != c) { buf.append(cc); if (cc == '\\') { buf.append(cc = nextChar()); } - if (cc == '\n') { - writeIndentedLine(); - startFlag = true; - } + + // Syntax error: unterminated string. Leave \n in nextChar, so it + // feeds back into the loop. + if (peek() == '\n') break; cc = nextChar(); } - buf.append(cc); - if (readForNewLine()) { - // push a newline into the stream - chars[pos--] = '\n'; + if (cc == otherQuote || cc == realQuote || cc == c) { + buf.append(realQuote); + if (readForNewLine()) { + // push a newline into the stream + chars[pos--] = '\n'; + } + } else { + // We've had a syntax error if the string wasn't terminated by EOL/ + // EOF, just abandon this statement. + inStatementFlag = false; } break; From 96a042f873d8634fd22ec803f0212525c5df6f21 Mon Sep 17 00:00:00 2001 From: George Bateman Date: Thu, 22 Jun 2017 12:41:42 +0100 Subject: [PATCH 2/8] Add handler for Antlr "unexpected char" There's a special message for curly quotes. --- build/shared/lib/languages/PDE.properties | 1 + java/src/processing/mode/java/JavaBuild.java | 28 +++++++++++++++++-- .../mode/java/preproc/PdePreprocessor.java | 10 ++++--- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/build/shared/lib/languages/PDE.properties b/build/shared/lib/languages/PDE.properties index 1a335b83d..d68177d7d 100644 --- a/build/shared/lib/languages/PDE.properties +++ b/build/shared/lib/languages/PDE.properties @@ -360,6 +360,7 @@ editor.status.missing.right_paren = Missing right parenthesis ")" editor.status.missing.left_curly_bracket = Missing left curly bracket "{" editor.status.missing.right_curly_bracket = Missing right curly bracket "}" editor.status.missing.add = Consider adding "%s" +editor.status.bad_curly_quote = Curly quotes like %s don't work. Use straight quotes. Ctrl-T to autocorrect. editor.status.reserved_words = "color" and "int" are reserved words & cannot be used as variable names editor.status.undefined_method = The function "%s(%s)" does not exist editor.status.undefined_constructor = The constructor "%s(%s)" does not exist diff --git a/java/src/processing/mode/java/JavaBuild.java b/java/src/processing/mode/java/JavaBuild.java index 88f6fc3be..e9f84170d 100644 --- a/java/src/processing/mode/java/JavaBuild.java +++ b/java/src/processing/mode/java/JavaBuild.java @@ -38,6 +38,7 @@ import org.apache.tools.ant.Project; import org.apache.tools.ant.ProjectHelper; import processing.app.Base; +import processing.app.Language; import processing.app.Library; import processing.app.Messages; import processing.app.Mode; @@ -341,9 +342,30 @@ public class JavaBuild { // System.err.println("and then she tells me " + tsre.toString()); // TODO not tested since removing ORO matcher.. ^ could be a problem - String mess = "^line (\\d+):(\\d+):\\s"; + String locationRegex = "^line (\\d+):(\\d+):\\s"; + String message = tsre.getMessage(); + String[] m; - String[] matches = PApplet.match(tsre.toString(), mess); + if (null != (m = PApplet.match(tsre.toString(), + "unexpected char: (.*)"))) { + char c = 0; + if (m[1].startsWith("0x")) { // Hex + c = (char) PApplet.unhex(m[1].substring(2)); + } else if (m[1].length() == 3) { // Quoted + c = m[1].charAt(1); + } else if (m[1].length() == 1) { // Alone + c = m[1].charAt(0); + } + if (c == '\u201C' || c == '\u201D' || // “” + c == '\u2018' || c == '\u2019') { // ‘’ + message = Language.interpolate("editor.status.bad_curly_quote", c); + } else if (c != 0) { + message = "Not expecting symbol " + m[1] + + ", which is " + Character.getName(c) + "."; + } + } + + String[] matches = PApplet.match(tsre.toString(), locationRegex); if (matches != null) { int errorLine = Integer.parseInt(matches[1]) - 1; int errorColumn = Integer.parseInt(matches[2]); @@ -358,7 +380,7 @@ public class JavaBuild { } errorLine -= sketch.getCode(errorFile).getPreprocOffset(); - throw new SketchException(tsre.getMessage(), + throw new SketchException(message, errorFile, errorLine, errorColumn); } else { diff --git a/java/src/processing/mode/java/preproc/PdePreprocessor.java b/java/src/processing/mode/java/preproc/PdePreprocessor.java index 5a68752d1..ea8723520 100644 --- a/java/src/processing/mode/java/preproc/PdePreprocessor.java +++ b/java/src/processing/mode/java/preproc/PdePreprocessor.java @@ -924,9 +924,11 @@ public class PdePreprocessor { checkForUnterminatedMultilineComment(program); - if (Preferences.getBoolean("preproc.substitute_unicode")) { - program = substituteUnicode(program); - } + // Removing all the Unicode characters makes detecting and reporting their + // preprocessor errors quite hard. +// if (Preferences.getBoolean("preproc.substitute_unicode")) { +// program = substituteUnicode(program); +// } // For 0215, adding } as a legitimate prefix to the import (along with // newline and semicolon) for cases where a tab ends with } and an import @@ -1478,4 +1480,4 @@ public class PdePreprocessor { } return sb.toString(); } -} \ No newline at end of file +} From cd7430fc108893daba0d44314bc7516605ae3f0b Mon Sep 17 00:00:00 2001 From: George Bateman Date: Sat, 24 Jun 2017 14:35:18 +0100 Subject: [PATCH 3/8] Handle curly quotes in error checker Also now prioritizes error messages on a single line for display to the user, since ECJ doesn't always get that right, reported mismatched argument lists when there's a syntax error and so on. --- app/src/processing/app/ui/Editor.java | 5 +- build/shared/lib/languages/PDE.properties | 1 + java/src/processing/mode/java/JavaEditor.java | 20 +++++ .../java/pdex/ErrorMessageSimplifier.java | 64 +++++++++++---- .../mode/java/pdex/JavaProblem.java | 77 +++++++++++++++++-- java/src/processing/mode/java/pdex/PDEX.java | 5 +- .../mode/java/pdex/SourceUtils.java | 4 +- 7 files changed, 153 insertions(+), 23 deletions(-) diff --git a/app/src/processing/app/ui/Editor.java b/app/src/processing/app/ui/Editor.java index 4cdc9817a..5a4969e5a 100644 --- a/app/src/processing/app/ui/Editor.java +++ b/app/src/processing/app/ui/Editor.java @@ -3106,9 +3106,10 @@ public abstract class Editor extends JFrame implements RunnerListener { /** - * @return the Problem for the first error or warning on 'line' + * @return the Problem for the most relevant error or warning on 'line', + * defaulting to the first. */ - Problem findProblem(int line) { + protected Problem findProblem(int line) { int currentTab = getSketch().getCurrentCodeIndex(); return problems.stream() .filter(p -> p.getTabIndex() == currentTab) diff --git a/build/shared/lib/languages/PDE.properties b/build/shared/lib/languages/PDE.properties index d68177d7d..7acab2a19 100644 --- a/build/shared/lib/languages/PDE.properties +++ b/build/shared/lib/languages/PDE.properties @@ -370,6 +370,7 @@ editor.status.undef_global_var = The global variable "%s" does not exist editor.status.undef_class = The class "%s" does not exist editor.status.undef_var = The variable "%s" does not exist editor.status.undef_name = The name "%s" cannot be recognized +editor.status.unterm_string_curly = String literal is not closed by a straight double quote. Curly quotes like %s won't help. editor.status.type_mismatch = Type mismatch, "%s" does not match with "%s" editor.status.unused_variable = The value of the local variable "%s" is not used editor.status.uninitialized_variable = The local variable "%s" may not have been initialized diff --git a/java/src/processing/mode/java/JavaEditor.java b/java/src/processing/mode/java/JavaEditor.java index 562efca8a..e9de556b3 100644 --- a/java/src/processing/mode/java/JavaEditor.java +++ b/java/src/processing/mode/java/JavaEditor.java @@ -2285,6 +2285,26 @@ public class JavaEditor extends Editor { } + /** + * @return the Problem for the most relevant error or warning on 'line', + * defaulting to the first. + */ + @Override + protected Problem findProblem(int line) { + List l = findProblems(line); + if (l.size() == 0) return null; + Problem worst = l.get(0); + + for (Problem p : l) { + if (p instanceof JavaProblem && ((!(worst instanceof JavaProblem)) || + ((JavaProblem)p).getPriority() > ((JavaProblem)worst).getPriority())) { + worst = p; + } + } + return worst; + } + + /** * Updates the error table in the Error Window. * Overridden to handle the fugly import suggestions text. diff --git a/java/src/processing/mode/java/pdex/ErrorMessageSimplifier.java b/java/src/processing/mode/java/pdex/ErrorMessageSimplifier.java index c77584e21..f165798d0 100644 --- a/java/src/processing/mode/java/pdex/ErrorMessageSimplifier.java +++ b/java/src/processing/mode/java/pdex/ErrorMessageSimplifier.java @@ -85,7 +85,7 @@ public class ErrorMessageSimplifier { /** * Tones down the jargon in the ecj reported errors. */ - public static String getSimplifiedErrorMessage(IProblem iprob) { + public static String getSimplifiedErrorMessage(IProblem iprob, String badCode) { if (iprob == null) return null; String args[] = iprob.getArguments(); @@ -97,6 +97,7 @@ public class ErrorMessageSimplifier { for (String arg : args) { Messages.log("Arg " + arg); } + Messages.log("Bad code: " + badCode); } String result = null; @@ -111,6 +112,15 @@ public class ErrorMessageSimplifier { case IProblem.ParsingErrorDeleteToken: if (args.length > 0) { + if (args[0].equalsIgnoreCase("Invalid Character")) { + result = getErrorMessageForCurlyQuote(badCode); + } + } + break; + + case IProblem.ParsingErrorDeleteTokens: + result = getErrorMessageForCurlyQuote(badCode); + if (result == null) { result = Language.interpolate("editor.status.error_on", args[0]); } break; @@ -136,13 +146,16 @@ public class ErrorMessageSimplifier { case IProblem.ParsingErrorInvalidToken: if (args.length > 0) { - if (args[1].equals("VariableDeclaratorId")) { - if (args[0].equals("int")) { + if (args[0].equals("int")) { + if (args[1].equals("VariableDeclaratorId")) { result = Language.text("editor.status.reserved_words"); } else { result = Language.interpolate("editor.status.error_on", args[0]); } - } else { + } else if (args[0].equalsIgnoreCase("Invalid Character")) { + result = getErrorMessageForCurlyQuote(badCode); + } + if (result == null) { result = Language.interpolate("editor.status.error_on", args[0]); } } @@ -165,6 +178,9 @@ public class ErrorMessageSimplifier { } break; + case IProblem.ParsingErrorReplaceTokens: + result = getErrorMessageForCurlyQuote(badCode); + case IProblem.UndefinedConstructor: if (args.length == 2) { String constructorName = args[0]; @@ -230,6 +246,13 @@ public class ErrorMessageSimplifier { } break; + case IProblem.UnterminatedString: + if (badCode.contains("“") || badCode.contains("”")) { + result = Language.interpolate("editor.status.unterm_string_curly", + badCode.replaceAll("[^“”]", "")); + } + break; + case IProblem.TypeMismatch: if (args.length > 1) { result = Language.interpolate("editor.status.type_mismatch", args[0], args[1]); @@ -261,16 +284,17 @@ public class ErrorMessageSimplifier { result = Language.interpolate("editor.status.hiding_enclosing_type", args[0]); } break; + } - default: - String message = iprob.getMessage(); - if (message != null) { - // Remove all instances of token - // "Syntax error on token 'blah', delete this token" - Matcher matcher = tokenRegExp.matcher(message); - message = matcher.replaceAll(""); - result = message; - } + if (result == null) { + String message = iprob.getMessage(); + if (message != null) { + // Remove all instances of token + // "Syntax error on token 'blah', delete this token" + Matcher matcher = tokenRegExp.matcher(message); + message = matcher.replaceAll(""); + result = message; + } } if (DEBUG) { @@ -323,6 +347,20 @@ public class ErrorMessageSimplifier { } + /** + * @param badCode The code which may contain curly quotes + * @return Friendly error message if there is a curly quote in badCode, + * null otherwise. + */ + static private String getErrorMessageForCurlyQuote(String badCode) { + if (badCode.contains("‘") || badCode.contains("’") || + badCode.contains("“") || badCode.contains("”")) { + return Language.interpolate("editor.status.bad_curly_quote", + badCode.replaceAll("[^‘’“”]", "")); + } else return null; + } + + // static private final String q(Object quotable) { // return "\"" + quotable + "\""; // } diff --git a/java/src/processing/mode/java/pdex/JavaProblem.java b/java/src/processing/mode/java/pdex/JavaProblem.java index bfe39b007..597648d31 100644 --- a/java/src/processing/mode/java/pdex/JavaProblem.java +++ b/java/src/processing/mode/java/pdex/JavaProblem.java @@ -20,7 +20,10 @@ along with this program; if not, write to the Free Software Foundation, Inc. package processing.mode.java.pdex; +import java.util.Arrays; + import org.eclipse.jdt.core.compiler.IProblem; +import static org.eclipse.jdt.core.compiler.IProblem.*; import processing.app.Problem; @@ -53,6 +56,52 @@ public class JavaProblem implements Problem { */ private int type; + /** + * Priority: bigger = higher. Currently 7 to 10 for errors, + * 4 for warning. + *

+ * The logic of the numbers in the priorityN arrays is that if ECJ wants a + * token got rid of entirely it's most likely the root of the problem. If it + * wants more tokens, that might have been caused by an unterminated string + * or something but it's also likely to be the “real” error. Only if the + * syntax is good are mismatched argument lists and so on of any real + * significance. These rankings are entirely made up so can be changed to + * support any other plausible scenario. + */ + private int priority; + + static private final int[] priority10 = { + ParsingError, + ParsingErrorDeleteToken, + ParsingErrorDeleteTokens, + ParsingErrorInvalidToken, + ParsingErrorMergeTokens, + ParsingErrorMisplacedConstruct, + ParsingErrorNoSuggestion, + ParsingErrorNoSuggestionForTokens, + ParsingErrorOnKeyword, + ParsingErrorOnKeywordNoSuggestion, + ParsingErrorReplaceTokens, + ParsingErrorUnexpectedEOF + }; + static private final int[] priority9 = { + InvalidCharacterConstant, + UnterminatedString + }; + static private final int[] priority8 = { + ParsingErrorInsertToComplete, + ParsingErrorInsertToCompletePhrase, + ParsingErrorInsertToCompleteScope, + ParsingErrorInsertTokenAfter, + ParsingErrorInsertTokenBefore, + }; + // Sorted so I can do a one-line binary search later. + static { + Arrays.sort(priority10); + Arrays.sort(priority9); + Arrays.sort(priority8); + } + /** * If the error is a 'cannot find type' contains the list of suggested imports */ @@ -60,11 +109,12 @@ public class JavaProblem implements Problem { public static final int ERROR = 1, WARNING = 2; - public JavaProblem(String message, int type, int tabIndex, int lineNumber) { + public JavaProblem(String message, int type, int tabIndex, int lineNumber, int priority) { this.message = message; this.type = type; this.tabIndex = tabIndex; this.lineNumber = lineNumber; + this.priority = priority; } /** @@ -72,16 +122,29 @@ public class JavaProblem implements Problem { * @param iProblem - The IProblem which is being wrapped * @param tabIndex - The tab number to which the error belongs to * @param lineNumber - Line number(pde code) of the error + * @param badCode - The code iProblem refers to. */ - public static JavaProblem fromIProblem(IProblem iProblem, int tabIndex, int lineNumber) { + public static JavaProblem fromIProblem(IProblem iProblem, + int tabIndex, int lineNumber, String badCode) { int type = 0; - if(iProblem.isError()) { + int priority = 0; + if (iProblem.isError()) { type = ERROR; + if (Arrays.binarySearch(priority10, iProblem.getID()) >= 0) { + priority = 10; + } else if (Arrays.binarySearch(priority9, iProblem.getID()) >= 0) { + priority = 9; + } else if (Arrays.binarySearch(priority8, iProblem.getID()) >= 0) { + priority = 8; + } else { + priority = 7; + } } else if (iProblem.isWarning()) { type = WARNING; + priority = 4; } - String message = ErrorMessageSimplifier.getSimplifiedErrorMessage(iProblem); - return new JavaProblem(message, type, tabIndex, lineNumber); + String message = ErrorMessageSimplifier.getSimplifiedErrorMessage(iProblem, badCode); + return new JavaProblem(message, type, tabIndex, lineNumber, priority); } public void setPDEOffsets(int startOffset, int stopOffset){ @@ -132,6 +195,10 @@ public class JavaProblem implements Problem { importSuggestions = a; } + public int getPriority() { + return priority; + } + @Override public String toString() { return "TAB " + tabIndex + ",LN " + lineNumber + "LN START OFF: " diff --git a/java/src/processing/mode/java/pdex/PDEX.java b/java/src/processing/mode/java/pdex/PDEX.java index 36add7bd1..80de6b574 100644 --- a/java/src/processing/mode/java/pdex/PDEX.java +++ b/java/src/processing/mode/java/pdex/PDEX.java @@ -1126,7 +1126,10 @@ public class PDEX { SketchInterval in = ps.mapJavaToSketch(start, stop); if (in == SketchInterval.BEFORE_START) return null; int line = ps.tabOffsetToTabLine(in.tabIndex, in.startTabOffset); - JavaProblem p = JavaProblem.fromIProblem(iproblem, in.tabIndex, line); + ps.sketch.updateSketchCodes(); // seems to be needed + String badCode = ps.sketch.getCode(in.tabIndex).getProgram() + .substring(in.startTabOffset, in.stopTabOffset); + JavaProblem p = JavaProblem.fromIProblem(iproblem, in.tabIndex, line, badCode); p.setPDEOffsets(in.startTabOffset, in.stopTabOffset); // Handle import suggestions diff --git a/java/src/processing/mode/java/pdex/SourceUtils.java b/java/src/processing/mode/java/pdex/SourceUtils.java index fecde5c94..9e575af88 100644 --- a/java/src/processing/mode/java/pdex/SourceUtils.java +++ b/java/src/processing/mode/java/pdex/SourceUtils.java @@ -355,7 +355,7 @@ public class SourceUtils { if (depth < 0) { JavaProblem problem = new JavaProblem("Found one too many } characters without { to match it.", - JavaProblem.ERROR, tabIndex, lineNumber); + JavaProblem.ERROR, tabIndex, lineNumber, 8); problem.setPDEOffsets(i - tabStartOffset, i - tabStartOffset + 1); problems.add(problem); continue tabLoop; @@ -364,7 +364,7 @@ public class SourceUtils { if (depth > 0) { JavaProblem problem = new JavaProblem("Found one too many { characters without } to match it.", - JavaProblem.ERROR, tabIndex, lineNumber - 1); + JavaProblem.ERROR, tabIndex, lineNumber - 1, 8); problem.setPDEOffsets(tabEndOffset - tabStartOffset - 2, tabEndOffset - tabStartOffset - 1); problems.add(problem); } From c583d2aaa9ddaf68dc2e80b4e697086d3dcc9d36 Mon Sep 17 00:00:00 2001 From: Jakub Valtar Date: Tue, 19 Sep 2017 16:23:38 +0200 Subject: [PATCH 4/8] To create Problems use pdeCode instead of messing with editor --- java/src/processing/mode/java/pdex/PDEX.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/java/src/processing/mode/java/pdex/PDEX.java b/java/src/processing/mode/java/pdex/PDEX.java index 80de6b574..47c32d4b0 100644 --- a/java/src/processing/mode/java/pdex/PDEX.java +++ b/java/src/processing/mode/java/pdex/PDEX.java @@ -1125,10 +1125,8 @@ public class PDEX { int stop = iproblem.getSourceEnd() + 1; // make it exclusive SketchInterval in = ps.mapJavaToSketch(start, stop); if (in == SketchInterval.BEFORE_START) return null; + String badCode = ps.pdeCode.substring(in.startPdeOffset, in.stopPdeOffset); int line = ps.tabOffsetToTabLine(in.tabIndex, in.startTabOffset); - ps.sketch.updateSketchCodes(); // seems to be needed - String badCode = ps.sketch.getCode(in.tabIndex).getProgram() - .substring(in.startTabOffset, in.stopTabOffset); JavaProblem p = JavaProblem.fromIProblem(iproblem, in.tabIndex, line, badCode); p.setPDEOffsets(in.startTabOffset, in.stopTabOffset); From f506a1df42111eda4b6616bac6436ba296c26b66 Mon Sep 17 00:00:00 2001 From: Jakub Valtar Date: Tue, 19 Sep 2017 16:54:37 +0200 Subject: [PATCH 5/8] In case of multiple problems on a line give priority to error --- app/src/processing/app/ui/Editor.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/app/src/processing/app/ui/Editor.java b/app/src/processing/app/ui/Editor.java index d45a5d447..4ced89bb8 100644 --- a/app/src/processing/app/ui/Editor.java +++ b/app/src/processing/app/ui/Editor.java @@ -3120,20 +3120,14 @@ public abstract class Editor extends JFrame implements RunnerListener { /** * @return the Problem for the most relevant error or warning on 'line', - * defaulting to the first. + * defaults to the first error, if there are no errors first warning. */ protected Problem findProblem(int line) { - int currentTab = getSketch().getCurrentCodeIndex(); - return problems.stream() - .filter(p -> p.getTabIndex() == currentTab) - .filter(p -> { - int pStartLine = p.getLineNumber(); - int pEndOffset = p.getStopOffset(); - int pEndLine = textarea.getLineOfOffset(pEndOffset); - return line >= pStartLine && line <= pEndLine; - }) - .findFirst() - .orElse(null); + List problems = findProblems(line); + for (Problem p : problems) { + if (p.isError()) return p; + } + return problems.isEmpty() ? null : problems.get(0); } From 46b5e29fb3e6e51981f66b8545578b5214b08ed2 Mon Sep 17 00:00:00 2001 From: Jakub Valtar Date: Tue, 19 Sep 2017 22:52:49 +0200 Subject: [PATCH 6/8] Gather error checking code in PDEX.ErrorChecker --- java/src/processing/mode/java/pdex/PDEX.java | 165 ++++++++++++++---- .../mode/java/pdex/PreprocessedSketch.java | 21 ++- .../mode/java/pdex/PreprocessingService.java | 11 +- .../mode/java/pdex/SourceUtils.java | 22 +-- 4 files changed, 150 insertions(+), 69 deletions(-) diff --git a/java/src/processing/mode/java/pdex/PDEX.java b/java/src/processing/mode/java/pdex/PDEX.java index 47c32d4b0..aa673afaa 100644 --- a/java/src/processing/mode/java/pdex/PDEX.java +++ b/java/src/processing/mode/java/pdex/PDEX.java @@ -48,6 +48,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Predicate; +import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -1083,32 +1084,18 @@ public class PDEX { IProblem[] iproblems = ps.compilationUnit.getProblems(); - { // Handle missing brace problems - IProblem missingBraceProblem = Arrays.stream(iproblems) - .filter(ErrorChecker::isMissingBraceProblem) - .findFirst() - // Ignore if it is at the end of file - .filter(p -> p.getSourceEnd() + 1 < ps.javaCode.length()) - // Ignore if the tab number does not match our detected tab number - .filter(p -> ps.missingBraceProblems.isEmpty() || - ps.missingBraceProblems.get(0).getTabIndex() == - ps.mapJavaToSketch(p.getSourceStart(), p.getSourceEnd()+1).tabIndex - ) - .orElse(null); - - // If there is missing brace ignore all other problems - if (missingBraceProblem != null) { - // Prefer ECJ problem, shows location more accurately - iproblems = new IProblem[]{missingBraceProblem}; - } else if (!ps.missingBraceProblems.isEmpty()) { - // Fallback to manual detection - problems.addAll(ps.missingBraceProblems); - } + { // Check for curly quotes + List curlyQuoteProblems = checkForCurlyQuotes(ps); + problems.addAll(curlyQuoteProblems); } - AtomicReference searchClassPath = new AtomicReference<>(null); + if (problems.isEmpty()) { // Check for missing braces + List missingBraceProblems = checkForMissingBraces(ps); + problems.addAll(missingBraceProblems); + } if (problems.isEmpty()) { + AtomicReference searchClassPath = new AtomicReference<>(null); List cuProblems = Arrays.stream(iproblems) // Filter Warnings if they are not enabled .filter(iproblem -> !(iproblem.isWarning() && !JavaMode.warningsEnabled)) @@ -1121,17 +1108,10 @@ public class PDEX { .contains("Syntax error, insert \":: IdentifierOrNew\"")) // Transform into our Problems .map(iproblem -> { - int start = iproblem.getSourceStart(); - int stop = iproblem.getSourceEnd() + 1; // make it exclusive - SketchInterval in = ps.mapJavaToSketch(start, stop); - if (in == SketchInterval.BEFORE_START) return null; - String badCode = ps.pdeCode.substring(in.startPdeOffset, in.stopPdeOffset); - int line = ps.tabOffsetToTabLine(in.tabIndex, in.startTabOffset); - JavaProblem p = JavaProblem.fromIProblem(iproblem, in.tabIndex, line, badCode); - p.setPDEOffsets(in.startTabOffset, in.stopTabOffset); + JavaProblem p = convertIProblem(iproblem, ps); // Handle import suggestions - if (JavaMode.importSuggestEnabled && isUndefinedTypeProblem(iproblem)) { + if (p != null && JavaMode.importSuggestEnabled && isUndefinedTypeProblem(iproblem)) { ClassPath cp = searchClassPath.updateAndGet(prev -> prev != null ? prev : new ClassPathFactory().createFromPaths(ps.searchClassPathArray)); String[] s = suggCache.computeIfAbsent(iproblem.getArguments()[0], @@ -1161,6 +1141,16 @@ public class PDEX { TimeUnit.MILLISECONDS); } + static private JavaProblem convertIProblem(IProblem iproblem, PreprocessedSketch ps) { + SketchInterval in = ps.mapJavaToSketch(iproblem); + if (in == SketchInterval.BEFORE_START) return null; + String badCode = ps.pdeCode.substring(in.startPdeOffset, in.stopPdeOffset); + int line = ps.tabOffsetToTabLine(in.tabIndex, in.startTabOffset); + JavaProblem p = JavaProblem.fromIProblem(iproblem, in.tabIndex, line, badCode); + p.setPDEOffsets(in.startTabOffset, in.stopTabOffset); + return p; + } + static private boolean isUndefinedTypeProblem(IProblem iproblem) { int id = iproblem.getID(); @@ -1186,6 +1176,119 @@ public class PDEX { } + private static final Pattern CURLY_QUOTE_REGEX = + Pattern.compile("([“”‘’])", Pattern.UNICODE_CHARACTER_CLASS); + + static private List checkForCurlyQuotes(PreprocessedSketch ps) { + List problems = new ArrayList<>(0); + + // Go through the scrubbed code and look for curly quotes (they should not be any) + Matcher matcher = CURLY_QUOTE_REGEX.matcher(ps.scrubbedPdeCode); + while (matcher.find()) { + int pdeOffset = matcher.start(); + String q = matcher.group(); + + int tabIndex = ps.pdeOffsetToTabIndex(pdeOffset); + int tabOffset = ps.pdeOffsetToTabOffset(tabIndex, pdeOffset); + int tabLine = ps.tabOffsetToTabLine(tabIndex, tabOffset); + + String message = Language.interpolate("editor.status.bad_curly_quote", q); + JavaProblem problem = new JavaProblem(message, JavaProblem.ERROR, tabIndex, tabLine, 10); + problem.setPDEOffsets(tabOffset, tabOffset+1); + + problems.add(problem); + } + + + // Go through iproblems and look for problems involving curly quotes + List problems2 = new ArrayList<>(0); + IProblem[] iproblems = ps.compilationUnit.getProblems(); + + for (IProblem iproblem : iproblems) { + switch (iproblem.getID()) { + case IProblem.ParsingErrorDeleteToken: + case IProblem.ParsingErrorDeleteTokens: + case IProblem.ParsingErrorInvalidToken: + case IProblem.ParsingErrorReplaceTokens: + case IProblem.UnterminatedString: + SketchInterval in = ps.mapJavaToSketch(iproblem); + if (in == SketchInterval.BEFORE_START) continue; + String badCode = ps.pdeCode.substring(in.startPdeOffset, in.stopPdeOffset); + matcher.reset(badCode); + while (matcher.find()) { + int offset = matcher.start(); + String q = matcher.group(); + int tabStart = in.startTabOffset + offset; + int tabStop = tabStart + 1; + // Prevent duplicate problems + if (problems.stream().noneMatch(p -> p.getStartOffset() == tabStart)) { + int line = ps.tabOffsetToTabLine(in.tabIndex, tabStart); + String message; + if (iproblem.getID() == IProblem.UnterminatedString) { + message = Language.interpolate("editor.status.unterm_string_curly", q); + } else { + message = Language.interpolate("editor.status.bad_curly_quote", q); + } + JavaProblem p = new JavaProblem(message, JavaProblem.ERROR, in.tabIndex, line, 10); + p.setPDEOffsets(tabStart, tabStop); + problems2.add(p); + } + } + } + } + + problems.addAll(problems2); + + return problems; + } + + + static private List checkForMissingBraces(PreprocessedSketch ps) { + List problems = new ArrayList<>(0); + for (int tabIndex = 0; tabIndex < ps.tabStartOffsets.length; tabIndex++) { + int tabStartOffset = ps.tabStartOffsets[tabIndex]; + int tabEndOffset = (tabIndex < ps.tabStartOffsets.length - 1) ? + ps.tabStartOffsets[tabIndex + 1] : ps.scrubbedPdeCode.length(); + int[] braceResult = SourceUtils.checkForMissingBraces(ps.scrubbedPdeCode, tabStartOffset, tabEndOffset); + if (braceResult[0] != 0) { + JavaProblem problem = + new JavaProblem(braceResult[0] < 0 + ? Language.interpolate("editor.status.missing.left_curly_bracket") + : Language.interpolate("editor.status.missing.right_curly_bracket"), + JavaProblem.ERROR, tabIndex, braceResult[1], 8); + problem.setPDEOffsets(braceResult[3], braceResult[3] + 1); + problems.add(problem); + } + } + + if (problems.isEmpty()) { + return problems; + } + + int problemTabIndex = problems.get(0).getTabIndex(); + + IProblem missingBraceProblem = Arrays.stream(ps.compilationUnit.getProblems()) + .filter(ErrorChecker::isMissingBraceProblem) + // Ignore if it is at the end of file + .filter(p -> p.getSourceEnd() + 1 < ps.javaCode.length()) + // Ignore if the tab number does not match our detected tab number + .filter(p -> problemTabIndex == ps.mapJavaToSketch(p).tabIndex) + .findFirst() + .orElse(null); + + // Prefer ECJ problem, shows location more accurately + if (missingBraceProblem != null) { + JavaProblem p = convertIProblem(missingBraceProblem, ps); + if (p != null) { + problems.clear(); + problems.add(p); + } + } + + return problems; + } + + static public String[] getImportSuggestions(ClassPath cp, String className) { RegExpResourceFilter regf = new RegExpResourceFilter( Pattern.compile(".*"), diff --git a/java/src/processing/mode/java/pdex/PreprocessedSketch.java b/java/src/processing/mode/java/pdex/PreprocessedSketch.java index 50838eff8..daaf22ec1 100644 --- a/java/src/processing/mode/java/pdex/PreprocessedSketch.java +++ b/java/src/processing/mode/java/pdex/PreprocessedSketch.java @@ -2,6 +2,7 @@ package processing.mode.java.pdex; import com.google.classpath.ClassPath; +import org.eclipse.jdt.core.compiler.IProblem; import org.eclipse.jdt.core.dom.ASTNode; import org.eclipse.jdt.core.dom.CompilationUnit; @@ -11,7 +12,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import processing.app.Problem; import processing.app.Sketch; import processing.core.PApplet; import processing.mode.java.pdex.TextTransform.OffsetMapper; @@ -30,13 +30,12 @@ public class PreprocessedSketch { public final int[] tabStartOffsets; + public final String scrubbedPdeCode; public final String pdeCode; public final String javaCode; public final OffsetMapper offsetMapper; - public final List missingBraceProblems; - public final boolean hasSyntaxErrors; public final boolean hasCompilationErrors; @@ -84,6 +83,12 @@ public class PreprocessedSketch { } + public SketchInterval mapJavaToSketch(IProblem iproblem) { + return mapJavaToSketch(iproblem.getSourceStart(), + iproblem.getSourceEnd() + 1); // make it exclusive + } + + public SketchInterval mapJavaToSketch(int startJavaOffset, int stopJavaOffset) { int length = stopJavaOffset - startJavaOffset; int startPdeOffset = javaOffsetToPdeOffset(startJavaOffset); @@ -120,7 +125,7 @@ public class PreprocessedSketch { } - private int pdeOffsetToTabIndex(int pdeOffset) { + public int pdeOffsetToTabIndex(int pdeOffset) { pdeOffset = Math.max(0, pdeOffset); int tab = Arrays.binarySearch(tabStartOffsets, pdeOffset); if (tab < 0) { @@ -130,7 +135,7 @@ public class PreprocessedSketch { } - private int pdeOffsetToTabOffset(int tabIndex, int pdeOffset) { + public int pdeOffsetToTabOffset(int tabIndex, int pdeOffset) { int tabStartOffset = tabStartOffsets[clipTabIndex(tabIndex)]; return pdeOffset - tabStartOffset; } @@ -210,13 +215,12 @@ public class PreprocessedSketch { public int[] tabStartOffsets = new int[0]; + public String scrubbedPdeCode; public String pdeCode; public String javaCode; public OffsetMapper offsetMapper; - public final List missingBraceProblems = new ArrayList<>(0); - public boolean hasSyntaxErrors; public boolean hasCompilationErrors; @@ -246,13 +250,12 @@ public class PreprocessedSketch { tabStartOffsets = b.tabStartOffsets; + scrubbedPdeCode = b.scrubbedPdeCode; pdeCode = b.pdeCode; javaCode = b.javaCode; offsetMapper = b.offsetMapper != null ? b.offsetMapper : OffsetMapper.EMPTY_MAPPER; - missingBraceProblems = Collections.unmodifiableList(b.missingBraceProblems); - hasSyntaxErrors = b.hasSyntaxErrors; hasCompilationErrors = b.hasCompilationErrors; diff --git a/java/src/processing/mode/java/pdex/PreprocessingService.java b/java/src/processing/mode/java/pdex/PreprocessingService.java index 5e9d3f0bf..86990342b 100644 --- a/java/src/processing/mode/java/pdex/PreprocessingService.java +++ b/java/src/processing/mode/java/pdex/PreprocessingService.java @@ -312,6 +312,8 @@ public class PreprocessingService { SourceUtils.scrubCommentsAndStrings(workBuffer); + result.scrubbedPdeCode = workBuffer.toString(); + Mode sketchMode = PdePreprocessor.parseMode(workBuffer); // Prepare transforms to convert pde code into parsable code @@ -394,15 +396,6 @@ public class PreprocessingService { } } - { // Check for missing braces - List missingBraceProblems = - SourceUtils.checkForMissingBraces(workBuffer, result.tabStartOffsets); - if (!missingBraceProblems.isEmpty()) { - result.missingBraceProblems.addAll(missingBraceProblems); - result.hasSyntaxErrors = true; - } - } - // Transform code to parsable state String parsableStage = toParsable.apply(); OffsetMapper parsableMapper = toParsable.getMapper(); diff --git a/java/src/processing/mode/java/pdex/SourceUtils.java b/java/src/processing/mode/java/pdex/SourceUtils.java index 8e9968f99..bb29cda84 100644 --- a/java/src/processing/mode/java/pdex/SourceUtils.java +++ b/java/src/processing/mode/java/pdex/SourceUtils.java @@ -331,27 +331,9 @@ public class SourceUtils { } - static public List checkForMissingBraces(StringBuilder p, int[] tabStartOffsets) { - List problems = new ArrayList<>(0); - for (int tabIndex = 0; tabIndex < tabStartOffsets.length; tabIndex++) { - int tabStartOffset = tabStartOffsets[tabIndex]; - int tabEndOffset = (tabIndex < tabStartOffsets.length - 1) ? - tabStartOffsets[tabIndex + 1] : p.length(); - int[] braceResult = checkForMissingBraces(p, tabStartOffset, tabEndOffset); - if (braceResult[0] != 0) { - JavaProblem problem = - new JavaProblem(braceResult[0] < 0 - ? "Found one too many } characters without { to match it." - : "Found one too many { characters without } to match it.", - JavaProblem.ERROR, tabIndex, braceResult[1], 8); - problem.setPDEOffsets(braceResult[3], braceResult[3] + 1); - problems.add(problem); - } - } - return problems; - } - + // TODO: move this to a better place when JavaBuild starts using JDT and we + // don't need to check errors at two different places [jv 2017-09-19] /** * Checks a single code fragment (such as a tab) for non-matching braces. * Broken out to allow easy use in JavaBuild. From 25ac5db0ab608bf3fe8251cae01494c92f451abe Mon Sep 17 00:00:00 2001 From: Jakub Valtar Date: Tue, 19 Sep 2017 23:04:56 +0200 Subject: [PATCH 7/8] Remove problem priority since we're going to do it manually in PDEX.ErrorChecker --- java/src/processing/mode/java/JavaEditor.java | 20 ------ .../mode/java/pdex/JavaProblem.java | 71 +------------------ java/src/processing/mode/java/pdex/PDEX.java | 6 +- 3 files changed, 6 insertions(+), 91 deletions(-) diff --git a/java/src/processing/mode/java/JavaEditor.java b/java/src/processing/mode/java/JavaEditor.java index e9de556b3..562efca8a 100644 --- a/java/src/processing/mode/java/JavaEditor.java +++ b/java/src/processing/mode/java/JavaEditor.java @@ -2285,26 +2285,6 @@ public class JavaEditor extends Editor { } - /** - * @return the Problem for the most relevant error or warning on 'line', - * defaulting to the first. - */ - @Override - protected Problem findProblem(int line) { - List l = findProblems(line); - if (l.size() == 0) return null; - Problem worst = l.get(0); - - for (Problem p : l) { - if (p instanceof JavaProblem && ((!(worst instanceof JavaProblem)) || - ((JavaProblem)p).getPriority() > ((JavaProblem)worst).getPriority())) { - worst = p; - } - } - return worst; - } - - /** * Updates the error table in the Error Window. * Overridden to handle the fugly import suggestions text. diff --git a/java/src/processing/mode/java/pdex/JavaProblem.java b/java/src/processing/mode/java/pdex/JavaProblem.java index 597648d31..b8ef76c0a 100644 --- a/java/src/processing/mode/java/pdex/JavaProblem.java +++ b/java/src/processing/mode/java/pdex/JavaProblem.java @@ -20,10 +20,7 @@ along with this program; if not, write to the Free Software Foundation, Inc. package processing.mode.java.pdex; -import java.util.Arrays; - import org.eclipse.jdt.core.compiler.IProblem; -import static org.eclipse.jdt.core.compiler.IProblem.*; import processing.app.Problem; @@ -56,52 +53,6 @@ public class JavaProblem implements Problem { */ private int type; - /** - * Priority: bigger = higher. Currently 7 to 10 for errors, - * 4 for warning. - *

- * The logic of the numbers in the priorityN arrays is that if ECJ wants a - * token got rid of entirely it's most likely the root of the problem. If it - * wants more tokens, that might have been caused by an unterminated string - * or something but it's also likely to be the “real” error. Only if the - * syntax is good are mismatched argument lists and so on of any real - * significance. These rankings are entirely made up so can be changed to - * support any other plausible scenario. - */ - private int priority; - - static private final int[] priority10 = { - ParsingError, - ParsingErrorDeleteToken, - ParsingErrorDeleteTokens, - ParsingErrorInvalidToken, - ParsingErrorMergeTokens, - ParsingErrorMisplacedConstruct, - ParsingErrorNoSuggestion, - ParsingErrorNoSuggestionForTokens, - ParsingErrorOnKeyword, - ParsingErrorOnKeywordNoSuggestion, - ParsingErrorReplaceTokens, - ParsingErrorUnexpectedEOF - }; - static private final int[] priority9 = { - InvalidCharacterConstant, - UnterminatedString - }; - static private final int[] priority8 = { - ParsingErrorInsertToComplete, - ParsingErrorInsertToCompletePhrase, - ParsingErrorInsertToCompleteScope, - ParsingErrorInsertTokenAfter, - ParsingErrorInsertTokenBefore, - }; - // Sorted so I can do a one-line binary search later. - static { - Arrays.sort(priority10); - Arrays.sort(priority9); - Arrays.sort(priority8); - } - /** * If the error is a 'cannot find type' contains the list of suggested imports */ @@ -109,12 +60,11 @@ public class JavaProblem implements Problem { public static final int ERROR = 1, WARNING = 2; - public JavaProblem(String message, int type, int tabIndex, int lineNumber, int priority) { + public JavaProblem(String message, int type, int tabIndex, int lineNumber) { this.message = message; this.type = type; this.tabIndex = tabIndex; this.lineNumber = lineNumber; - this.priority = priority; } /** @@ -127,24 +77,13 @@ public class JavaProblem implements Problem { public static JavaProblem fromIProblem(IProblem iProblem, int tabIndex, int lineNumber, String badCode) { int type = 0; - int priority = 0; - if (iProblem.isError()) { + if(iProblem.isError()) { type = ERROR; - if (Arrays.binarySearch(priority10, iProblem.getID()) >= 0) { - priority = 10; - } else if (Arrays.binarySearch(priority9, iProblem.getID()) >= 0) { - priority = 9; - } else if (Arrays.binarySearch(priority8, iProblem.getID()) >= 0) { - priority = 8; - } else { - priority = 7; - } } else if (iProblem.isWarning()) { type = WARNING; - priority = 4; } String message = ErrorMessageSimplifier.getSimplifiedErrorMessage(iProblem, badCode); - return new JavaProblem(message, type, tabIndex, lineNumber, priority); + return new JavaProblem(message, type, tabIndex, lineNumber); } public void setPDEOffsets(int startOffset, int stopOffset){ @@ -195,10 +134,6 @@ public class JavaProblem implements Problem { importSuggestions = a; } - public int getPriority() { - return priority; - } - @Override public String toString() { return "TAB " + tabIndex + ",LN " + lineNumber + "LN START OFF: " diff --git a/java/src/processing/mode/java/pdex/PDEX.java b/java/src/processing/mode/java/pdex/PDEX.java index aa673afaa..5d37a9d51 100644 --- a/java/src/processing/mode/java/pdex/PDEX.java +++ b/java/src/processing/mode/java/pdex/PDEX.java @@ -1193,7 +1193,7 @@ public class PDEX { int tabLine = ps.tabOffsetToTabLine(tabIndex, tabOffset); String message = Language.interpolate("editor.status.bad_curly_quote", q); - JavaProblem problem = new JavaProblem(message, JavaProblem.ERROR, tabIndex, tabLine, 10); + JavaProblem problem = new JavaProblem(message, JavaProblem.ERROR, tabIndex, tabLine); problem.setPDEOffsets(tabOffset, tabOffset+1); problems.add(problem); @@ -1229,7 +1229,7 @@ public class PDEX { } else { message = Language.interpolate("editor.status.bad_curly_quote", q); } - JavaProblem p = new JavaProblem(message, JavaProblem.ERROR, in.tabIndex, line, 10); + JavaProblem p = new JavaProblem(message, JavaProblem.ERROR, in.tabIndex, line); p.setPDEOffsets(tabStart, tabStop); problems2.add(p); } @@ -1255,7 +1255,7 @@ public class PDEX { new JavaProblem(braceResult[0] < 0 ? Language.interpolate("editor.status.missing.left_curly_bracket") : Language.interpolate("editor.status.missing.right_curly_bracket"), - JavaProblem.ERROR, tabIndex, braceResult[1], 8); + JavaProblem.ERROR, tabIndex, braceResult[1]); problem.setPDEOffsets(braceResult[3], braceResult[3] + 1); problems.add(problem); } From 8c66a8c33dd0e41b42f627f5b222a11ba96d5756 Mon Sep 17 00:00:00 2001 From: Jakub Valtar Date: Tue, 19 Sep 2017 23:14:26 +0200 Subject: [PATCH 8/8] Make sure we don't try to get code out of bounds --- java/src/processing/mode/java/pdex/PDEX.java | 4 ++-- .../src/processing/mode/java/pdex/PreprocessedSketch.java | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/java/src/processing/mode/java/pdex/PDEX.java b/java/src/processing/mode/java/pdex/PDEX.java index 5d37a9d51..9f84ccbcf 100644 --- a/java/src/processing/mode/java/pdex/PDEX.java +++ b/java/src/processing/mode/java/pdex/PDEX.java @@ -1144,7 +1144,7 @@ public class PDEX { static private JavaProblem convertIProblem(IProblem iproblem, PreprocessedSketch ps) { SketchInterval in = ps.mapJavaToSketch(iproblem); if (in == SketchInterval.BEFORE_START) return null; - String badCode = ps.pdeCode.substring(in.startPdeOffset, in.stopPdeOffset); + String badCode = ps.getPdeCode(in); int line = ps.tabOffsetToTabLine(in.tabIndex, in.startTabOffset); JavaProblem p = JavaProblem.fromIProblem(iproblem, in.tabIndex, line, badCode); p.setPDEOffsets(in.startTabOffset, in.stopTabOffset); @@ -1213,7 +1213,7 @@ public class PDEX { case IProblem.UnterminatedString: SketchInterval in = ps.mapJavaToSketch(iproblem); if (in == SketchInterval.BEFORE_START) continue; - String badCode = ps.pdeCode.substring(in.startPdeOffset, in.stopPdeOffset); + String badCode = ps.getPdeCode(in); matcher.reset(badCode); while (matcher.find()) { int offset = matcher.start(); diff --git a/java/src/processing/mode/java/pdex/PreprocessedSketch.java b/java/src/processing/mode/java/pdex/PreprocessedSketch.java index daaf22ec1..3dc328967 100644 --- a/java/src/processing/mode/java/pdex/PreprocessedSketch.java +++ b/java/src/processing/mode/java/pdex/PreprocessedSketch.java @@ -77,6 +77,14 @@ public class PreprocessedSketch { } + public String getPdeCode(SketchInterval si) { + if (si == SketchInterval.BEFORE_START) return ""; + int stop = Math.min(si.stopPdeOffset, pdeCode.length()); + int start = Math.min(si.startPdeOffset, stop); + return pdeCode.substring(start, stop); + } + + public SketchInterval mapJavaToSketch(ASTNode node) { return mapJavaToSketch(node.getStartPosition(), node.getStartPosition() + node.getLength());