eessppeelllloo/processing4
1
0
Fork 0
mirror of https://github.com/processing/processing4.git synced 2026-01-29 03:11:08 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5152 from GKFX/featurefixbadquotes

Browse Source 
Handle curly quotes well
This commit is contained in:
Ben Fry
2017-09-24 11:00:34 -04:00
committed by GitHub
parent fb6a4a01df 632d19c6ca
commit bc4471bf38
11 changed files with 287 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
java/src/processing/mode/java/AutoFormat.java
View File

@@ -639,25 +639,48 @@ public class AutoFormat implements Formatter {
break;
case '"':
case '“':
case '”':
case '\'':
case '‘':
case '’':
inStatementFlag = true;
buf.append(c);
char realQuote = c;
if (c == '“' || c == '”') realQuote = '"';
if (c == '‘' || c == '’') realQuote = '\'';
buf.append(realQuote);
char otherQuote = c;
if (c == '“') otherQuote = '”';
if (c == '”') otherQuote = '“';
if (c == '‘') otherQuote = '’';
if (c == '’') otherQuote = '‘';
char cc = nextChar();
while (!EOF && cc != c) {
// In a proper string, all the quotes tested are c. In a curly-quoted
// string, there are three possible end quotes: c, its reverse, and
// the correct straight quote.
while (!EOF && cc != otherQuote && cc != realQuote && cc != c) {
buf.append(cc);
if (cc == '\\') {
buf.append(cc = nextChar());
}
if (cc == '\n') {
writeIndentedLine();
startFlag = true;
}
// Syntax error: unterminated string. Leave \n in nextChar, so it
// feeds back into the loop.
if (peek() == '\n') break;
cc = nextChar();
}
buf.append(cc);
if (readForNewLine()) {
// push a newline into the stream
chars[pos--] = '\n';
if (cc == otherQuote || cc == realQuote || cc == c) {
buf.append(realQuote);
if (readForNewLine()) {
// push a newline into the stream
chars[pos--] = '\n';
}
} else {
// We've had a syntax error if the string wasn't terminated by EOL/
// EOF, just abandon this statement.
inStatementFlag = false;
}
break;

28
java/src/processing/mode/java/JavaBuild.java
View File

@@ -38,6 +38,7 @@ import org.apache.tools.ant.Project;
import org.apache.tools.ant.ProjectHelper;
import processing.app.Base;
import processing.app.Language;
import processing.app.Library;
import processing.app.Messages;
import processing.app.Mode;
@@ -345,9 +346,30 @@ public class JavaBuild {
// System.err.println("and then she tells me " + tsre.toString());
// TODO not tested since removing ORO matcher.. ^ could be a problem
String mess = "^line (\\d+):(\\d+):\\s";
String locationRegex = "^line (\\d+):(\\d+):\\s";
String message = tsre.getMessage();
String[] m;
String[] matches = PApplet.match(tsre.toString(), mess);
if (null != (m = PApplet.match(tsre.toString(),
"unexpected char: (.*)"))) {
char c = 0;
if (m[1].startsWith("0x")) { // Hex
c = (char) PApplet.unhex(m[1].substring(2));
} else if (m[1].length() == 3) { // Quoted
c = m[1].charAt(1);
} else if (m[1].length() == 1) { // Alone
c = m[1].charAt(0);
}
if (c == '\u201C' || c == '\u201D' || // “”
c == '\u2018' || c == '\u2019') { // ‘’
message = Language.interpolate("editor.status.bad_curly_quote", c);
} else if (c != 0) {
message = "Not expecting symbol " + m[1] +
", which is " + Character.getName(c) + ".";
}
}
String[] matches = PApplet.match(tsre.toString(), locationRegex);
if (matches != null) {
int errorLine = Integer.parseInt(matches[1]) - 1;
int errorColumn = Integer.parseInt(matches[2]);
@@ -362,7 +384,7 @@ public class JavaBuild {
}
errorLine -= sketch.getCode(errorFile).getPreprocOffset();
throw new SketchException(tsre.getMessage(),
throw new SketchException(message,
errorFile, errorLine, errorColumn);
} else {

64
java/src/processing/mode/java/pdex/ErrorMessageSimplifier.java
View File

@@ -85,7 +85,7 @@ public class ErrorMessageSimplifier {
/**
* Tones down the jargon in the ecj reported errors.
*/
public static String getSimplifiedErrorMessage(IProblem iprob) {
public static String getSimplifiedErrorMessage(IProblem iprob, String badCode) {
if (iprob == null) return null;
String args[] = iprob.getArguments();
@@ -97,6 +97,7 @@ public class ErrorMessageSimplifier {
for (String arg : args) {
Messages.log("Arg " + arg);
}
Messages.log("Bad code: " + badCode);
}
String result = null;
@@ -111,6 +112,15 @@ public class ErrorMessageSimplifier {
case IProblem.ParsingErrorDeleteToken:
if (args.length > 0) {
if (args[0].equalsIgnoreCase("Invalid Character")) {
result = getErrorMessageForCurlyQuote(badCode);
}
}
break;
case IProblem.ParsingErrorDeleteTokens:
result = getErrorMessageForCurlyQuote(badCode);
if (result == null) {
result = Language.interpolate("editor.status.error_on", args[0]);
}
break;
@@ -136,13 +146,16 @@ public class ErrorMessageSimplifier {
case IProblem.ParsingErrorInvalidToken:
if (args.length > 0) {
if (args[1].equals("VariableDeclaratorId")) {
if (args[0].equals("int")) {
if (args[0].equals("int")) {
if (args[1].equals("VariableDeclaratorId")) {
result = Language.text("editor.status.reserved_words");
} else {
result = Language.interpolate("editor.status.error_on", args[0]);
}
} else {
} else if (args[0].equalsIgnoreCase("Invalid Character")) {
result = getErrorMessageForCurlyQuote(badCode);
}
if (result == null) {
result = Language.interpolate("editor.status.error_on", args[0]);
}
}
@@ -165,6 +178,9 @@ public class ErrorMessageSimplifier {
}
break;
case IProblem.ParsingErrorReplaceTokens:
result = getErrorMessageForCurlyQuote(badCode);
case IProblem.UndefinedConstructor:
if (args.length == 2) {
String constructorName = args[0];
@@ -230,6 +246,13 @@ public class ErrorMessageSimplifier {
}
break;
case IProblem.UnterminatedString:
if (badCode.contains("“") || badCode.contains("”")) {
result = Language.interpolate("editor.status.unterm_string_curly",
badCode.replaceAll("[^“”]", ""));
}
break;
case IProblem.TypeMismatch:
if (args.length > 1) {
result = Language.interpolate("editor.status.type_mismatch", args[0], args[1]);
@@ -261,16 +284,17 @@ public class ErrorMessageSimplifier {
result = Language.interpolate("editor.status.hiding_enclosing_type", args[0]);
}
break;
}
default:
String message = iprob.getMessage();
if (message != null) {
// Remove all instances of token
// "Syntax error on token 'blah', delete this token"
Matcher matcher = tokenRegExp.matcher(message);
message = matcher.replaceAll("");
result = message;
}
if (result == null) {
String message = iprob.getMessage();
if (message != null) {
// Remove all instances of token
// "Syntax error on token 'blah', delete this token"
Matcher matcher = tokenRegExp.matcher(message);
message = matcher.replaceAll("");
result = message;
}
}
if (DEBUG) {
@@ -323,6 +347,20 @@ public class ErrorMessageSimplifier {
}
/**
* @param badCode The code which may contain curly quotes
* @return Friendly error message if there is a curly quote in badCode,
* null otherwise.
*/
static private String getErrorMessageForCurlyQuote(String badCode) {
if (badCode.contains("‘") || badCode.contains("’") ||
badCode.contains("“") || badCode.contains("”")) {
return Language.interpolate("editor.status.bad_curly_quote",
badCode.replaceAll("[^‘’“”]", ""));
} else return null;
}
// static private final String q(Object quotable) {
// return "\"" + quotable + "\"";
// }

6
java/src/processing/mode/java/pdex/JavaProblem.java
View File

@@ -72,15 +72,17 @@ public class JavaProblem implements Problem {
* @param iProblem - The IProblem which is being wrapped
* @param tabIndex - The tab number to which the error belongs to
* @param lineNumber - Line number(pde code) of the error
* @param badCode - The code iProblem refers to.
*/
public static JavaProblem fromIProblem(IProblem iProblem, int tabIndex, int lineNumber) {
public static JavaProblem fromIProblem(IProblem iProblem,
int tabIndex, int lineNumber, String badCode) {
int type = 0;
if(iProblem.isError()) {
type = ERROR;
} else if (iProblem.isWarning()) {
type = WARNING;
}
String message = ErrorMessageSimplifier.getSimplifiedErrorMessage(iProblem);
String message = ErrorMessageSimplifier.getSimplifiedErrorMessage(iProblem, badCode);
return new JavaProblem(message, type, tabIndex, lineNumber);
}

164
java/src/processing/mode/java/pdex/PDEX.java
View File

@@ -48,6 +48,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -1083,32 +1084,18 @@ public class PDEX {
IProblem[] iproblems = ps.compilationUnit.getProblems();
{ // Handle missing brace problems
IProblem missingBraceProblem = Arrays.stream(iproblems)
.filter(ErrorChecker::isMissingBraceProblem)
.findFirst()
// Ignore if it is at the end of file
.filter(p -> p.getSourceEnd() + 1 < ps.javaCode.length())
// Ignore if the tab number does not match our detected tab number
.filter(p -> ps.missingBraceProblems.isEmpty() ||
ps.missingBraceProblems.get(0).getTabIndex() ==
ps.mapJavaToSketch(p.getSourceStart(), p.getSourceEnd()+1).tabIndex
)
.orElse(null);
// If there is missing brace ignore all other problems
if (missingBraceProblem != null) {
// Prefer ECJ problem, shows location more accurately
iproblems = new IProblem[]{missingBraceProblem};
} else if (!ps.missingBraceProblems.isEmpty()) {
// Fallback to manual detection
problems.addAll(ps.missingBraceProblems);
}
{ // Check for curly quotes
List<JavaProblem> curlyQuoteProblems = checkForCurlyQuotes(ps);
problems.addAll(curlyQuoteProblems);
}
AtomicReference<ClassPath> searchClassPath = new AtomicReference<>(null);
if (problems.isEmpty()) { // Check for missing braces
List<JavaProblem> missingBraceProblems = checkForMissingBraces(ps);
problems.addAll(missingBraceProblems);
}
if (problems.isEmpty()) {
AtomicReference<ClassPath> searchClassPath = new AtomicReference<>(null);
List<Problem> cuProblems = Arrays.stream(iproblems)
// Filter Warnings if they are not enabled
.filter(iproblem -> !(iproblem.isWarning() && !JavaMode.warningsEnabled))
@@ -1121,16 +1108,10 @@ public class PDEX {
.contains("Syntax error, insert \":: IdentifierOrNew\""))
// Transform into our Problems
.map(iproblem -> {
int start = iproblem.getSourceStart();
int stop = iproblem.getSourceEnd() + 1; // make it exclusive
SketchInterval in = ps.mapJavaToSketch(start, stop);
if (in == SketchInterval.BEFORE_START) return null;
int line = ps.tabOffsetToTabLine(in.tabIndex, in.startTabOffset);
JavaProblem p = JavaProblem.fromIProblem(iproblem, in.tabIndex, line);
p.setPDEOffsets(in.startTabOffset, in.stopTabOffset);
JavaProblem p = convertIProblem(iproblem, ps);
// Handle import suggestions
if (JavaMode.importSuggestEnabled && isUndefinedTypeProblem(iproblem)) {
if (p != null && JavaMode.importSuggestEnabled && isUndefinedTypeProblem(iproblem)) {
ClassPath cp = searchClassPath.updateAndGet(prev -> prev != null ?
prev : new ClassPathFactory().createFromPaths(ps.searchClassPathArray));
String[] s = suggCache.computeIfAbsent(iproblem.getArguments()[0],
@@ -1160,6 +1141,16 @@ public class PDEX {
TimeUnit.MILLISECONDS);
}
static private JavaProblem convertIProblem(IProblem iproblem, PreprocessedSketch ps) {
SketchInterval in = ps.mapJavaToSketch(iproblem);
if (in == SketchInterval.BEFORE_START) return null;
String badCode = ps.getPdeCode(in);
int line = ps.tabOffsetToTabLine(in.tabIndex, in.startTabOffset);
JavaProblem p = JavaProblem.fromIProblem(iproblem, in.tabIndex, line, badCode);
p.setPDEOffsets(in.startTabOffset, in.stopTabOffset);
return p;
}
static private boolean isUndefinedTypeProblem(IProblem iproblem) {
int id = iproblem.getID();
@@ -1185,6 +1176,119 @@ public class PDEX {
}
private static final Pattern CURLY_QUOTE_REGEX =
Pattern.compile("([“”‘’])", Pattern.UNICODE_CHARACTER_CLASS);
static private List<JavaProblem> checkForCurlyQuotes(PreprocessedSketch ps) {
List<JavaProblem> problems = new ArrayList<>(0);
// Go through the scrubbed code and look for curly quotes (they should not be any)
Matcher matcher = CURLY_QUOTE_REGEX.matcher(ps.scrubbedPdeCode);
while (matcher.find()) {
int pdeOffset = matcher.start();
String q = matcher.group();
int tabIndex = ps.pdeOffsetToTabIndex(pdeOffset);
int tabOffset = ps.pdeOffsetToTabOffset(tabIndex, pdeOffset);
int tabLine = ps.tabOffsetToTabLine(tabIndex, tabOffset);
String message = Language.interpolate("editor.status.bad_curly_quote", q);
JavaProblem problem = new JavaProblem(message, JavaProblem.ERROR, tabIndex, tabLine);
problem.setPDEOffsets(tabOffset, tabOffset+1);
problems.add(problem);
}
// Go through iproblems and look for problems involving curly quotes
List<JavaProblem> problems2 = new ArrayList<>(0);
IProblem[] iproblems = ps.compilationUnit.getProblems();
for (IProblem iproblem : iproblems) {
switch (iproblem.getID()) {
case IProblem.ParsingErrorDeleteToken:
case IProblem.ParsingErrorDeleteTokens:
case IProblem.ParsingErrorInvalidToken:
case IProblem.ParsingErrorReplaceTokens:
case IProblem.UnterminatedString:
SketchInterval in = ps.mapJavaToSketch(iproblem);
if (in == SketchInterval.BEFORE_START) continue;
String badCode = ps.getPdeCode(in);
matcher.reset(badCode);
while (matcher.find()) {
int offset = matcher.start();
String q = matcher.group();
int tabStart = in.startTabOffset + offset;
int tabStop = tabStart + 1;
// Prevent duplicate problems
if (problems.stream().noneMatch(p -> p.getStartOffset() == tabStart)) {
int line = ps.tabOffsetToTabLine(in.tabIndex, tabStart);
String message;
if (iproblem.getID() == IProblem.UnterminatedString) {
message = Language.interpolate("editor.status.unterm_string_curly", q);
} else {
message = Language.interpolate("editor.status.bad_curly_quote", q);
}
JavaProblem p = new JavaProblem(message, JavaProblem.ERROR, in.tabIndex, line);
p.setPDEOffsets(tabStart, tabStop);
problems2.add(p);
}
}
}
}
problems.addAll(problems2);
return problems;
}
static private List<JavaProblem> checkForMissingBraces(PreprocessedSketch ps) {
List<JavaProblem> problems = new ArrayList<>(0);
for (int tabIndex = 0; tabIndex < ps.tabStartOffsets.length; tabIndex++) {
int tabStartOffset = ps.tabStartOffsets[tabIndex];
int tabEndOffset = (tabIndex < ps.tabStartOffsets.length - 1) ?
ps.tabStartOffsets[tabIndex + 1] : ps.scrubbedPdeCode.length();
int[] braceResult = SourceUtils.checkForMissingBraces(ps.scrubbedPdeCode, tabStartOffset, tabEndOffset);
if (braceResult[0] != 0) {
JavaProblem problem =
new JavaProblem(braceResult[0] < 0
? Language.interpolate("editor.status.missing.left_curly_bracket")
: Language.interpolate("editor.status.missing.right_curly_bracket"),
JavaProblem.ERROR, tabIndex, braceResult[1]);
problem.setPDEOffsets(braceResult[3], braceResult[3] + 1);
problems.add(problem);
}
}
if (problems.isEmpty()) {
return problems;
}
int problemTabIndex = problems.get(0).getTabIndex();
IProblem missingBraceProblem = Arrays.stream(ps.compilationUnit.getProblems())
.filter(ErrorChecker::isMissingBraceProblem)
// Ignore if it is at the end of file
.filter(p -> p.getSourceEnd() + 1 < ps.javaCode.length())
// Ignore if the tab number does not match our detected tab number
.filter(p -> problemTabIndex == ps.mapJavaToSketch(p).tabIndex)
.findFirst()
.orElse(null);
// Prefer ECJ problem, shows location more accurately
if (missingBraceProblem != null) {
JavaProblem p = convertIProblem(missingBraceProblem, ps);
if (p != null) {
problems.clear();
problems.add(p);
}
}
return problems;
}
static public String[] getImportSuggestions(ClassPath cp, String className) {
RegExpResourceFilter regf = new RegExpResourceFilter(
Pattern.compile(".*"),

29
java/src/processing/mode/java/pdex/PreprocessedSketch.java
View File

@@ -2,6 +2,7 @@ package processing.mode.java.pdex;
import com.google.classpath.ClassPath;
import org.eclipse.jdt.core.compiler.IProblem;
import org.eclipse.jdt.core.dom.ASTNode;
import org.eclipse.jdt.core.dom.CompilationUnit;
@@ -11,7 +12,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import processing.app.Problem;
import processing.app.Sketch;
import processing.core.PApplet;
import processing.mode.java.pdex.TextTransform.OffsetMapper;
@@ -30,13 +30,12 @@ public class PreprocessedSketch {
public final int[] tabStartOffsets;
public final String scrubbedPdeCode;
public final String pdeCode;
public final String javaCode;
public final OffsetMapper offsetMapper;
public final List<Problem> missingBraceProblems;
public final boolean hasSyntaxErrors;
public final boolean hasCompilationErrors;
@@ -78,12 +77,26 @@ public class PreprocessedSketch {
}
public String getPdeCode(SketchInterval si) {
if (si == SketchInterval.BEFORE_START) return "";
int stop = Math.min(si.stopPdeOffset, pdeCode.length());
int start = Math.min(si.startPdeOffset, stop);
return pdeCode.substring(start, stop);
}
public SketchInterval mapJavaToSketch(ASTNode node) {
return mapJavaToSketch(node.getStartPosition(),
node.getStartPosition() + node.getLength());
}
public SketchInterval mapJavaToSketch(IProblem iproblem) {
return mapJavaToSketch(iproblem.getSourceStart(),
iproblem.getSourceEnd() + 1); // make it exclusive
}
public SketchInterval mapJavaToSketch(int startJavaOffset, int stopJavaOffset) {
int length = stopJavaOffset - startJavaOffset;
int startPdeOffset = javaOffsetToPdeOffset(startJavaOffset);
@@ -120,7 +133,7 @@ public class PreprocessedSketch {
}
private int pdeOffsetToTabIndex(int pdeOffset) {
public int pdeOffsetToTabIndex(int pdeOffset) {
pdeOffset = Math.max(0, pdeOffset);
int tab = Arrays.binarySearch(tabStartOffsets, pdeOffset);
if (tab < 0) {
@@ -130,7 +143,7 @@ public class PreprocessedSketch {
}
private int pdeOffsetToTabOffset(int tabIndex, int pdeOffset) {
public int pdeOffsetToTabOffset(int tabIndex, int pdeOffset) {
int tabStartOffset = tabStartOffsets[clipTabIndex(tabIndex)];
return pdeOffset - tabStartOffset;
}
@@ -210,13 +223,12 @@ public class PreprocessedSketch {
public int[] tabStartOffsets = new int[0];
public String scrubbedPdeCode;
public String pdeCode;
public String javaCode;
public OffsetMapper offsetMapper;
public final List<Problem> missingBraceProblems = new ArrayList<>(0);
public boolean hasSyntaxErrors;
public boolean hasCompilationErrors;
@@ -246,13 +258,12 @@ public class PreprocessedSketch {
tabStartOffsets = b.tabStartOffsets;
scrubbedPdeCode = b.scrubbedPdeCode;
pdeCode = b.pdeCode;
javaCode = b.javaCode;
offsetMapper = b.offsetMapper != null ? b.offsetMapper : OffsetMapper.EMPTY_MAPPER;
missingBraceProblems = Collections.unmodifiableList(b.missingBraceProblems);
hasSyntaxErrors = b.hasSyntaxErrors;
hasCompilationErrors = b.hasCompilationErrors;

11
java/src/processing/mode/java/pdex/PreprocessingService.java
View File

@@ -312,6 +312,8 @@ public class PreprocessingService {
SourceUtils.scrubCommentsAndStrings(workBuffer);
result.scrubbedPdeCode = workBuffer.toString();
Mode sketchMode = PdePreprocessor.parseMode(workBuffer);
// Prepare transforms to convert pde code into parsable code
@@ -394,15 +396,6 @@ public class PreprocessingService {
}
}
{ // Check for missing braces
List<JavaProblem> missingBraceProblems =
SourceUtils.checkForMissingBraces(workBuffer, result.tabStartOffsets);
if (!missingBraceProblems.isEmpty()) {
result.missingBraceProblems.addAll(missingBraceProblems);
result.hasSyntaxErrors = true;
}
}
// Transform code to parsable state
String parsableStage = toParsable.apply();
OffsetMapper parsableMapper = toParsable.getMapper();

22
java/src/processing/mode/java/pdex/SourceUtils.java
View File

@@ -331,27 +331,9 @@ public class SourceUtils {
}
static public List<JavaProblem> checkForMissingBraces(StringBuilder p, int[] tabStartOffsets) {
List<JavaProblem> problems = new ArrayList<>(0);
for (int tabIndex = 0; tabIndex < tabStartOffsets.length; tabIndex++) {
int tabStartOffset = tabStartOffsets[tabIndex];
int tabEndOffset = (tabIndex < tabStartOffsets.length - 1) ?
tabStartOffsets[tabIndex + 1] : p.length();
int[] braceResult = checkForMissingBraces(p, tabStartOffset, tabEndOffset);
if (braceResult[0] != 0) {
JavaProblem problem =
new JavaProblem(braceResult[0] < 0
? "Found one too many } characters without { to match it."
: "Found one too many { characters without } to match it.",
JavaProblem.ERROR, tabIndex, braceResult[1]);
problem.setPDEOffsets(braceResult[3], braceResult[3] + 1);
problems.add(problem);
}
}
return problems;
}
// TODO: move this to a better place when JavaBuild starts using JDT and we
// don't need to check errors at two different places [jv 2017-09-19]
/**
* Checks a single code fragment (such as a tab) for non-matching braces.
* Broken out to allow easy use in JavaBuild.

10
java/src/processing/mode/java/preproc/PdePreprocessor.java
View File

@@ -924,9 +924,11 @@ public class PdePreprocessor {
checkForUnterminatedMultilineComment(program);
if (Preferences.getBoolean("preproc.substitute_unicode")) {
program = substituteUnicode(program);
}
// Removing all the Unicode characters makes detecting and reporting their
// preprocessor errors quite hard.
// if (Preferences.getBoolean("preproc.substitute_unicode")) {
// program = substituteUnicode(program);
// }
// For 0215, adding } as a legitimate prefix to the import (along with
// newline and semicolon) for cases where a tab ends with } and an import
@@ -1478,4 +1480,4 @@ public class PdePreprocessor {
}
return sb.toString();
}
}
}