diff --git a/android/core/src/processing/core/PApplet.java b/android/core/src/processing/core/PApplet.java index a69983eca..4e6706bf0 100644 --- a/android/core/src/processing/core/PApplet.java +++ b/android/core/src/processing/core/PApplet.java @@ -4151,9 +4151,23 @@ public class PApplet extends Activity implements PConstants, Runnable { // DATA I/O + /** + * @webref input:files + * @param filename name of a file in the data folder or a URL. + * @see XML#parse(String) + * @see PApplet#loadBytes(String) + * @see PApplet#loadStrings(String) + * @see PApplet#loadTable(String) + */ public XML loadXML(String filename) { + return loadXML(filename, null); + } + + + // version that uses 'options' though there are currently no supported options + public XML loadXML(String filename, String options) { try { - return new XML(this, filename); + return new XML(createInput(filename), options); } catch (Exception e) { e.printStackTrace(); return null; @@ -4161,9 +4175,14 @@ public class PApplet extends Activity implements PConstants, Runnable { } - static public XML loadXML(File file) { + public XML parseXML(String xmlString) { + return parseXML(xmlString, null); + } + + + public XML parseXML(String xmlString, String options) { try { - return new XML(file); + return XML.parse(xmlString, options); } catch (Exception e) { e.printStackTrace(); return null; @@ -4171,17 +4190,80 @@ public class PApplet extends Activity implements PConstants, Runnable { } + public boolean saveXML(XML xml, String filename) { + return saveXML(xml, filename, null); + } + + + public boolean saveXML(XML xml, String filename, String options) { + return xml.save(saveFile(filename), options); + } + + + public Table createTable() { + return new Table(); + } + + + /** + * @webref input:files + * @param filename name of a file in the data folder or a URL. + * @see PApplet#loadBytes(String) + * @see PApplet#loadStrings(String) + * @see PApplet#loadXML(String) + */ public Table loadTable(String filename) { - return new Table(this, filename); + return loadTable(filename, null); } - static public Table loadTable(File file) { - return new Table(file); + public Table loadTable(String filename, String options) { + try { + String ext = checkExtension(filename); + if (ext != null) { + if (ext.equals("csv") || ext.equals("tsv")) { + if (options == null) { + options = ext; + } else { + options = ext + "," + options; + } + } + } + return new Table(createInput(filename), options); + + } catch (IOException e) { + e.printStackTrace(); + return null; + } } - ////////////////////////////////////////////////////////////// + public boolean saveTable(Table table, String filename) { + return saveTable(table, filename, null); + } + + + public boolean saveTable(Table table, String filename, String options) { + try { + table.save(saveFile(filename), options); + return true; + } catch (IOException e) { + e.printStackTrace(); + } + return false; + } + + + protected String checkExtension(String filename) { + int index = filename.lastIndexOf('.'); + if (index == -1) { + return null; + } + return filename.substring(index + 1).toLowerCase(); + } + + + // FONT I/O diff --git a/android/core/src/processing/core/PGraphicsAndroid2D.java b/android/core/src/processing/core/PGraphicsAndroid2D.java index ac342bc5b..d1ebc4696 100644 --- a/android/core/src/processing/core/PGraphicsAndroid2D.java +++ b/android/core/src/processing/core/PGraphicsAndroid2D.java @@ -1081,7 +1081,7 @@ public class PGraphicsAndroid2D extends PGraphics { } else if (extension.equals("svgz")) { try { InputStream input = new GZIPInputStream(parent.createInput(filename)); - XML xml = new XML(PApplet.createReader(input)); + XML xml = new XML(input); svg = new PShapeSVG(xml); } catch (Exception e) { e.printStackTrace(); diff --git a/android/core/src/processing/data/Table.java b/android/core/src/processing/data/Table.java index 344e4271b..339f3e49d 100644 --- a/android/core/src/processing/data/Table.java +++ b/android/core/src/processing/data/Table.java @@ -24,8 +24,11 @@ package processing.data; import java.io.*; -import java.lang.reflect.Array; -import java.sql.*; +import java.lang.reflect.*; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; import java.util.*; import processing.core.PApplet; @@ -50,6 +53,7 @@ import processing.core.PConstants; //

By default, empty rows are skipped and so are lines that start with the // # character. Using # at the beginning of a line indicates a comment.

+// attempt at a CSV spec: http://tools.ietf.org/html/rfc4180 /** *

Generic class for handling tabular data, typically from a CSV, TSV, or @@ -59,14 +63,17 @@ import processing.core.PConstants; * often with the data in quotes. TSV files use tabs as separators, and usually * don't bother with the quotes.

*

File names should end with .csv if they're comma separated.

+ * + * @webref data:composite */ -public class Table implements Iterable { +public class Table { protected int rowCount; // protected boolean skipEmptyRows = true; // protected boolean skipCommentLines = true; - protected boolean commaSeparatedValues = false; - protected boolean awfulCSV = false; +// protected String extension = null; +// protected boolean commaSeparatedValues = false; +// protected boolean awfulCSV = false; protected String missingString = null; protected int missingInt = 0; @@ -79,78 +86,63 @@ public class Table implements Iterable { HashMapBlows[] columnCategories; HashMap columnIndices; -// static final int TSV = 1; -// static final int CSV = 2; -// static final int AWFUL_CSV = 3; - -// boolean typed; - - // untyped data -// protected String[][] data; -// protected Object[] data; // [row][column] protected Object[] columns; // [column] - // typed data static final int STRING = 0; static final int INT = 1; static final int LONG = 2; static final int FLOAT = 3; static final int DOUBLE = 4; static final int CATEGORICAL = 5; -// static final int TIME = 5; int[] columnTypes; -// int[][] intData; // [column][row] -// long[][] longData; -// float[][] floatData; -// double[][] doubleData; -// Object[][] objectData; + protected RowIterator rowIterator; /** * Creates a new, empty table. Use addRow() to add additional rows. */ public Table() { - columns = new Object[0]; - columnTypes = new int[0]; - columnCategories = new HashMapBlows[0]; + init(); } - public Table(File file) { - this(PApplet.createReader(file)); + public Table(File file) throws IOException { + this(file, null); + } + + + // version that uses a File object; future releases (or data types) + // may include additional optimizations here + public Table(File file, String options) throws IOException { + parse(new FileInputStream(file), checkOptions(file, options)); + } + + + public Table(InputStream input) throws IOException { + this(input, null); } /** - * Can handle TSV or CSV files. - * @param parent - * @param filename + * Read the table from a stream. Possible options include: + *
    + *
  • csv - parse the table as comma-separated values + *
  • tsv - parse the table as tab-separated values + *
  • newlines - this CSV file contains newlines inside individual cells + *
  • header - this table has a header (title) row + *
+ * @param input + * @param options + * @throws IOException */ - public Table(PApplet parent, String filename) { - this(parent.createReader(filename)); - } - - - public Table(BufferedReader reader) { - columns = new Object[0]; - columnTypes = new int[0]; - columnCategories = new HashMapBlows[0]; - try { - boolean csv = peekCSV(reader); - if (csv) { - parseCSV(reader); - } else { - parseTSV(reader); - } - } catch (IOException e) { - e.printStackTrace(); - } + public Table(InputStream input, String options) throws IOException { + parse(input, options); } public Table(ResultSet rs) { - this(); + init(); try { ResultSetMetaData rsmd = rs.getMetaData(); @@ -207,97 +199,97 @@ public class Table implements Iterable { } - /** - * Guess whether this file is tab separated or comma separated by checking - * whether there are more tabs or commas in the first 100 characters. - */ - protected boolean peekCSV(BufferedReader reader) throws IOException { - char[] buffer = new char[100]; - int remaining = buffer.length; - reader.mark(remaining); -// int count = 0; - int commas = 0; - int tabs = 0; - for (int i = 0; i < remaining; i++) { - int c = reader.read(); - if (c == -1) break; - if (c == ',') commas++; - if (c == '\t') tabs++; - } - reader.reset(); - return (commas > tabs); + protected void init() { + columns = new Object[0]; + columnTypes = new int[0]; + columnCategories = new HashMapBlows[0]; } - public void parse(BufferedReader reader) throws IOException { - if (commaSeparatedValues) { - if (awfulCSV) { - parseAwfulCSV(reader); - } else { - parseCSV(reader); + protected String checkOptions(File file, String options) throws IOException { + String extension = null; + String filename = file.getName(); + int dotIndex = filename.lastIndexOf('.'); + if (dotIndex != -1) { + extension = filename.substring(dotIndex + 1).toLowerCase(); + if (!extension.equals("csv") && + !extension.equals("tsv")) { + // ignore extension + extension = null; } - } else { - parseTSV(reader); + } + if (extension == null) { + if (options == null) { + throw new IOException("This table filename has no extension, and no options are set."); + } + } else { // extension is not null + if (options == null) { + options = extension; + } else { + // prepend the extension, it will be overridden if there's an option for it. + options = extension + "," + options; + } + } + return options; + } + + + protected void parse(InputStream input, String options) throws IOException { + init(); + + boolean awfulCSV = false; + boolean header = false; + String extension = null; + if (options != null) { + String[] opts = PApplet.splitTokens(options, " ,"); + for (String opt : opts) { + if (opt.equals("tsv")) { + extension = "tsv"; + } else if (opt.equals("csv")) { + extension = "csv"; + } else if (opt.equals("newlines")) { + awfulCSV = true; + } else if (opt.equals("header")) { + header = true; + } else { + throw new IllegalArgumentException("'" + opt + "' is not a valid option for loading a Table"); + } + } + } + + BufferedReader reader = PApplet.createReader(input); + if (awfulCSV) { + parseAwfulCSV(reader, header); + } else if ("tsv".equals(extension)) { + parseBasic(reader, header, true); + } else if ("csv".equals(extension)) { + parseBasic(reader, header, false); } } - public void parseTSV(BufferedReader reader) throws IOException { - parseBasic(reader, true); -// String line = null; -// int row = 0; -// if (rowCount == 0) { -// setRowCount(10); -// } -// while ((line = reader.readLine()) != null) { -// if (row == getRowCount()) { -// setRowCount(row << 1); -// } -// setRow(row, PApplet.split(line, '\t')); -// row++; -// } -// // shorten or lengthen based on what's left -// if (row != getRowCount()) { -// setRowCount(row); -// } - } - - - public void parseCSV(BufferedReader reader) throws IOException { - parseBasic(reader, false); -// String line = null; -// int row = 0; -// if (rowCount == 0) { -// setRowCount(10); -// } -// while ((line = reader.readLine()) != null) { -// if (row == getRowCount()) { -// setRowCount(row << 1); -// } -// setRow(row, splitLineCSV(line)); -// row++; -// } -// // shorten or lengthen based on what's left -// if (row != getRowCount()) { -// setRowCount(row); -// } - } - - - protected void parseBasic(BufferedReader reader, boolean tsv) throws IOException { + protected void parseBasic(BufferedReader reader, + boolean header, boolean tsv) throws IOException { String line = null; int row = 0; if (rowCount == 0) { setRowCount(10); } - int prev = 0; //-1; + //int prev = 0; //-1; while ((line = reader.readLine()) != null) { if (row == getRowCount()) { setRowCount(row << 1); } - setRow(row, tsv ? PApplet.split(line, '\t') : splitLineCSV(line)); - row++; + if (row == 0 && header) { + setColumnTitles(tsv ? PApplet.split(line, '\t') : splitLineCSV(line)); + header = false; + } else { + setRow(row, tsv ? PApplet.split(line, '\t') : splitLineCSV(line)); + row++; + } + /* + // this is problematic unless we're going to calculate rowCount first if (row % 10000 == 0) { if (row < rowCount) { int pct = (100 * row) / rowCount; @@ -312,6 +304,7 @@ public class Table implements Iterable { e.printStackTrace(); } } + */ } // shorten or lengthen based on what's left if (row != getRowCount()) { @@ -320,86 +313,13 @@ public class Table implements Iterable { } - public void convertTSV(BufferedReader reader, File outputFile) throws IOException { - convertBasic(reader, true, outputFile); - } +// public void convertTSV(BufferedReader reader, File outputFile) throws IOException { +// convertBasic(reader, true, outputFile); +// } - protected void convertBasic(BufferedReader reader, boolean tsv, - File outputFile) throws IOException { - FileOutputStream fos = new FileOutputStream(outputFile); - BufferedOutputStream bos = new BufferedOutputStream(fos, 16384); - DataOutputStream output = new DataOutputStream(bos); - output.writeInt(0); // come back for row count - output.writeInt(getColumnCount()); - if (columnTitles != null) { - output.writeBoolean(true); - for (String title : columnTitles) { - output.writeUTF(title); - } - } else { - output.writeBoolean(false); - } - for (int type : columnTypes) { - output.writeInt(type); - } - - String line = null; - //setRowCount(1); - int prev = -1; - int row = 0; - while ((line = reader.readLine()) != null) { - convertRow(output, tsv ? PApplet.split(line, '\t') : splitLineCSV(line)); - row++; - - if (row % 10000 == 0) { - if (row < rowCount) { - int pct = (100 * row) / rowCount; - if (pct != prev) { - System.out.println(pct + "%"); - prev = pct; - } - } -// try { -// Thread.sleep(5); -// } catch (InterruptedException e) { -// e.printStackTrace(); -// } - } - } - // shorten or lengthen based on what's left -// if (row != getRowCount()) { -// setRowCount(row); -// } - - // has to come afterwards, since these tables get built out during the conversion - int col = 0; - for (HashMapBlows hmb : columnCategories) { - if (hmb == null) { - output.writeInt(0); - } else { - hmb.write(output); - hmb.writeln(PApplet.createWriter(new File(columnTitles[col] + ".categories"))); -// output.writeInt(hmb.size()); -// for (Map.Entry e : hmb.entrySet()) { -// output.writeUTF(e.getKey()); -// output.writeInt(e.getValue()); -// } - } - col++; - } - - output.flush(); - output.close(); - - // come back and write the row count - RandomAccessFile raf = new RandomAccessFile(outputFile, "rw"); - raf.writeInt(rowCount); - raf.close(); - } - - - public void parseAwfulCSV(BufferedReader reader) throws IOException { + protected void parseAwfulCSV(BufferedReader reader, + boolean header) throws IOException { char[] c = new char[100]; int count = 0; boolean insideQuote = false; @@ -438,20 +358,23 @@ public class Table implements Iterable { if (ch == '\"') { insideQuote = true; - } else if (ch == '\r') { - // check to see if next is a '\n' - reader.mark(1); - if (reader.read() != '\n') { - reader.reset(); + } else if (ch == '\r' || ch == '\n') { + if (ch == '\r') { + // check to see if next is a '\n' + reader.mark(1); + if (reader.read() != '\n') { + reader.reset(); + } } setString(row, col, new String(c, 0, count)); count = 0; - row++; - col = 0; - - } else if (ch == '\n') { - setString(row, col, new String(c, 0, count)); - count = 0; + if (row == 0 && header) { + // Use internal row removal (efficient because only one row). + removeTitleRow(); + // Un-set the header variable so that next time around, we don't + // just get stuck into a loop, removing the 0th row repeatedly. + header = false; + } row++; col = 0; @@ -477,11 +400,6 @@ public class Table implements Iterable { } - protected String[] splitLine(String line) { - return commaSeparatedValues ? splitLineCSV(line) : PApplet.split(line, '\t'); - } - - /** * Parse a line of text as comma-separated values, returning each value as * one entry in an array of String objects. Remove quotes from entries that @@ -489,7 +407,7 @@ public class Table implements Iterable { * @param line line of text to be parsed * @return an array of the individual values formerly separated by commas */ - static public String[] splitLineCSV(String line) { + static protected String[] splitLineCSV(String line) { char[] c = line.toCharArray(); int rough = 1; // at least one boolean quote = false; @@ -535,7 +453,7 @@ public class Table implements Iterable { } - static int nextComma(char[] c, int index) { + static protected int nextComma(char[] c, int index) { boolean quote = false; for (int i = index; i < c.length; i++) { if (!quote && (c[i] == ',')) { @@ -548,7 +466,195 @@ public class Table implements Iterable { } - public void writeTSV(PrintWriter writer) { + // A 'Class' object is used here, so the syntax for this function is: + // Table t = loadTable("cars3.tsv", "header"); + // Record[] records = (Record[]) t.parse(Record.class); + // While t.parse("Record") might be nicer, the class is likely to be an + // inner class (another tab in a PDE sketch) or even inside a package, + // so additional information would be needed to locate it. The name of the + // inner class would be "SketchName$Record" which isn't acceptable syntax + // to make people use. Better to just introduce the '.class' syntax. + + // Unlike the Table class itself, this accepts char and boolean fields in + // the target class, since they're much more prevalent, and don't require + // a zillion extra methods and special cases in the rest of the class here. + + // since this is likely an inner class, needs a reference to its parent, + // because that's passed to the constructor parameter (inserted by the + // compiler) of an inner class by the runtime. + + /** incomplete, do not use */ + public void parseInto(Object enclosingObject, String fieldName) { + Class target = null; + Object outgoing = null; + Field targetField = null; + try { + // Object targetObject, + // Class target -> get this from the type of fieldName +// Class sketchClass = sketch.getClass(); + Class sketchClass = enclosingObject.getClass(); + targetField = sketchClass.getDeclaredField(fieldName); +// PApplet.println("found " + targetField); + Class targetArray = targetField.getType(); + if (!targetArray.isArray()) { + // fieldName is not an array + } else { + target = targetArray.getComponentType(); + outgoing = Array.newInstance(target, getRowCount()); + } + } catch (NoSuchFieldException e) { + e.printStackTrace(); + } catch (SecurityException e) { + e.printStackTrace(); + } + +// Object enclosingObject = sketch; +// PApplet.println("enclosing obj is " + enclosingObject); + Class enclosingClass = target.getEnclosingClass(); + Constructor con = null; + + try { + if (enclosingClass == null) { + con = target.getDeclaredConstructor(); //new Class[] { }); +// PApplet.println("no enclosing class"); + } else { + con = target.getDeclaredConstructor(new Class[] { enclosingClass }); +// PApplet.println("enclosed by " + enclosingClass.getName()); + } + if (!con.isAccessible()) { +// System.out.println("setting constructor to public"); + con.setAccessible(true); + } + } catch (SecurityException e) { + e.printStackTrace(); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } + + Field[] fields = target.getDeclaredFields(); + ArrayList inuse = new ArrayList(); + for (Field field : fields) { + String name = field.getName(); + if (getColumnIndex(name, false) != -1) { +// System.out.println("found field " + name); + if (!field.isAccessible()) { +// PApplet.println(" changing field access"); + field.setAccessible(true); + } + inuse.add(field); + } else { +// System.out.println("skipping field " + name); + } + } + + int index = 0; + try { + for (TableRow row : rows()) { + Object item = null; + if (enclosingClass == null) { + //item = target.newInstance(); + item = con.newInstance(); + } else { + item = con.newInstance(new Object[] { enclosingObject }); + } + //Object item = defaultCons.newInstance(new Object[] { }); + for (Field field : inuse) { + String name = field.getName(); + //PApplet.println("gonna set field " + name); + + if (field.getType() == String.class) { + field.set(item, row.getString(name)); + + } else if (field.getType() == Integer.TYPE) { + field.setInt(item, row.getInt(name)); + + } else if (field.getType() == Long.TYPE) { + field.setLong(item, row.getLong(name)); + + } else if (field.getType() == Float.TYPE) { + field.setFloat(item, row.getFloat(name)); + + } else if (field.getType() == Double.TYPE) { + field.setDouble(item, row.getDouble(name)); + + } else if (field.getType() == Boolean.TYPE) { + String content = row.getString(name); + if (content != null) { + // Only bother setting if it's true, + // otherwise false by default anyway. + if (content.toLowerCase().equals("true") || + content.equals("1")) { + field.setBoolean(item, true); + } + } +// if (content == null) { +// field.setBoolean(item, false); // necessary? +// } else if (content.toLowerCase().equals("true")) { +// field.setBoolean(item, true); +// } else if (content.equals("1")) { +// field.setBoolean(item, true); +// } else { +// field.setBoolean(item, false); // necessary? +// } + } else if (field.getType() == Character.TYPE) { + String content = row.getString(name); + if (content != null && content.length() > 0) { + // Otherwise set to \0 anyway + field.setChar(item, content.charAt(0)); + } + } + } +// list.add(item); + Array.set(outgoing, index++, item); + } + if (!targetField.isAccessible()) { +// PApplet.println("setting target field to public"); + targetField.setAccessible(true); + } + // Set the array in the sketch +// targetField.set(sketch, outgoing); + targetField.set(enclosingObject, outgoing); + + } catch (InstantiationException e) { + e.printStackTrace(); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } + } + + + public void save(File file, String options) throws IOException { + save(new FileOutputStream(file), checkOptions(file, options)); + } + + + public void save(OutputStream output, String options) { + PrintWriter writer = PApplet.createWriter(output); + if (options != null) { + String[] opts = PApplet.splitTokens(options, ", "); + for (String opt : opts) { + if (opt.equals("csv")) { + writeCSV(writer); + } else if (opt.equals("tsv")) { + writeTSV(writer); + } else if (opt.equals("html")) { + writeHTML(writer); + } else { + throw new IllegalArgumentException("'" + opt + "' not understood. " + + "Only csv, tsv, and html are " + + "accepted as save parameters"); + } + } + } + writer.close(); + } + + + protected void writeTSV(PrintWriter writer) { if (columnTitles != null) { for (int col = 0; col < columns.length; col++) { if (col != 0) { @@ -578,7 +684,7 @@ public class Table implements Iterable { } - public void writeCSV(PrintWriter writer) { + protected void writeCSV(PrintWriter writer) { if (columnTitles != null) { for (int col = 0; col < columns.length; col++) { if (col != 0) { @@ -647,21 +753,29 @@ public class Table implements Iterable { } - public void writeHTML(PrintWriter writer) { - writer.println(""); + protected void writeHTML(PrintWriter writer) { + writer.println(""); + + writer.println(""); + writer.println(" "); + writer.println(""); + + writer.println(""); + writer.println("
"); for (int row = 0; row < getRowCount(); row++) { - writer.println(" "); + writer.println(" "); for (int col = 0; col < getColumnCount(); col++) { String entry = getString(row, col); - writer.print(" "); - writer.println(""); + writer.println(" "); } - writer.println(" "); + writer.println(" "); } - writer.println("
"); + writer.print(" "); writeEntryHTML(writer, entry); -// String clean = (entry == null) ? "" : HTMLFairy.encodeEntities(entry); -// writer.println(" " + clean + "
"); + writer.println(" "); + writer.println(""); + + writer.println(""); writer.flush(); } @@ -669,10 +783,23 @@ public class Table implements Iterable { protected void writeEntryHTML(PrintWriter writer, String entry) { //char[] chars = entry.toCharArray(); for (char c : entry.toCharArray()) { //chars) { - if (c < 32 || c > 127) { - writer.print("&#"); - writer.print((int) c); - writer.print(';'); + if (c == '<') { + writer.print("<"); + } else if (c == '>') { + writer.print(">"); + } else if (c == '&') { + writer.print("&"); + } else if (c == '\'') { + writer.print("'"); + } else if (c == '"') { + writer.print("""); + + // not necessary with UTF-8? +// } else if (c < 32 || c > 127) { +// writer.print("&#"); +// writer.print((int) c); +// writer.print(';'); + } else { writer.print(c); } @@ -680,34 +807,6 @@ public class Table implements Iterable { } - /** - * Write this table as a TSV file. - * Exceptions will be printed, but not thrown. - * @param file the location to write to. - * @return true if written successfully - */ - public boolean writeCSV(File file) { - try { - writeCSV(new PrintWriter(new FileWriter(file))); - } catch (IOException e) { - e.printStackTrace(); - return false; - } - return true; - } - - - public boolean writeTSV(File file) { - try { - writeTSV(new PrintWriter(new FileWriter(file))); - } catch (IOException e) { - e.printStackTrace(); - return false; - } - return true; - } - - // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . @@ -775,15 +874,15 @@ public class Table implements Iterable { } - public void removeColumn(String dead) { - removeColumn(getColumnIndex(dead)); + public void removeColumn(String columnName) { + removeColumn(getColumnIndex(columnName)); } - public void removeColumn(int index) { + public void removeColumn(int column) { Object[] temp = new Object[columns.length + 1]; - System.arraycopy(columns, 0, temp, 0, index); - System.arraycopy(columns, index+1, temp, index, (columns.length - index) + 1); + System.arraycopy(columns, 0, temp, 0, column); + System.arraycopy(columns, column+1, temp, column, (columns.length - column) + 1); columns = temp; } @@ -940,7 +1039,7 @@ public class Table implements Iterable { * Set the entire table to a specific data type. */ public void setTableType(String type) { - for (int col = 0; col < columns.length; col++) { + for (int col = 0; col < getColumnCount(); col++) { setColumnType(col, type); } } @@ -948,14 +1047,23 @@ public class Table implements Iterable { /** * Set the titles (and if a second column is present) the data types for - * this table based on a file loaded separately. + * this table based on a file loaded separately. This will look for the + * title in column 0, and the type in column 1. Better yet, specify a + * column named "title" and another named "type" in the dictionary table + * to future-proof the code. * @param dictionary */ public void setColumnTypes(Table dictionary) { - setColumnTitles(dictionary.getStringColumn(0)); + int titleCol = 0; + int typeCol = 1; + if (dictionary.hasColumnTitles()) { + titleCol = dictionary.getColumnIndex("title", true); + typeCol = dictionary.getColumnIndex("type", true); + } + setColumnTitles(dictionary.getStringColumn(titleCol)); if (dictionary.getColumnCount() > 1) { for (int i = 0; i < dictionary.getRowCount(); i++) { - setColumnType(i, dictionary.getString(i, 1)); + setColumnType(i, dictionary.getString(i, typeCol)); } } } @@ -966,7 +1074,9 @@ public class Table implements Iterable { /** * Remove the first row from the data set, and use it as the column titles. + * Use loadTable("table.csv", "header") instead. */ + @Deprecated public String[] removeTitleRow() { String[] titles = getStringRow(0); removeRow(0); @@ -994,6 +1104,11 @@ public class Table implements Iterable { } + public boolean hasColumnTitles() { + return columnTitles != null; + } + + public String[] getColumnTitles() { return columnTitles; } @@ -1004,21 +1119,21 @@ public class Table implements Iterable { } - public int getColumnIndex(String name) { - return getColumnIndex(name, true); + public int getColumnIndex(String columnName) { + return getColumnIndex(columnName, true); } /** * Get the index of a column. * @param name Name of the column. - * @param report Whether to print to System.err if the column wasn't found. + * @param report Whether to throw an exception if the column wasn't found. * @return index of the found column, or -1 if not found. */ protected int getColumnIndex(String name, boolean report) { if (columnTitles == null) { if (report) { - System.err.println("Can't get column indices because no column titles are set."); + throw new IllegalArgumentException("This table has no header, so no column titles are set."); } return -1; } @@ -1033,7 +1148,9 @@ public class Table implements Iterable { Integer index = columnIndices.get(name); if (index == null) { if (report) { - System.err.println("No column named '" + name + "' was found."); + // Throws an exception here because the name is known and therefore most useful. + // (Rather than waiting for it to fail inside, say, getInt()) + throw new IllegalArgumentException("This table has no column named '" + name + "'"); } return -1; } @@ -1067,6 +1184,11 @@ public class Table implements Iterable { } + public int lastRowIndex() { + return getRowCount() - 1; + } + + public void setRowCount(int newCount) { if (newCount != rowCount) { if (newCount > 1000000) { @@ -1099,17 +1221,19 @@ public class Table implements Iterable { } - public void addRow() { + public TableRow addRow() { setRowCount(rowCount + 1); + return new RowPointer(this, rowCount - 1); } - public void addRow(String[] columns) { - setRow(getRowCount(), columns); + public TableRow addRow(Object[] columnData) { + setRow(getRowCount(), columnData); + return new RowPointer(this, rowCount - 1); } - public void insertRow(int insert, String[] data) { + public void insertRow(int insert, Object[] columnData) { for (int col = 0; col < columns.length; col++) { switch (columnTypes[col]) { case CATEGORICAL: @@ -1150,12 +1274,12 @@ public class Table implements Iterable { } } } - setRow(insert, data); + setRow(insert, columnData); rowCount++; } - public void removeRow(int dead) { + public void removeRow(int row) { for (int col = 0; col < columns.length; col++) { switch (columnTypes[col]) { case CATEGORICAL: @@ -1164,8 +1288,8 @@ public class Table implements Iterable { // int[] intData = (int[]) columns[col]; // System.arraycopy(intData, 0, intTemp, 0, dead); // System.arraycopy(intData, dead+1, intTemp, dead, (rowCount - dead) + 1); - System.arraycopy(columns[col], 0, intTemp, 0, dead); - System.arraycopy(columns[col], dead+1, intTemp, dead, (rowCount - dead) - 1); + System.arraycopy(columns[col], 0, intTemp, 0, row); + System.arraycopy(columns[col], row+1, intTemp, row, (rowCount - row) - 1); columns[col] = intTemp; break; } @@ -1174,8 +1298,8 @@ public class Table implements Iterable { // long[] longData = (long[]) columns[col]; // System.arraycopy(longData, 0, longTemp, 0, dead); // System.arraycopy(longData, dead+1, longTemp, dead, (rowCount - dead) + 1); - System.arraycopy(columns[col], 0, longTemp, 0, dead); - System.arraycopy(columns[col], dead+1, longTemp, dead, (rowCount - dead) - 1); + System.arraycopy(columns[col], 0, longTemp, 0, row); + System.arraycopy(columns[col], row+1, longTemp, row, (rowCount - row) - 1); columns[col] = longTemp; break; } @@ -1184,8 +1308,8 @@ public class Table implements Iterable { // float[] floatData = (float[]) columns[col]; // System.arraycopy(floatData, 0, floatTemp, 0, dead); // System.arraycopy(floatData, dead+1, floatTemp, dead, (rowCount - dead) + 1); - System.arraycopy(columns[col], 0, floatTemp, 0, dead); - System.arraycopy(columns[col], dead+1, floatTemp, dead, (rowCount - dead) - 1); + System.arraycopy(columns[col], 0, floatTemp, 0, row); + System.arraycopy(columns[col], row+1, floatTemp, row, (rowCount - row) - 1); columns[col] = floatTemp; break; } @@ -1194,15 +1318,15 @@ public class Table implements Iterable { // double[] doubleData = (double[]) columns[col]; // System.arraycopy(doubleData, 0, doubleTemp, 0, dead); // System.arraycopy(doubleData, dead+1, doubleTemp, dead, (rowCount - dead) + 1); - System.arraycopy(columns[col], 0, doubleTemp, 0, dead); - System.arraycopy(columns[col], dead+1, doubleTemp, dead, (rowCount - dead) - 1); + System.arraycopy(columns[col], 0, doubleTemp, 0, row); + System.arraycopy(columns[col], row+1, doubleTemp, row, (rowCount - row) - 1); columns[col] = doubleTemp; break; } case STRING: { String[] stringTemp = new String[rowCount-1]; - System.arraycopy(columns[col], 0, stringTemp, 0, dead); - System.arraycopy(columns[col], dead+1, stringTemp, dead, (rowCount - dead) - 1); + System.arraycopy(columns[col], 0, stringTemp, 0, row); + System.arraycopy(columns[col], row+1, stringTemp, row, (rowCount - row) - 1); columns[col] = stringTemp; } } @@ -1211,6 +1335,7 @@ public class Table implements Iterable { } + /* public void setRow(int row, String[] pieces) { checkSize(row, pieces.length - 1); // pieces.length may be less than columns.length, so loop over pieces @@ -1221,135 +1346,123 @@ public class Table implements Iterable { protected void setRowCol(int row, int col, String piece) { + switch (columnTypes[col]) { + case STRING: + String[] stringData = (String[]) columns[col]; + stringData[row] = piece; + break; + case INT: + int[] intData = (int[]) columns[col]; + intData[row] = PApplet.parseInt(piece, missingInt); + break; + case LONG: + long[] longData = (long[]) columns[col]; + try { + longData[row] = Long.parseLong(piece); + } catch (NumberFormatException nfe) { + longData[row] = missingLong; + } + break; + case FLOAT: + float[] floatData = (float[]) columns[col]; + floatData[row] = PApplet.parseFloat(piece, missingFloat); + break; + case DOUBLE: + double[] doubleData = (double[]) columns[col]; + try { + doubleData[row] = Double.parseDouble(piece); + } catch (NumberFormatException nfe) { + doubleData[row] = missingDouble; + } + break; + case CATEGORICAL: + int[] indexData = (int[]) columns[col]; + indexData[row] = columnCategories[col].index(piece); + break; + default: + throw new IllegalArgumentException("That's not a valid column type."); + } + } + */ + + + public void setRow(int row, Object[] pieces) { + checkSize(row, pieces.length - 1); + // pieces.length may be less than columns.length, so loop over pieces + for (int col = 0; col < pieces.length; col++) { + setRowCol(row, col, pieces[col]); + } + } + + + protected void setRowCol(int row, int col, Object piece) { switch (columnTypes[col]) { case STRING: String[] stringData = (String[]) columns[col]; - stringData[row] = piece; + if (piece == null) { + stringData[row] = null; +// } else if (piece instanceof String) { +// stringData[row] = (String) piece; + } else { + // Calls toString() on the object, which is 'return this' for String + stringData[row] = String.valueOf(piece); + } break; case INT: int[] intData = (int[]) columns[col]; - intData[row] = PApplet.parseInt(piece, missingInt); + //intData[row] = PApplet.parseInt(piece, missingInt); + if (piece == null) { + intData[row] = missingInt; + } else if (piece instanceof Integer) { + intData[row] = (Integer) piece; + } else { + intData[row] = PApplet.parseInt(String.valueOf(piece), missingInt); + } break; case LONG: long[] longData = (long[]) columns[col]; - try { - longData[row] = Long.parseLong(piece); - } catch (NumberFormatException nfe) { + if (piece == null) { longData[row] = missingLong; + } else if (piece instanceof Long) { + longData[row] = (Long) piece; + } else { + try { + longData[row] = Long.parseLong(String.valueOf(piece)); + } catch (NumberFormatException nfe) { + longData[row] = missingLong; + } } break; case FLOAT: float[] floatData = (float[]) columns[col]; - floatData[row] = PApplet.parseFloat(piece, missingFloat); + if (piece == null) { + floatData[row] = missingFloat; + } else if (piece instanceof Float) { + floatData[row] = (Float) piece; + } else { + floatData[row] = PApplet.parseFloat(String.valueOf(piece), missingFloat); + } break; case DOUBLE: double[] doubleData = (double[]) columns[col]; - try { - doubleData[row] = Double.parseDouble(piece); - } catch (NumberFormatException nfe) { + if (piece == null) { doubleData[row] = missingDouble; + } else if (piece instanceof Double) { + doubleData[row] = (Double) piece; + } else { + try { + doubleData[row] = Double.parseDouble(String.valueOf(piece)); + } catch (NumberFormatException nfe) { + doubleData[row] = missingDouble; + } } break; case CATEGORICAL: int[] indexData = (int[]) columns[col]; - indexData[row] = columnCategories[col].index(piece); - break; - default: - throw new IllegalArgumentException("That's not a valid column type."); - } - } - - - public void convertRow(DataOutputStream output, String[] pieces) throws IOException { - if (pieces.length > getColumnCount()) { - throw new IllegalArgumentException("Row with too many columns: " + - PApplet.join(pieces, ",")); - } - // pieces.length may be less than columns.length, so loop over pieces - for (int col = 0; col < pieces.length; col++) { - switch (columnTypes[col]) { - case STRING: - output.writeUTF(pieces[col]); - break; - case INT: - output.writeInt(PApplet.parseInt(pieces[col], missingInt)); - break; - case LONG: - try { - output.writeLong(Long.parseLong(pieces[col])); - } catch (NumberFormatException nfe) { - output.writeLong(missingLong); - } - break; - case FLOAT: - output.writeFloat(PApplet.parseFloat(pieces[col], missingFloat)); - break; - case DOUBLE: - try { - output.writeDouble(Double.parseDouble(pieces[col])); - } catch (NumberFormatException nfe) { - output.writeDouble(missingDouble); - } - break; - case CATEGORICAL: - output.writeInt(columnCategories[col].index(pieces[col])); - break; - } - } - for (int col = pieces.length; col < getColumnCount(); col++) { - switch (columnTypes[col]) { - case STRING: - output.writeUTF(""); - break; - case INT: - output.writeInt(missingInt); - break; - case LONG: - output.writeLong(missingLong); - break; - case FLOAT: - output.writeFloat(missingFloat); - break; - case DOUBLE: - output.writeDouble(missingDouble); - break; - case CATEGORICAL: - output.writeInt(missingCategory); - break; - - } - } - } - - - protected void convertRowCol(DataOutputStream output, int row, int col, String piece) { - switch (columnTypes[col]) { - case STRING: - String[] stringData = (String[]) columns[col]; - stringData[row] = piece; - break; - case INT: - int[] intData = (int[]) columns[col]; - intData[row] = PApplet.parseInt(piece, missingInt); - break; - case LONG: - long[] longData = (long[]) columns[col]; - try { - longData[row] = Long.parseLong(piece); - } catch (NumberFormatException nfe) { - longData[row] = missingLong; - } - break; - case FLOAT: - float[] floatData = (float[]) columns[col]; - floatData[row] = PApplet.parseFloat(piece, missingFloat); - break; - case DOUBLE: - double[] doubleData = (double[]) columns[col]; - try { - doubleData[row] = Double.parseDouble(piece); - } catch (NumberFormatException nfe) { - doubleData[row] = missingDouble; + if (piece == null) { + indexData[row] = missingCategory; + } else { + indexData[row] = columnCategories[col].index(String.valueOf(piece)); } break; default: @@ -1361,90 +1474,155 @@ public class Table implements Iterable { // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . - protected RowIterator rowIterator; + public TableRow getRow(int row) { + return new RowPointer(this, row); + } + /** - * Note that this one iterator instance is shared by any calls to iterate the - * rows of this table. This is very efficient, but not very thread-safe. If - * you want to iterate in a multi-threaded manner, use createIterator(). + * Note that this one iterator instance is shared by any calls to iterate + * the rows of this table. This is very efficient, but not thread-safe. + * If you want to iterate in a multi-threaded manner, don't use the iterator. */ - public Iterator iterator() { - if (rowIterator == null) { - rowIterator = new RowIterator(); - } - rowIterator.reset(); - return rowIterator; - } - - - public Iterator createIterator() { - return new RowIterator(); - } - - - // temporary objects inside loop! garbage collection! argh! -// public Iterator iterator() { -// return new RowIterator(); -// } - - - class RowIterator implements Iterator { - int row; - TableRow tableRow = new TableRow() { - public String getString(int column) { - return Table.this.getString(row, column); - } - - public String getString(String columnName) { - return Table.this.getString(row, columnName); - } - - public int getInt(int column) { - return Table.this.getInt(row, column); - } - - public int getInt(String columnName) { - return Table.this.getInt(row, columnName); - } - - public long getLong(int column) { - return Table.this.getLong(row, column); - } - - public long getLong(String columnName) { - return Table.this.getLong(row, columnName); - } - - public float getFloat(int column) { - return Table.this.getFloat(row, column); - } - - public float getFloat(String columnName) { - return Table.this.getFloat(row, columnName); - } - - public double getDouble(int column) { - return Table.this.getDouble(row, column); - } - - public double getDouble(String columnName) { - return Table.this.getDouble(row, columnName); + public Iterable rows() { + return new Iterable() { + public Iterator iterator() { + if (rowIterator == null) { + rowIterator = new RowIterator(Table.this); + } else { + rowIterator.reset(); + } + return rowIterator; } }; + } + + + public Iterator rows(int[] indices) { + return new RowIndexIterator(this, indices); + } + + + // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + + + static class RowPointer implements TableRow { + Table table; + int row; + + public RowPointer(Table table, int row) { + this.table = table; + this.row = row; + } + + public void setRow(int row) { + this.row = row; + } + + public String getString(int column) { + return table.getString(row, column); + } + + public String getString(String columnName) { + return table.getString(row, columnName); + } + + public int getInt(int column) { + return table.getInt(row, column); + } + + public int getInt(String columnName) { + return table.getInt(row, columnName); + } + + public long getLong(int column) { + return table.getLong(row, column); + } + + public long getLong(String columnName) { + return table.getLong(row, columnName); + } + + public float getFloat(int column) { + return table.getFloat(row, column); + } + + public float getFloat(String columnName) { + return table.getFloat(row, columnName); + } + + public double getDouble(int column) { + return table.getDouble(row, column); + } + + public double getDouble(String columnName) { + return table.getDouble(row, columnName); + } + + public void setString(int column, String value) { + table.setString(row, column, value); + } + + public void setString(String columnName, String value) { + table.setString(row, columnName, value); + } + + public void setInt(int column, int value) { + table.setInt(row, column, value); + } + + public void setInt(String columnName, int value) { + table.setInt(row, columnName, value); + } + + public void setLong(int column, long value) { + table.setLong(row, column, value); + } + + public void setLong(String columnName, long value) { + table.setLong(row, columnName, value); + } + + public void setFloat(int column, float value) { + table.setFloat(row, column, value); + } + + public void setFloat(String columnName, float value) { + table.setFloat(row, columnName, value); + } + + public void setDouble(int column, double value) { + table.setDouble(row, column, value); + } + + public void setDouble(String columnName, double value) { + table.setDouble(row, columnName, value); + } + } + + + static class RowIterator implements Iterator { + Table table; + RowPointer rp; + int row; + + public RowIterator(Table table) { + this.table = table; + row = -1; + rp = new RowPointer(table, row); + } public void remove() { - removeRow(row); + table.removeRow(row); } public TableRow next() { - ++row; -// iteratorRow.setRow(row); -// return iteratorRow; - return tableRow; + rp.setRow(++row); + return rp; } public boolean hasNext() { - return row+1 < getRowCount(); + return row+1 < table.getRowCount(); } public void reset() { @@ -1453,6 +1631,40 @@ public class Table implements Iterable { } + static class RowIndexIterator implements Iterator { + Table table; + RowPointer rp; + int[] indices; + int index; + + public RowIndexIterator(Table table, int[] indices) { + this.table = table; + this.indices = indices; + index = -1; + // just set to something arbitrary + rp = new RowPointer(table, -1); + } + + public void remove() { + table.removeRow(indices[index]); + } + + public TableRow next() { + rp.setRow(indices[++index]); + return rp; + } + + public boolean hasNext() { + //return row+1 < table.getRowCount(); + return index + 1 < indices.length; + } + + public void reset() { + index = -1; + } + } + + static public Iterator createIterator(final ResultSet rs) { return new Iterator() { boolean already; @@ -1558,6 +1770,22 @@ public class Table implements Iterable { throw new RuntimeException(e); } } + + public void setString(int column, String value) { immutable(); } + public void setString(String columnName, String value) { immutable(); } + public void setInt(int column, int value) { immutable(); } + public void setInt(String columnName, int value) { immutable(); } + public void setLong(int column, long value) { immutable(); } + public void setLong(String columnName, long value) { immutable(); } + public void setFloat(int column, float value) { immutable(); } + public void setFloat(String columnName, float value) { immutable(); } + public void setDouble(int column, double value) { immutable(); } + public void setDouble(String columnName, double value) { immutable(); } + + private void immutable() { + throw new IllegalArgumentException("This TableRow cannot be modified."); + } + }; } @@ -1593,9 +1821,9 @@ public class Table implements Iterable { } - public void setInt(int row, int column, int what) { + public void setInt(int row, int column, int value) { if (columnTypes[column] == STRING) { - setString(row, column, String.valueOf(what)); + setString(row, column, String.valueOf(value)); } else { checkSize(row, column); @@ -1603,11 +1831,17 @@ public class Table implements Iterable { throw new IllegalArgumentException("Column " + column + " is not an int column."); } int[] intData = (int[]) columns[column]; - intData[row] = what; + intData[row] = value; } } + public void setInt(int row, String columnName, int value) { + setInt(row, getColumnIndex(columnName), value); + } + + + public int[] getIntColumn(String name) { int col = getColumnIndex(name); return (col == -1) ? null : getIntColumn(col); @@ -1663,9 +1897,9 @@ public class Table implements Iterable { } - public void setLong(int row, int column, long what) { + public void setLong(int row, int column, long value) { if (columnTypes[column] == STRING) { - setString(row, column, String.valueOf(what)); + setString(row, column, String.valueOf(value)); } else { checkSize(row, column); @@ -1673,11 +1907,16 @@ public class Table implements Iterable { throw new IllegalArgumentException("Column " + column + " is not a 'long' column."); } long[] longData = (long[]) columns[column]; - longData[row] = what; + longData[row] = value; } } + public void setLong(int row, String columnName, long value) { + setLong(row, getColumnIndex(columnName), value); + } + + public long[] getLongColumn(String name) { int col = getColumnIndex(name); return (col == -1) ? null : getLongColumn(col); @@ -1734,9 +1973,9 @@ public class Table implements Iterable { } - public void setFloat(int row, int column, float what) { + public void setFloat(int row, int column, float value) { if (columnTypes[column] == STRING) { - setString(row, column, String.valueOf(what)); + setString(row, column, String.valueOf(value)); } else { checkSize(row, column); @@ -1744,11 +1983,16 @@ public class Table implements Iterable { throw new IllegalArgumentException("Column " + column + " is not a float column."); } float[] longData = (float[]) columns[column]; - longData[row] = what; + longData[row] = value; } } + public void setFloat(int row, String columnName, float value) { + setFloat(row, getColumnIndex(columnName), value); + } + + public float[] getFloatColumn(String name) { int col = getColumnIndex(name); return (col == -1) ? null : getFloatColumn(col); @@ -1804,9 +2048,9 @@ public class Table implements Iterable { } - public void setDouble(int row, int column, double what) { + public void setDouble(int row, int column, double value) { if (columnTypes[column] == STRING) { - setString(row, column, String.valueOf(what)); + setString(row, column, String.valueOf(value)); } else { checkSize(row, column); @@ -1814,11 +2058,16 @@ public class Table implements Iterable { throw new IllegalArgumentException("Column " + column + " is not a 'double' column."); } double[] doubleData = (double[]) columns[column]; - doubleData[row] = what; + doubleData[row] = value; } } + public void setDouble(int row, String columnName, double value) { + setDouble(row, getColumnIndex(columnName), value); + } + + public double[] getDoubleColumn(String name) { int col = getColumnIndex(name); return (col == -1) ? null : getDoubleColumn(col); @@ -1906,7 +2155,7 @@ public class Table implements Iterable { * Get a String value from the table. If the row is longer than the table * @param row * @param col - * @return + * @return the String defined by the row and col variables */ public String getString(int row, int col) { checkBounds(row, col); @@ -1932,19 +2181,19 @@ public class Table implements Iterable { } - public void setString(int row, int column, String what) { + public void setString(int row, int column, String value) { checkSize(row, column); if (columnTypes[column] != STRING) { throw new IllegalArgumentException("Column " + column + " is not a String column."); } String[] stringData = (String[]) columns[column]; - stringData[row] = what; + stringData[row] = value; } - public void setString(int row, String columnName, String what) { + public void setString(int row, String columnName, String value) { int column = checkColumnIndex(columnName); - setString(row, column, what); + setString(row, column, value); } @@ -1976,38 +2225,125 @@ public class Table implements Iterable { /** - * Set all 'null' entries to "" (zero length String objects). - * If columns are typed, then this will only apply to String columns. + * Return the row that contains the first String that matches. + * @param value the String to match + * @param column the column to search */ - public void makeNullEmpty() { - for (int col = 0; col < columns.length; col++) { - if (columnTypes[col] == STRING) { - String[] stringData = (String[]) columns[col]; + public int findRowIndex(String value, int column) { + checkBounds(-1, column); + if (columnTypes[column] == STRING) { + String[] stringData = (String[]) columns[column]; + if (value == null) { for (int row = 0; row < rowCount; row++) { - if (stringData[row] == null) { - stringData[row] = ""; + if (stringData[row] == null) return row; + } + } else { + for (int row = 0; row < rowCount; row++) { + if (stringData[row] != null && stringData[row].equals(value)) { + return row; } } } + } else { // less efficient, includes conversion as necessary + for (int row = 0; row < rowCount; row++) { + String str = getString(row, column); + if (str == null) { + if (value == null) { + return row; + } + } else if (str.equals(value)) { + return row; + } + } } + return -1; } /** - * Set all "" entries (zero length String objects) to null values. - * If columns are typed, then this will only apply to String columns. + * Return the row that contains the first String that matches. + * @param value the String to match + * @param columnName the column to search */ - public void makeEmptyNull() { - for (int col = 0; col < columns.length; col++) { - if (columnTypes[col] == STRING) { - String[] stringData = (String[]) columns[col]; + public int findRowIndex(String value, String columnName) { + return findRowIndex(value, getColumnIndex(columnName)); + } + + + /** + * Return a list of rows that contain the String passed in. If there are no + * matches, a zero length array will be returned (not a null array). + * @param value the String to match + * @param column the column to search + */ + public int[] findRowIndices(String value, int column) { + int[] outgoing = new int[rowCount]; + int count = 0; + + checkBounds(-1, column); + if (columnTypes[column] == STRING) { + String[] stringData = (String[]) columns[column]; + if (value == null) { for (int row = 0; row < rowCount; row++) { - if (stringData[row] != null && stringData[row].length() == 0) { - stringData[row] = null; + if (stringData[row] == null) { + outgoing[count++] = row; + } + } + } else { + for (int row = 0; row < rowCount; row++) { + if (stringData[row] != null && stringData[row].equals(value)) { + outgoing[count++] = row; } } } + } else { // less efficient, includes conversion as necessary + for (int row = 0; row < rowCount; row++) { + String str = getString(row, column); + if (str == null) { + if (value == null) { + outgoing[count++] = row; + } + } else if (str.equals(value)) { + outgoing[count++] = row; + } + } } + return PApplet.subset(outgoing, 0, count); + } + + + /** + * Return a list of rows that contain the String passed in. If there are no + * matches, a zero length array will be returned (not a null array). + * @param value the String to match + * @param columnName the column to search + */ + public int[] findRowIndices(String value, String columnName) { + return findRowIndices(value, getColumnIndex(columnName)); + } + + + // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + + + public TableRow findRow(String value, int column) { + int row = findRowIndex(value, column); + return (row == -1) ? null : new RowPointer(this, row); + } + + + public TableRow findRow(String value, String columnName) { + return findRow(value, getColumnIndex(columnName)); + } + + + public Iterator findRows(String value, int column) { + return new RowIndexIterator(this, findRowIndices(value, column)); + } + + + public Iterator findRows(String value, String columnName) { + return findRows(value, getColumnIndex(columnName)); } @@ -2015,26 +2351,177 @@ public class Table implements Iterable { /** - * Searches the entire table for float values. - * Returns missing float (Float.NaN by default) if no valid numbers found. + * Return the row that contains the first String that matches. + * @param regexp the String to match + * @param column the column to search */ - public float getMaxFloat() { - boolean found = false; - float max = PConstants.MIN_FLOAT; - for (int row = 0; row < getRowCount(); row++) { - for (int col = 0; col < getColumnCount(); col++) { - float value = getFloat(row, col); - if (!Float.isNaN(value)) { // TODO no, this should be comparing to the missing value - if (!found) { - max = value; - found = true; - } else if (value > max) { - max = value; - } + public int matchRowIndex(String regexp, int column) { + checkBounds(-1, column); + if (columnTypes[column] == STRING) { + String[] stringData = (String[]) columns[column]; + for (int row = 0; row < rowCount; row++) { + if (stringData[row] != null && + PApplet.match(stringData[row], regexp) != null) { + return row; + } + } + } else { // less efficient, includes conversion as necessary + for (int row = 0; row < rowCount; row++) { + String str = getString(row, column); + if (str != null && + PApplet.match(str, regexp) != null) { + return row; } } } - return found ? max : missingFloat; + return -1; + } + + + /** + * Return the row that contains the first String that matches. + * @param what the String to match + * @param columnName the column to search + */ + public int matchRowIndex(String what, String columnName) { + return matchRowIndex(what, getColumnIndex(columnName)); + } + + + /** + * Return a list of rows that contain the String passed in. If there are no + * matches, a zero length array will be returned (not a null array). + * @param what the String to match + * @param column the column to search + */ + public int[] matchRowIndices(String regexp, int column) { + int[] outgoing = new int[rowCount]; + int count = 0; + + checkBounds(-1, column); + if (columnTypes[column] == STRING) { + String[] stringData = (String[]) columns[column]; + for (int row = 0; row < rowCount; row++) { + if (stringData[row] != null && + PApplet.match(stringData[row], regexp) != null) { + outgoing[count++] = row; + } + } + } else { // less efficient, includes conversion as necessary + for (int row = 0; row < rowCount; row++) { + String str = getString(row, column); + if (str != null && + PApplet.match(str, regexp) != null) { + outgoing[count++] = row; + } + } + } + return PApplet.subset(outgoing, 0, count); + } + + + /** + * Return a list of rows that match the regex passed in. If there are no + * matches, a zero length array will be returned (not a null array). + * @param what the String to match + * @param columnName the column to search + */ + public int[] matchRowIndices(String what, String columnName) { + return matchRowIndices(what, getColumnIndex(columnName)); + } + + + // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + + + public TableRow matchRow(String regexp, int column) { + int row = matchRowIndex(regexp, column); + return (row == -1) ? null : new RowPointer(this, row); + } + + + public TableRow matchRow(String regexp, String columnName) { + return matchRow(regexp, getColumnIndex(columnName)); + } + + + public Iterator matchRows(String value, int column) { + return new RowIndexIterator(this, matchRowIndices(value, column)); + } + + + public Iterator matchRows(String value, String columnName) { + return matchRows(value, getColumnIndex(columnName)); + } + + + // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + + + /** + * Replace a String with another. Set empty entries null by using + * replace("", null) or use replace(null, "") to go the other direction. + * If this is a typed table, only String columns will be modified. + * @param orig + * @param replacement + */ + public void replace(String orig, String replacement) { + for (int col = 0; col < columns.length; col++) { + replace(orig, replacement, col); + } + } + + + public void replace(String orig, String replacement, int col) { + if (columnTypes[col] == STRING) { + String[] stringData = (String[]) columns[col]; + for (int row = 0; row < rowCount; row++) { + if (stringData[row].equals(orig)) { + stringData[row] = replacement; + } + } + } + } + + + public void replace(String orig, String replacement, String colName) { + replace(orig, replacement, getColumnIndex(colName)); + } + + + // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + + + public void replaceAll(String orig, String replacement) { + for (int col = 0; col < columns.length; col++) { + replaceAll(orig, replacement, col); + } + } + + + public void replaceAll(String regex, String replacement, int column) { + checkBounds(-1, column); + if (columnTypes[column] == STRING) { + String[] stringData = (String[]) columns[column]; + for (int row = 0; row < rowCount; row++) { + if (stringData[row] != null) { + stringData[row] = stringData[row].replaceAll(regex, replacement); + } + } + } else { + throw new IllegalArgumentException("replaceAll() can only be used on String columns"); + } + } + + + /** + * Run String.replaceAll() on all entries in a column. + * Only works with columns that are already String values. + * @param what the String to match + * @param columnName the column to search + */ + public void replaceAll(String regex, String replacement, String columnName) { + replaceAll(regex, replacement, getColumnIndex(columnName)); } @@ -2080,236 +2567,35 @@ public class Table implements Iterable { } - public void removeTokens(String tokens, String column) { - removeTokens(tokens, getColumnIndex(column)); - } - - - // TODO this isn't i18n correct, and it's a dumb implementation -// public void removeLetters(int column) { -// String alphabet = "abcdefghijklmnopqrstuvwxyz"; -// removeTokens(alphabet + alphabet.toUpperCase(), column); -// } - - - // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . - - - /** - * Return the row that contains the first String that matches. - * @param what the String to match - * @param column the column to search - */ - public int findRow(String what, int column) { - checkBounds(-1, column); - if (columnTypes[column] == STRING) { - String[] stringData = (String[]) columns[column]; - if (what == null) { - for (int row = 0; row < rowCount; row++) { - if (stringData[row] == null) return row; - } - } else { - for (int row = 0; row < rowCount; row++) { - if (stringData[row] != null && stringData[row].equals(what)) { - return row; - } - } - } - } else { // less efficient, includes conversion as necessary - for (int row = 0; row < rowCount; row++) { - String str = getString(row, column); - if (str == null) { - if (what == null) { - return row; - } - } else if (str.equals(what)) { - return row; - } - } - } - return -1; - } - - - /** - * Return the row that contains the first String that matches. - * @param what the String to match - * @param columnName the column to search - */ - public int findRow(String what, String columnName) { - return findRow(what, getColumnIndex(columnName)); - } - - - /** - * Return a list of rows that contain the String passed in. If there are no - * matches, a zero length array will be returned (not a null array). - * @param what the String to match - * @param column the column to search - */ - public int[] findRows(String what, int column) { - int[] outgoing = new int[rowCount]; - int count = 0; - - checkBounds(-1, column); - if (columnTypes[column] == STRING) { - String[] stringData = (String[]) columns[column]; - if (what == null) { - for (int row = 0; row < rowCount; row++) { - if (stringData[row] == null) { - outgoing[count++] = row; - } - } - } else { - for (int row = 0; row < rowCount; row++) { - if (stringData[row] != null && stringData[row].equals(what)) { - outgoing[count++] = row; - } - } - } - } else { // less efficient, includes conversion as necessary - for (int row = 0; row < rowCount; row++) { - String str = getString(row, column); - if (str == null) { - if (what == null) { - outgoing[count++] = row; - } - } else if (str.equals(what)) { - outgoing[count++] = row; - } - } - } - return PApplet.subset(outgoing, 0, count); - } - - - /** - * Return a list of rows that contain the String passed in. If there are no - * matches, a zero length array will be returned (not a null array). - * @param what the String to match - * @param columnName the column to search - */ - public int[] findRows(String what, String columnName) { - return findRows(what, getColumnIndex(columnName)); + public void removeTokens(String tokens, String columnName) { + removeTokens(tokens, getColumnIndex(columnName)); } // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . - /** - * Return the row that contains the first String that matches. - * @param what the String to match - * @param column the column to search - */ - public int matchRow(String regexp, int column) { - checkBounds(-1, column); - if (columnTypes[column] == STRING) { - String[] stringData = (String[]) columns[column]; - for (int row = 0; row < rowCount; row++) { - if (stringData[row] != null && - PApplet.match(stringData[row], regexp) != null) { - return row; - } - } - } else { // less efficient, includes conversion as necessary - for (int row = 0; row < rowCount; row++) { - String str = getString(row, column); - if (str != null && - PApplet.match(str, regexp) != null) { - return row; - } - } + public void trim() { + for (int col = 0; col < getColumnCount(); col++) { + trim(col); } - return -1; } - /** - * Return the row that contains the first String that matches. - * @param what the String to match - * @param columnName the column to search - */ - public int matchRow(String what, String columnName) { - return matchRow(what, getColumnIndex(columnName)); - } - - - /** - * Return a list of rows that contain the String passed in. If there are no - * matches, a zero length array will be returned (not a null array). - * @param what the String to match - * @param column the column to search - */ - public int[] matchRows(String regexp, int column) { - int[] outgoing = new int[rowCount]; - int count = 0; - - checkBounds(-1, column); - if (columnTypes[column] == STRING) { - String[] stringData = (String[]) columns[column]; - for (int row = 0; row < rowCount; row++) { - if (stringData[row] != null && - PApplet.match(stringData[row], regexp) != null) { - outgoing[count++] = row; - } - } - } else { // less efficient, includes conversion as necessary - for (int row = 0; row < rowCount; row++) { - String str = getString(row, column); - if (str != null && - PApplet.match(str, regexp) != null) { - outgoing[count++] = row; - } - } - } - return PApplet.subset(outgoing, 0, count); - } - - - /** - * Return a list of rows that match the regex passed in. If there are no - * matches, a zero length array will be returned (not a null array). - * @param what the String to match - * @param columnName the column to search - */ - public int[] matchRows(String what, String columnName) { - return matchRows(what, getColumnIndex(columnName)); - } - - - // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . - - - /** - * Return a list of rows that contain the String passed in. If there are no - * matches, a zero length array will be returned (not a null array). - * @param what the String to match - * @param column the column to search - */ - public void replaceAll(String regex, String replacement, int column) { - checkBounds(-1, column); + public void trim(int column) { if (columnTypes[column] == STRING) { String[] stringData = (String[]) columns[column]; for (int row = 0; row < rowCount; row++) { if (stringData[row] != null) { - stringData[row] = stringData[row].replaceAll(regex, replacement); + stringData[row] = PApplet.trim(stringData[row]); } } - } else { - throw new IllegalArgumentException("replaceAll() can only be used on String columns"); } } - /** - * Run String.replaceAll() on all entries in a column. - * Only works with columns that are already String values. - * @param what the String to match - * @param columnName the column to search - */ - public void replaceAll(String regex, String replacement, String columnName) { - replaceAll(regex, replacement, getColumnIndex(columnName)); + public void trim(String columnName) { + trim(getColumnIndex(columnName)); } @@ -2349,31 +2635,6 @@ public class Table implements Iterable { // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . - public Table createSubset(int[] rowSubset) { - Table newbie = new Table(); - newbie.setColumnTitles(columnTitles); // also sets columns.length - newbie.columnTypes = columnTypes; - newbie.setRowCount(rowSubset.length); - - for (int i = 0; i < rowSubset.length; i++) { - int row = rowSubset[i]; - for (int col = 0; col < columns.length; col++) { - switch (columnTypes[col]) { - case STRING: newbie.setString(i, col, getString(row, col)); break; - case INT: newbie.setInt(i, col, getInt(row, col)); break; - case LONG: newbie.setLong(i, col, getLong(row, col)); break; - case FLOAT: newbie.setFloat(i, col, getFloat(row, col)); break; - case DOUBLE: newbie.setDouble(i, col, getDouble(row, col)); break; - } - } - } - return newbie; - } - - - // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . - - class HashMapBlows { HashMap dataToIndex = new HashMap(); ArrayList indexToData = new ArrayList(); @@ -2424,19 +2685,8 @@ public class Table implements Iterable { } } -// class HashMapBlows extends HashMap { -// -// int index(String what) { -// Integer value = get(what); -// if (value != null) { -// return value; -// } -// -// int v = size(); -// put(what, v); -// return v; -// } -// } + + // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . class HashMapSucks extends HashMap { @@ -2458,12 +2708,15 @@ public class Table implements Iterable { } - public String[] getUnique(String column) { + // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + + + protected String[] getUnique(String column) { return getUnique(getColumnIndex(column)); } - public String[] getUnique(int column) { + protected String[] getUnique(int column) { HashMapSucks found = new HashMapSucks(); for (int row = 0; row < getRowCount(); row++) { found.check(getString(row, column)); @@ -2474,12 +2727,12 @@ public class Table implements Iterable { } - public HashMap getUniqueCount(String columnName) { + protected HashMap getUniqueCount(String columnName) { return getUniqueCount(getColumnIndex(columnName)); } - public HashMap getUniqueCount(int column) { + protected HashMap getUniqueCount(int column) { HashMapSucks outgoing = new HashMapSucks(); for (int row = 0; row < rowCount; row++) { String entry = getString(row, column); @@ -2497,7 +2750,7 @@ public class Table implements Iterable { * found in the first column, getColumnRowLookup(0) would return an object * that would map each name back to its row. */ - public HashMap getRowLookup(int col) { + protected HashMap getRowLookup(int col) { HashMap outgoing = new HashMap(); for (int row = 0; row < getRowCount(); row++) { outgoing.put(getString(row, col), row); @@ -2606,14 +2859,230 @@ public class Table implements Iterable { // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . - public void trim() { - for (int col = 0; col < columns.length; col++) { - String[] stringData = (String[]) columns[col]; - for (int row = 0; row < rowCount; row++) { - if (stringData[row] != null) { - stringData[row] = PApplet.trim(stringData[row]); + // TODO naming/whether to include + protected Table createSubset(int[] rowSubset) { + Table newbie = new Table(); + newbie.setColumnTitles(columnTitles); // also sets columns.length + newbie.columnTypes = columnTypes; + newbie.setRowCount(rowSubset.length); + + for (int i = 0; i < rowSubset.length; i++) { + int row = rowSubset[i]; + for (int col = 0; col < columns.length; col++) { + switch (columnTypes[col]) { + case STRING: newbie.setString(i, col, getString(row, col)); break; + case INT: newbie.setInt(i, col, getInt(row, col)); break; + case LONG: newbie.setLong(i, col, getLong(row, col)); break; + case FLOAT: newbie.setFloat(i, col, getFloat(row, col)); break; + case DOUBLE: newbie.setDouble(i, col, getDouble(row, col)); break; } } } + return newbie; } + + + // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + + + /** + * Searches the entire table for float values. + * Returns missing float (Float.NaN by default) if no valid numbers found. + */ + protected float getMaxFloat() { + boolean found = false; + float max = PConstants.MIN_FLOAT; + for (int row = 0; row < getRowCount(); row++) { + for (int col = 0; col < getColumnCount(); col++) { + float value = getFloat(row, col); + if (!Float.isNaN(value)) { // TODO no, this should be comparing to the missing value + if (!found) { + max = value; + found = true; + } else if (value > max) { + max = value; + } + } + } + } + return found ? max : missingFloat; + } + + + // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + + + // converts a TSV or CSV file to binary.. do not use + protected void convertBasic(BufferedReader reader, boolean tsv, + File outputFile) throws IOException { + FileOutputStream fos = new FileOutputStream(outputFile); + BufferedOutputStream bos = new BufferedOutputStream(fos, 16384); + DataOutputStream output = new DataOutputStream(bos); + output.writeInt(0); // come back for row count + output.writeInt(getColumnCount()); + if (columnTitles != null) { + output.writeBoolean(true); + for (String title : columnTitles) { + output.writeUTF(title); + } + } else { + output.writeBoolean(false); + } + for (int type : columnTypes) { + output.writeInt(type); + } + + String line = null; + //setRowCount(1); + int prev = -1; + int row = 0; + while ((line = reader.readLine()) != null) { + convertRow(output, tsv ? PApplet.split(line, '\t') : splitLineCSV(line)); + row++; + + if (row % 10000 == 0) { + if (row < rowCount) { + int pct = (100 * row) / rowCount; + if (pct != prev) { + System.out.println(pct + "%"); + prev = pct; + } + } +// try { +// Thread.sleep(5); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } + } + } + // shorten or lengthen based on what's left +// if (row != getRowCount()) { +// setRowCount(row); +// } + + // has to come afterwards, since these tables get built out during the conversion + int col = 0; + for (HashMapBlows hmb : columnCategories) { + if (hmb == null) { + output.writeInt(0); + } else { + hmb.write(output); + hmb.writeln(PApplet.createWriter(new File(columnTitles[col] + ".categories"))); +// output.writeInt(hmb.size()); +// for (Map.Entry e : hmb.entrySet()) { +// output.writeUTF(e.getKey()); +// output.writeInt(e.getValue()); +// } + } + col++; + } + + output.flush(); + output.close(); + + // come back and write the row count + RandomAccessFile raf = new RandomAccessFile(outputFile, "rw"); + raf.writeInt(rowCount); + raf.close(); + } + + + protected void convertRow(DataOutputStream output, String[] pieces) throws IOException { + if (pieces.length > getColumnCount()) { + throw new IllegalArgumentException("Row with too many columns: " + + PApplet.join(pieces, ",")); + } + // pieces.length may be less than columns.length, so loop over pieces + for (int col = 0; col < pieces.length; col++) { + switch (columnTypes[col]) { + case STRING: + output.writeUTF(pieces[col]); + break; + case INT: + output.writeInt(PApplet.parseInt(pieces[col], missingInt)); + break; + case LONG: + try { + output.writeLong(Long.parseLong(pieces[col])); + } catch (NumberFormatException nfe) { + output.writeLong(missingLong); + } + break; + case FLOAT: + output.writeFloat(PApplet.parseFloat(pieces[col], missingFloat)); + break; + case DOUBLE: + try { + output.writeDouble(Double.parseDouble(pieces[col])); + } catch (NumberFormatException nfe) { + output.writeDouble(missingDouble); + } + break; + case CATEGORICAL: + output.writeInt(columnCategories[col].index(pieces[col])); + break; + } + } + for (int col = pieces.length; col < getColumnCount(); col++) { + switch (columnTypes[col]) { + case STRING: + output.writeUTF(""); + break; + case INT: + output.writeInt(missingInt); + break; + case LONG: + output.writeLong(missingLong); + break; + case FLOAT: + output.writeFloat(missingFloat); + break; + case DOUBLE: + output.writeDouble(missingDouble); + break; + case CATEGORICAL: + output.writeInt(missingCategory); + break; + + } + } + } + + + /* + private void convertRowCol(DataOutputStream output, int row, int col, String piece) { + switch (columnTypes[col]) { + case STRING: + String[] stringData = (String[]) columns[col]; + stringData[row] = piece; + break; + case INT: + int[] intData = (int[]) columns[col]; + intData[row] = PApplet.parseInt(piece, missingInt); + break; + case LONG: + long[] longData = (long[]) columns[col]; + try { + longData[row] = Long.parseLong(piece); + } catch (NumberFormatException nfe) { + longData[row] = missingLong; + } + break; + case FLOAT: + float[] floatData = (float[]) columns[col]; + floatData[row] = PApplet.parseFloat(piece, missingFloat); + break; + case DOUBLE: + double[] doubleData = (double[]) columns[col]; + try { + doubleData[row] = Double.parseDouble(piece); + } catch (NumberFormatException nfe) { + doubleData[row] = missingDouble; + } + break; + default: + throw new IllegalArgumentException("That's not a valid column type."); + } + } + */ } diff --git a/android/core/src/processing/data/TableRow.java b/android/core/src/processing/data/TableRow.java index 5a10e3aa4..d01dcd1e7 100644 --- a/android/core/src/processing/data/TableRow.java +++ b/android/core/src/processing/data/TableRow.java @@ -12,4 +12,15 @@ public interface TableRow { public float getFloat(String columnName); public double getDouble(int column); public double getDouble(String columnName); + + public void setString(int column, String value); + public void setString(String columnName, String value); + public void setInt(int column, int value); + public void setInt(String columnName, int value); + public void setLong(int column, long value); + public void setLong(String columnName, long value); + public void setFloat(int column, float value); + public void setFloat(String columnName, float value); + public void setDouble(int column, double value); + public void setDouble(String columnName, double value); } diff --git a/android/core/src/processing/data/XML.java b/android/core/src/processing/data/XML.java index 338ab2087..40ca83a7c 100644 --- a/android/core/src/processing/data/XML.java +++ b/android/core/src/processing/data/XML.java @@ -49,8 +49,8 @@ public class XML implements Serializable { /** The internal representation, a DOM node. */ protected Node node; - /** Cached locally because it's used often. */ - protected String name; +// /** Cached locally because it's used often. */ +// protected String name; /** The parent element. */ protected XML parent; @@ -62,26 +62,40 @@ public class XML implements Serializable { protected XML() { } - /** - * Begin parsing XML data passed in from a PApplet. This code - * wraps exception handling, for more advanced exception handling, - * use the constructor that takes a Reader or InputStream. - * - * @throws SAXException - * @throws ParserConfigurationException - * @throws IOException - */ - public XML(PApplet parent, String filename) throws IOException, ParserConfigurationException, SAXException { - this(parent.createReader(filename)); - } - +// /** +// * Begin parsing XML data passed in from a PApplet. This code +// * wraps exception handling, for more advanced exception handling, +// * use the constructor that takes a Reader or InputStream. +// * +// * @throws SAXException +// * @throws ParserConfigurationException +// * @throws IOException +// */ +// public XML(PApplet parent, String filename) throws IOException, ParserConfigurationException, SAXException { +// this(parent.createReader(filename)); +// } public XML(File file) throws IOException, ParserConfigurationException, SAXException { - this(PApplet.createReader(file)); + this(file, null); } - public XML(Reader reader) throws IOException, ParserConfigurationException, SAXException { + public XML(File file, String options) throws IOException, ParserConfigurationException, SAXException { + this(PApplet.createReader(file), options); + } + + + public XML(InputStream input) throws IOException, ParserConfigurationException, SAXException { + this(input, null); + } + + + public XML(InputStream input, String options) throws IOException, ParserConfigurationException, SAXException { + this(PApplet.createReader(input), options); + } + + + protected XML(Reader reader, String options) throws IOException, ParserConfigurationException, SAXException { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); // Prevent 503 errors from www.w3.org @@ -115,7 +129,8 @@ public class XML implements Serializable { // Document document = builder.parse(dataPath("1_alt.html")); Document document = builder.parse(new InputSource(reader)); node = document.getDocumentElement(); - name = node.getNodeName(); +// name = node.getNodeName(); + // NodeList nodeList = document.getDocumentElement().getChildNodes(); // for (int i = 0; i < nodeList.getLength(); i++) { // } @@ -124,46 +139,51 @@ public class XML implements Serializable { // TODO is there a more efficient way of doing this? wow. - // i.e. can we use one static document object for all PNodeXML objects? - public XML(String name) { - try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - Document document = builder.newDocument(); - node = document.createElement(name); - - this.name = name; - this.parent = null; - - } catch (ParserConfigurationException e) { - e.printStackTrace(); - } + public XML(String name) throws ParserConfigurationException { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.newDocument(); + node = document.createElement(name); +// this.name = name; + this.parent = null; } protected XML(XML parent, Node node) { this.node = node; this.parent = parent; - this.name = node.getNodeName(); +// this.name = node.getNodeName(); } - static public XML parse(String xml) { - try { - return new XML(new StringReader(xml)); - } catch (Exception e) { - e.printStackTrace(); - return null; - } + /** + * xxxxxxx + * + * @webref xml:method + * @brief Converts String content to an XML object + * @param data the content to be parsed as XML + * @return an XML object, or null + * @throws SAXException + * @throws ParserConfigurationException + * @throws IOException + * @see PApplet#loadXML(String) + */ + static public XML parse(String data) throws IOException, ParserConfigurationException, SAXException { + return XML.parse(data, null); } - public boolean save(OutputStream output) { + static public XML parse(String data, String options) throws IOException, ParserConfigurationException, SAXException { + return new XML(new StringReader(data), null); + } + + + protected boolean save(OutputStream output) { return save(PApplet.createWriter(output)); } - public boolean save(File file) { + public boolean save(File file, String options) { return save(PApplet.createWriter(file)); } @@ -189,7 +209,7 @@ public class XML implements Serializable { /** * Internal function; not included in reference. */ - protected Node getNode() { + protected Object getNative() { return node; } @@ -203,7 +223,8 @@ public class XML implements Serializable { * @return the name, or null if the element only contains #PCDATA. */ public String getName() { - return name; +// return name; + return node.getNodeName(); } /** @@ -213,7 +234,7 @@ public class XML implements Serializable { public void setName(String newName) { Document document = node.getOwnerDocument(); node = document.renameNode(node, null, newName); - name = node.getNodeName(); +// name = node.getNodeName(); } @@ -334,6 +355,9 @@ public class XML implements Serializable { * @return the first matching element */ public XML getChild(String name) { + if (name.length() > 0 && name.charAt(0) == '/') { + throw new IllegalArgumentException("getChild() should not begin with a slash"); + } if (name.indexOf('/') != -1) { return getChildRecursive(PApplet.split(name, '/'), 0); } @@ -392,6 +416,9 @@ public class XML implements Serializable { * @author processing.org */ public XML[] getChildren(String name) { + if (name.length() > 0 && name.charAt(0) == '/') { + throw new IllegalArgumentException("getChildren() should not begin with a slash"); + } if (name.indexOf('/') != -1) { return getChildrenRecursive(PApplet.split(name, '/'), 0); } @@ -441,7 +468,7 @@ public class XML implements Serializable { public XML addChild(XML child) { Document document = node.getOwnerDocument(); - Node newChild = document.importNode(child.getNode(), true); + Node newChild = document.importNode((Node) child.getNative(), true); return appendChild(newChild); } @@ -467,36 +494,53 @@ public class XML implements Serializable { } - /** Remove whitespace nodes. */ - public void trim() { -//// public static boolean isWhitespace(XML xml) { -//// if (xml.node.getNodeType() != Node.TEXT_NODE) -//// return false; -//// Matcher m = whitespace.matcher(xml.node.getNodeValue()); -//// return m.matches(); -//// } -// trim(this); +// /** Remove whitespace nodes. */ +// public void trim() { +////// public static boolean isWhitespace(XML xml) { +////// if (xml.node.getNodeType() != Node.TEXT_NODE) +////// return false; +////// Matcher m = whitespace.matcher(xml.node.getNodeValue()); +////// return m.matches(); +////// } +//// trim(this); +//// } +// +// checkChildren(); +// int index = 0; +// for (int i = 0; i < children.length; i++) { +// if (i != index) { +// children[index] = children[i]; +// } +// Node childNode = (Node) children[i].getNative(); +// if (childNode.getNodeType() != Node.TEXT_NODE || +// children[i].getContent().trim().length() > 0) { +// children[i].trim(); +// index++; +// } +// } +// if (index != children.length) { +// children = (XML[]) PApplet.subset(children, 0, index); +// } +// +// // possibility, but would have to re-parse the object +//// helpdesk.objects.com.au/java/how-do-i-remove-whitespace-from-an-xml-document +//// TransformerFactory factory = TransformerFactory.newInstance(); +//// Transformer transformer = factory.newTransformer(new StreamSource("strip-space.xsl")); +//// DOMSource source = new DOMSource(document); +//// StreamResult result = new StreamResult(System.out); +//// transformer.transform(source, result); +// +//// +//// +//// +//// +//// +//// +//// +//// +//// // } -// -// -// protected void trim() { - checkChildren(); - int index = 0; - for (int i = 0; i < children.length; i++) { - if (i != index) { - children[index] = children[i]; - } - Node childNode = children[i].getNode(); - if (childNode.getNodeType() != Node.TEXT_NODE || - children[i].getContent().trim().length() > 0) { - children[i].trim(); - index++; - } - } - if (index != children.length) { - children = (XML[]) PApplet.subset(children, 0, index); - } - } /** @@ -718,20 +762,30 @@ public class XML implements Serializable { } + /** + * Format this XML data as a String. + * @param indent -1 for a single line (and no declaration), >= 0 for indents and newlines + */ public String format(int indent) { try { - DOMSource dumSource = new DOMSource(node); // entities = doctype.getEntities() - TransformerFactory tf = TransformerFactory.newInstance(); - Transformer transformer = tf.newTransformer(); - // if this is the root, output the decl, if not, hide it + TransformerFactory factory = TransformerFactory.newInstance(); + if (indent != -1) { + factory.setAttribute("indent-number", indent); + } + Transformer transformer = factory.newTransformer(); + + // Add the XML declaration at the top if this node is the root and we're + // not writing to a single line (indent = -1 means single line). if (indent == -1 || parent != null) { transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); } else { transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "no"); } // transformer.setOutputProperty(OutputKeys.DOCTYPE_SYSTEM, "sample.dtd"); + transformer.setOutputProperty(OutputKeys.METHOD, "xml"); + // transformer.setOutputProperty(OutputKeys.CDATA_SECTION_ELEMENTS, "yes"); // huh? // transformer.setOutputProperty(OutputKeys.DOCTYPE_PUBLIC, @@ -745,18 +799,45 @@ public class XML implements Serializable { // transformer.setOutputProperty(OutputKeys.CDATA_SECTION_ELEMENTS // indent by default, but sometimes this needs to be turned off if (indent != 0) { - transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", String.valueOf(indent)); + //transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", String.valueOf(indent)); transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + } else { + transformer.setOutputProperty(OutputKeys.INDENT, "no"); } // Properties p = transformer.getOutputProperties(); // for (Object key : p.keySet()) { // System.out.println(key + " -> " + p.get(key)); // } - StringWriter sw = new StringWriter(); - StreamResult sr = new StreamResult(sw); - transformer.transform(dumSource, sr); - return sw.toString(); + // If you smell something, that's because this code stinks. No matter + // the settings of the Transformer object, if the XML document already + // has whitespace elements, it won't bother re-indenting/re-formatting. + // So instead, transform the data once into a single line string. + // If indent is -1, then we're done. Otherwise re-run and the settings + // of the factory will kick in. If you know a better way to do this, + // please contribute. I've wasted too much of my Sunday on it. But at + // least the Giants are getting blown out by the Falcons. + + StringWriter tempWriter = new StringWriter(); + StreamResult tempResult = new StreamResult(tempWriter); + transformer.transform(new DOMSource(node), tempResult); + String[] tempLines = PApplet.split(tempWriter.toString(), '\n'); + if (tempLines[0].startsWith(" get this from the type of fieldName // Class sketchClass = sketch.getClass(); - Class sketchClass = enclosingObject.getClass(); + Class sketchClass = enclosingObject.getClass(); targetField = sketchClass.getDeclaredField(fieldName); // PApplet.println("found " + targetField); - Class targetArray = targetField.getType(); + Class targetArray = targetField.getType(); if (!targetArray.isArray()) { // fieldName is not an array } else { @@ -510,8 +510,8 @@ public class Table { // Object enclosingObject = sketch; // PApplet.println("enclosing obj is " + enclosingObject); - Class enclosingClass = target.getEnclosingClass(); - Constructor con = null; + Class enclosingClass = target.getEnclosingClass(); + Constructor con = null; try { if (enclosingClass == null) { @@ -1595,7 +1595,6 @@ public class Table { table.setDouble(row, column, value); } - @Override public void setDouble(String columnName, double value) { table.setDouble(row, columnName, value); } diff --git a/core/todo.txt b/core/todo.txt index b7a0089b0..2d0fd3350 100644 --- a/core/todo.txt +++ b/core/todo.txt @@ -34,28 +34,6 @@ X better to do this instead of bringing back the magic event X or implementing the magic event on Android X also problematic with it not being called now - -decisions on data -X are we comfortable with setInt/Float/etc instead of just set(int, blah) -X yes, better to have parity -X too weird to have to explain why getXxx() needs types and set() doesn't -X get/set with Java's 'Map' class? -X really useful, but leaning toward not including it -X or leave it in as an advanced feature? -X createXxx() methods less important -X XML.parse() - or new XML("") or new XML("tag") -_ add parseXML() and parseJSONObject(x) -X api note: size() used in data classes -X length() too confusing w/ array.length being built-in (when to use ()?) -X size() a bit confusing with the p5 size command, but less problematic -X also shorter than getCount() or getLength() -_ why not HashMap and ArrayList for JSON? -_ could enable loadHash() and loadArray() functions -_ or loadDict() and loadList() as the case might be -_ JSONObject.has(key) vs XML.hasAttribute(attr) vs HashMap.containsKey() -_ and how it should be handled with hash/dict - - cleaning/earlier C textureWrap() CLAMP and REPEAT now added C begin/endContour() @@ -77,7 +55,56 @@ X saveTable("filename.tsv") or saveTable("filename.txt", "tsv") X createTable() method in PApplet X removed getUniqueXxxx() and some others, pending names -_ OutOfMemory in image() +xml library +X removed 'name' field to avoid possibility of random errors +X confirmed that DOM "correct" version includes the text nodes +X new XML(name) also throws an ex, use createXML() or appendChild("name") +X remove XML.parse() from the reference (it throws an exception) +X use parseXML() instead +o do we need a trim() method for XML? +o use XSL transform to strip whitespace +o helpdesk.objects.com.au/java/how-do-i-remove-whitespace-from-an-xml-document +X messy, just not great +o isWhitespace() method for nodes? (that's the capitalization used in Character) +X doesn't help much +o get back to PhiLho once finished +X XML toString(0) means no indents or newlines +X but no way to remove indents and still have newlines... +X toString(-1)? a new method? +X format(2), format(4)... toString() -> default to 2 params +X fix this across the other items +X look into json and how it would work wrt XML +o 1) we bring back getFloatAttribute() et al., +o and make getFloat() be equivalent to parseFloat(xml.getContent()) +o 2) we keep getFloat() like it is, and add getFloatContent(), etc. +o 3) we deprecate our nice short getFloat/getInt/etc and go with +o getXxxxAttribute() and getXxxxContent() methods. +X not gonna do getFloatContent() since it's not really any shorter +X beginning slash in getChild() threw an NPE + +decisions on data +X are we comfortable with setInt/Float/etc instead of just set(int, blah) +X yes, better to have parity +X too weird to have to explain why getXxx() needs types and set() doesn't +X get/set with Java's 'Map' class? +X really useful, but leaning toward not including it +X or leave it in as an advanced feature? +X createXxx() methods less important +X XML.parse() - or new XML("") or new XML("tag") +_ add parseXML() and parseJSONObject(x) +X api note: size() used in data classes +X length() too confusing w/ array.length being built-in (when to use ()?) +X size() a bit confusing with the p5 size command, but less problematic +X also shorter than getCount() or getLength() +_ why not HashMap and ArrayList for JSON? +_ could enable loadHash() and loadArray() functions +_ or loadDict() and loadList() as the case might be +_ JSONObject.has(key) vs XML.hasAttribute(attr) vs HashMap.containsKey() +_ and how it should be handled with hash/dict +_ right now using hasKey().. in JSONObject + + +_ image caches not being properly disposed (weak references broken?) _ http://code.google.com/p/processing/issues/detail?id=1353 _ shader syntax (Andres request) @@ -130,33 +157,6 @@ _ colorCalc() methods added to PShape.. should just be used from PGraphics _ loadShape() needs to live in PApplet _ make PShapeOpenGL a cache object -xml library -X removed 'name' field to avoid possibility of random errors -X confirmed that DOM "correct" version includes the text nodes -X new XML(name) also throws an ex, use createXML() or appendChild("name") -X remove XML.parse() from the reference (it throws an exception) -X use parseXML() instead -o do we need a trim() method for XML? -o use XSL transform to strip whitespace -o helpdesk.objects.com.au/java/how-do-i-remove-whitespace-from-an-xml-document -X messy, just not great -o isWhitespace() method for nodes? (that's the capitalization used in Character) -X doesn't help much -o get back to PhiLho once finished -X XML toString(0) means no indents or newlines -X but no way to remove indents and still have newlines... -X toString(-1)? a new method? -X format(2), format(4)... toString() -> default to 2 params -X fix this across the other items -X look into json and how it would work wrt XML -o 1) we bring back getFloatAttribute() et al., -o and make getFloat() be equivalent to parseFloat(xml.getContent()) -o 2) we keep getFloat() like it is, and add getFloatContent(), etc. -o 3) we deprecate our nice short getFloat/getInt/etc and go with -o getXxxxAttribute() and getXxxxContent() methods. -X not gonna do getFloatContent() since it's not really any shorter -X beginning slash in getChild() threw an NPE - async requests Request r = createRequest("http://p5.org/feed/13134.jpg"); Request r = createRequest("http://p5.org/feed/13134.jpg", "callbackName"); @@ -227,9 +227,6 @@ _ maybe a hack where a new menubar is added? _ splice() throws ClassCastException when used with objects like PVector _ http://code.google.com/p/processing/issues/detail?id=1407 -_ add "CGAffineTransformInvert: singular matrix" problem to the Wiki -_ http://code.google.com/p/processing/issues/detail?id=1363 - api to be fixed/removed _ remove PImage.delete() and friends from PImage, Movie, etc. _ delete()/dispose() being used in the movie @@ -533,6 +530,12 @@ _ document somehow.. svg viewer will be discontinued _ http://www.adobe.com/svg/eol.html +CORE / PGraphicsJava2D + +_ add "CGAffineTransformInvert: singular matrix" problem to the Wiki +_ http://code.google.com/p/processing/issues/detail?id=1363 + + CORE / OpenGL (Andres) _ ortho() issues