diff --git a/core/src/processing/core/PApplet.java b/core/src/processing/core/PApplet.java index fb9a5e3e3..58b1b5d2a 100644 --- a/core/src/processing/core/PApplet.java +++ b/core/src/processing/core/PApplet.java @@ -5761,7 +5761,8 @@ public class PApplet extends Applet public Table createTable() { - return new Table(this); +// return new Table(this); + return new Table(); } @@ -5779,7 +5780,8 @@ public class PApplet extends Applet public Table loadTable(String filename, String options) { try { - return new Table(this, filename, options); +// return new Table(this, filename, options); + return new Table(createInput(filename), options); } catch (IOException e) { e.printStackTrace(); return null; @@ -5787,11 +5789,30 @@ public class PApplet extends Applet } + public void saveTable(Table table, String filename) { + saveTable(table, filename, null); + } + + + public void saveTable(Table table, String filename, String options) { + try { + table.save(saveFile(filename), options); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // static public Table loadTable(File file) { // return new Table(this, file); // } + static public String[] fixOptions(String options) { + return options == null ? new String[0] : options.split("\\s*,\\s*"); + } + + ////////////////////////////////////////////////////////////// diff --git a/core/src/processing/data/Table.java b/core/src/processing/data/Table.java index 7d09f4747..08fb7f57b 100644 --- a/core/src/processing/data/Table.java +++ b/core/src/processing/data/Table.java @@ -73,7 +73,7 @@ public class Table { // protected boolean skipCommentLines = true; protected String extension = null; // protected boolean commaSeparatedValues = false; - protected boolean awfulCSV = false; +// protected boolean awfulCSV = false; protected String missingString = null; protected int missingInt = 0; @@ -113,20 +113,21 @@ public class Table { // double[][] doubleData; // Object[][] objectData; - PApplet sketch; +// PApplet sketch; /** * Creates a new, empty table. Use addRow() to add additional rows. */ public Table() { - init(null); +// init(null); + init(); } - public Table(PApplet sketch) { - init(sketch); - } +// public Table(PApplet sketch) { +// init(sketch); +// } // public Table(File file) { @@ -134,95 +135,43 @@ public class Table { // } - public Table(PApplet parent, String filename) throws IOException { - this(parent, filename, null); +// public Table(PApplet parent, String filename) throws IOException { +// this(parent, filename, null); +// } + + + public Table(File file) throws IOException { + this(file, null); + } + + + // version that uses a File object; future releases (or data types) + // may include additional optimizations here + public Table(File file, String options) throws IOException { + parse(new FileInputStream(file), checkOptions(file, options)); } /** - * Can handle TSV or CSV files. - * @param parent - * @param filename + * Read the table from a stream. Possible options include: + * + * @param input + * @param options * @throws IOException */ - public Table(PApplet parent, String filename, String options) throws IOException { - init(parent); - //String[] opts = PApplet.split(options, ','); - //PApplet.trim(opts); - - // try to determine file type from extension - String extension = null; - int dotIndex = filename.lastIndexOf('.'); - if (dotIndex != -1) { - extension = filename.substring(dotIndex + 1).toLowerCase(); - if (!extension.equals("csv") && - !extension.equals("tsv")) { - // ignore extension - extension = null; - } - } - - String[] opts = null; - if (options != null) { - opts = options.split("\\s*,\\s*"); -// PApplet.println("options:"); -// PApplet.println(opts); - - for (String opt : opts) { - if (opt.equals("tsv")) { - extension = "tsv"; - } else if (opt.equals("csv")) { - extension = "csv"; - } else if (opt.equals("newlines")) { - awfulCSV = true; - } else { - System.err.println(opt + " is not a valid option for loading a Table"); - } - } - } - - BufferedReader reader = parent.createReader(filename); - if (awfulCSV) { - parseAwfulCSV(reader); - } else if ("tsv".equals(extension)) { - parseBasic(reader, true); - } else if ("csv".equals(extension)) { - parseBasic(reader, false); - } - //read(); + public Table(InputStream input, String options) throws IOException { + parse(input, options); } -// public Table(BufferedReader reader) { -// read(reader); -// } - - - protected void init(PApplet sketch) { - this.sketch = sketch; - columns = new Object[0]; - columnTypes = new int[0]; - columnCategories = new HashMapBlows[0]; - } - - -// protected void read(BufferedReader reader) { -// init(); -// try { -// boolean csv = peekCSV(reader); -// if (csv) { -// parseCSV(reader); -// } else { -// parseTSV(reader); -// } -// } catch (IOException e) { -// e.printStackTrace(); -// } -// } - - public Table(ResultSet rs) { - init(null); +// init(null); + init(); try { ResultSetMetaData rsmd = rs.getMetaData(); @@ -279,86 +228,80 @@ public class Table { } - /** - * Guess whether this file is tab separated or comma separated by checking - * whether there are more tabs or commas in the first 100 characters. - */ - /* - protected boolean peekCSV(BufferedReader reader) throws IOException { - char[] buffer = new char[100]; - int remaining = buffer.length; - reader.mark(remaining); -// int count = 0; - int commas = 0; - int tabs = 0; - for (int i = 0; i < remaining; i++) { - int c = reader.read(); - if (c == -1) break; - if (c == ',') commas++; - if (c == '\t') tabs++; - } - reader.reset(); - return (commas > tabs); + protected void init() { + columns = new Object[0]; + columnTypes = new int[0]; + columnCategories = new HashMapBlows[0]; } - public void parse(BufferedReader reader) throws IOException { - if (commaSeparatedValues) { - if (awfulCSV) { - parseAwfulCSV(reader); - } else { - parseCSV(reader); + protected String checkOptions(File file, String options) throws IOException { + String extension = null; + String filename = file.getName(); + int dotIndex = filename.lastIndexOf('.'); + if (dotIndex != -1) { + extension = filename.substring(dotIndex + 1).toLowerCase(); + if (!extension.equals("csv") && + !extension.equals("tsv")) { + // ignore extension + extension = null; } - } else { - parseTSV(reader); + } + if (extension == null) { + if (options == null) { + throw new IOException("This table filename has no extension, and no options are set."); + } + } else { // extension is not null + if (options == null) { + options = extension; + } else { + // prepend the extension, it will be overridden if there's an option for it. + options = extension + "," + options; + } + } + return options; + } + + + protected void parse(InputStream input, String options) throws IOException { + init(); + + String[] opts = PApplet.fixOptions(options); +// if (options != null) { +// opts = options.split("\\s*,\\s*"); +//// PApplet.println("options:"); +//// PApplet.println(opts); + + boolean awfulCSV = false; + boolean header = false; + for (String opt : opts) { + if (opt.equals("tsv")) { + extension = "tsv"; + } else if (opt.equals("csv")) { + extension = "csv"; + } else if (opt.equals("newlines")) { + awfulCSV = true; + } else if (opt.equals("header")) { + header = true; + } else { + throw new IllegalArgumentException("'" + opt + "' is not a valid option for loading a Table"); + } + } +// } + + BufferedReader reader = PApplet.createReader(input); + if (awfulCSV) { + parseAwfulCSV(reader, header); + } else if ("tsv".equals(extension)) { + parseBasic(reader, header, true); + } else if ("csv".equals(extension)) { + parseBasic(reader, header, false); } } - */ - public void parseTSV(BufferedReader reader) throws IOException { - parseBasic(reader, true); -// String line = null; -// int row = 0; -// if (rowCount == 0) { -// setRowCount(10); -// } -// while ((line = reader.readLine()) != null) { -// if (row == getRowCount()) { -// setRowCount(row << 1); -// } -// setRow(row, PApplet.split(line, '\t')); -// row++; -// } -// // shorten or lengthen based on what's left -// if (row != getRowCount()) { -// setRowCount(row); -// } - } - - - public void parseCSV(BufferedReader reader) throws IOException { - parseBasic(reader, false); -// String line = null; -// int row = 0; -// if (rowCount == 0) { -// setRowCount(10); -// } -// while ((line = reader.readLine()) != null) { -// if (row == getRowCount()) { -// setRowCount(row << 1); -// } -// setRow(row, splitLineCSV(line)); -// row++; -// } -// // shorten or lengthen based on what's left -// if (row != getRowCount()) { -// setRowCount(row); -// } - } - - - protected void parseBasic(BufferedReader reader, boolean tsv) throws IOException { + protected void parseBasic(BufferedReader reader, + boolean header, boolean tsv) throws IOException { String line = null; int row = 0; if (rowCount == 0) { @@ -394,86 +337,13 @@ public class Table { } - public void convertTSV(BufferedReader reader, File outputFile) throws IOException { - convertBasic(reader, true, outputFile); - } +// public void convertTSV(BufferedReader reader, File outputFile) throws IOException { +// convertBasic(reader, true, outputFile); +// } - protected void convertBasic(BufferedReader reader, boolean tsv, - File outputFile) throws IOException { - FileOutputStream fos = new FileOutputStream(outputFile); - BufferedOutputStream bos = new BufferedOutputStream(fos, 16384); - DataOutputStream output = new DataOutputStream(bos); - output.writeInt(0); // come back for row count - output.writeInt(getColumnCount()); - if (columnTitles != null) { - output.writeBoolean(true); - for (String title : columnTitles) { - output.writeUTF(title); - } - } else { - output.writeBoolean(false); - } - for (int type : columnTypes) { - output.writeInt(type); - } - - String line = null; - //setRowCount(1); - int prev = -1; - int row = 0; - while ((line = reader.readLine()) != null) { - convertRow(output, tsv ? PApplet.split(line, '\t') : splitLineCSV(line)); - row++; - - if (row % 10000 == 0) { - if (row < rowCount) { - int pct = (100 * row) / rowCount; - if (pct != prev) { - System.out.println(pct + "%"); - prev = pct; - } - } -// try { -// Thread.sleep(5); -// } catch (InterruptedException e) { -// e.printStackTrace(); -// } - } - } - // shorten or lengthen based on what's left -// if (row != getRowCount()) { -// setRowCount(row); -// } - - // has to come afterwards, since these tables get built out during the conversion - int col = 0; - for (HashMapBlows hmb : columnCategories) { - if (hmb == null) { - output.writeInt(0); - } else { - hmb.write(output); - hmb.writeln(PApplet.createWriter(new File(columnTitles[col] + ".categories"))); -// output.writeInt(hmb.size()); -// for (Map.Entry e : hmb.entrySet()) { -// output.writeUTF(e.getKey()); -// output.writeInt(e.getValue()); -// } - } - col++; - } - - output.flush(); - output.close(); - - // come back and write the row count - RandomAccessFile raf = new RandomAccessFile(outputFile, "rw"); - raf.writeInt(rowCount); - raf.close(); - } - - - protected void parseAwfulCSV(BufferedReader reader) throws IOException { + protected void parseAwfulCSV(BufferedReader reader, + boolean header) throws IOException { char[] c = new char[100]; int count = 0; boolean insideQuote = false; @@ -512,20 +382,23 @@ public class Table { if (ch == '\"') { insideQuote = true; - } else if (ch == '\r') { - // check to see if next is a '\n' - reader.mark(1); - if (reader.read() != '\n') { - reader.reset(); + } else if (ch == '\r' || ch == '\n') { + if (ch == '\r') { + // check to see if next is a '\n' + reader.mark(1); + if (reader.read() != '\n') { + reader.reset(); + } } setString(row, col, new String(c, 0, count)); count = 0; - row++; - col = 0; - - } else if (ch == '\n') { - setString(row, col, new String(c, 0, count)); - count = 0; + if (row == 0 && header) { + // Use internal row removal (efficient because only one row). + removeTitleRow(); + // Un-set the header variable so that next time around, we don't + // just get stuck into a loop, removing the 0th row repeatedly. + header = false; + } row++; col = 0; @@ -563,7 +436,7 @@ public class Table { * @param line line of text to be parsed * @return an array of the individual values formerly separated by commas */ - static public String[] splitLineCSV(String line) { + static protected String[] splitLineCSV(String line) { char[] c = line.toCharArray(); int rough = 1; // at least one boolean quote = false; @@ -609,7 +482,7 @@ public class Table { } - static int nextComma(char[] c, int index) { + static protected int nextComma(char[] c, int index) { boolean quote = false; for (int i = index; i < c.length; i++) { if (!quote && (c[i] == ',')) { @@ -640,16 +513,18 @@ public class Table { // compiler) of an inner class by the runtime. /** incomplete, do not use */ - public void parseInto(String fieldName) { +// public void parseInto(PApplet sketch, String fieldName) { + public void parseInto(Object enclosingObject, String fieldName) { Class target = null; Object outgoing = null; Field targetField = null; try { // Object targetObject, // Class target -> get this from the type of fieldName - Class sketchClass = sketch.getClass(); +// Class sketchClass = sketch.getClass(); + Class sketchClass = enclosingObject.getClass(); targetField = sketchClass.getDeclaredField(fieldName); - PApplet.println("found " + targetField); +// PApplet.println("found " + targetField); Class targetArray = targetField.getType(); if (!targetArray.isArray()) { // fieldName is not an array @@ -663,22 +538,21 @@ public class Table { e.printStackTrace(); } - Object enclosingObject = sketch; - PApplet.println("enclosing obj is " + enclosingObject); +// Object enclosingObject = sketch; +// PApplet.println("enclosing obj is " + enclosingObject); Class enclosingClass = target.getEnclosingClass(); Constructor con = null; try { if (enclosingClass == null) { con = target.getDeclaredConstructor(); //new Class[] { }); - PApplet.println("no enclosing class"); +// PApplet.println("no enclosing class"); } else { con = target.getDeclaredConstructor(new Class[] { enclosingClass }); -// con = target.getConstructor(enclosingClass); - PApplet.println("enclosed by " + enclosingClass.getName()); +// PApplet.println("enclosed by " + enclosingClass.getName()); } if (!con.isAccessible()) { - System.out.println("setting constructor to public"); +// System.out.println("setting constructor to public"); con.setAccessible(true); } } catch (SecurityException e) { @@ -692,32 +566,18 @@ public class Table { for (Field field : fields) { String name = field.getName(); if (getColumnIndex(name, false) != -1) { - System.out.println("found field " + name); +// System.out.println("found field " + name); if (!field.isAccessible()) { - PApplet.println(" changing field access"); +// PApplet.println(" changing field access"); field.setAccessible(true); } inuse.add(field); } else { - System.out.println("skipping field " + name); +// System.out.println("skipping field " + name); } } -// Constructor[] cons = target.getDeclaredConstructors(); -// //for (Method m : methods) { -// Constructor defaultCons = null; -// for (Constructor c : cons) { -// //System.out.println("found " + c.getParameterTypes()); -// if (c.getParameterTypes().length == 0) { -// System.out.println("found default"); -// defaultCons = c; -// c.setAccessible(true); -// } else { -// PApplet.println(c.getParameterTypes()); -// } -// } int index = 0; -// ArrayList list = new ArrayList(); try { for (TableRow row : getRows()) { Object item = null; @@ -778,11 +638,12 @@ public class Table { Array.set(outgoing, index++, item); } if (!targetField.isAccessible()) { - PApplet.println("setting target field to public"); +// PApplet.println("setting target field to public"); targetField.setAccessible(true); } // Set the array in the sketch - targetField.set(sketch, outgoing); +// targetField.set(sketch, outgoing); + targetField.set(enclosingObject, outgoing); } catch (InstantiationException e) { e.printStackTrace(); @@ -795,100 +656,33 @@ public class Table { } } - /* - public Object[] parse(Class target) { - Field[] fields = target.getDeclaredFields(); - ArrayList inuse = new ArrayList(); - for (Field field : fields) { - String name = field.getName(); - if (getColumnIndex(name, false) != -1) { - System.out.println("found field " + name); - inuse.add(field); - } else { - System.out.println("skipping field " + name); - } - } - Constructor[] cons = target.getDeclaredConstructors(); - //for (Method m : methods) { - Constructor defaultCons = null; - for (Constructor c : cons) { - //System.out.println("found " + c.getParameterTypes()); - if (c.getParameterTypes().length == 0) { - System.out.println("found default"); - defaultCons = c; - c.setAccessible(true); - } else { - PApplet.println(c.getParameterTypes()); - } - } - ArrayList list = new ArrayList(); - try { - for (TableRow row : getRows()) { - Object item = target.newInstance(); - //Object item = defaultCons.newInstance(new Object[] { }); - for (Field field : inuse) { - String name = field.getName(); - if (field.getType() == String.class) { - field.set(item, row.getString(name)); - - } else if (field.getType() == Integer.TYPE) { - field.setInt(item, row.getInt(name)); - - } else if (field.getType() == Long.TYPE) { - field.setLong(item, row.getLong(name)); - - } else if (field.getType() == Float.TYPE) { - field.setFloat(item, row.getFloat(name)); - - } else if (field.getType() == Double.TYPE) { - field.setDouble(item, row.getDouble(name)); - - } else if (field.getType() == Boolean.TYPE) { - String content = row.getString(name); - if (content != null) { - // Only bother setting if it's true, - // otherwise false by default anyway. - if (content.toLowerCase().equals("true") || - content.equals("1")) { - field.setBoolean(item, true); - } - } -// if (content == null) { -// field.setBoolean(item, false); // necessary? -// } else if (content.toLowerCase().equals("true")) { -// field.setBoolean(item, true); -// } else if (content.equals("1")) { -// field.setBoolean(item, true); -// } else { -// field.setBoolean(item, false); // necessary? -// } - } else if (field.getType() == Character.TYPE) { - String content = row.getString(name); - if (content != null && content.length() > 0) { - // Otherwise set to \0 anyway - field.setChar(item, content.charAt(0)); - } - } - } - list.add(item); - } - } catch (InstantiationException e) { - e.printStackTrace(); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } catch (IllegalArgumentException e) { - e.printStackTrace(); -// } catch (InvocationTargetException e) { -// e.printStackTrace(); - } - Object[] outgoing = new Object[list.size()]; - return list.toArray(outgoing); + public void save(File file, String options) throws IOException { + save(new FileOutputStream(file), checkOptions(file, options)); } - */ - public void writeTSV(PrintWriter writer) { + public void save(OutputStream output, String options) { + PrintWriter writer = PApplet.createWriter(output); + String[] opts = PApplet.fixOptions(options); + for (String opt : opts) { + if (opt.equals("csv")) { + writeCSV(writer); + } else if (opt.equals("tsv")) { + writeTSV(writer); + } else if (opt.equals("html")) { + writeHTML(writer); + } else { + throw new IllegalArgumentException("'" + opt + "' not understood. " + + "Only csv, tsv, and html are " + + "accepted as save parameters"); + } + } + writer.close(); + } + + + protected void writeTSV(PrintWriter writer) { if (columnTitles != null) { for (int col = 0; col < columns.length; col++) { if (col != 0) { @@ -918,7 +712,7 @@ public class Table { } - public void writeCSV(PrintWriter writer) { + protected void writeCSV(PrintWriter writer) { if (columnTitles != null) { for (int col = 0; col < columns.length; col++) { if (col != 0) { @@ -987,21 +781,29 @@ public class Table { } - public void writeHTML(PrintWriter writer) { - writer.println(""); + protected void writeHTML(PrintWriter writer) { + writer.println(""); + + writer.println(""); + writer.println(" "); + writer.println(""); + + writer.println(""); + writer.println("
"); for (int row = 0; row < getRowCount(); row++) { - writer.println(" "); + writer.println(" "); for (int col = 0; col < getColumnCount(); col++) { String entry = getString(row, col); - writer.print(" "); - writer.println(""); + writer.println(" "); } - writer.println(" "); + writer.println(" "); } - writer.println("
"); + writer.print(" "); writeEntryHTML(writer, entry); -// String clean = (entry == null) ? "" : HTMLFairy.encodeEntities(entry); -// writer.println(" " + clean + "
"); + writer.println(" "); + writer.println(""); + + writer.println(""); writer.flush(); } @@ -1009,10 +811,23 @@ public class Table { protected void writeEntryHTML(PrintWriter writer, String entry) { //char[] chars = entry.toCharArray(); for (char c : entry.toCharArray()) { //chars) { - if (c < 32 || c > 127) { - writer.print("&#"); - writer.print((int) c); - writer.print(';'); + if (c == '<') { + writer.print("<"); + } else if (c == '>') { + writer.print(">"); + } else if (c == '&') { + writer.print("&"); + } else if (c == '\'') { + writer.print("'"); + } else if (c == '"') { + writer.print("""); + + // not necessary with UTF-8? +// } else if (c < 32 || c > 127) { +// writer.print("&#"); +// writer.print((int) c); +// writer.print(';'); + } else { writer.print(c); } @@ -1020,34 +835,6 @@ public class Table { } - /** - * Write this table as a TSV file. - * Exceptions will be printed, but not thrown. - * @param file the location to write to. - * @return true if written successfully - */ - public boolean writeCSV(File file) { - try { - writeCSV(new PrintWriter(new FileWriter(file))); - } catch (IOException e) { - e.printStackTrace(); - return false; - } - return true; - } - - - public boolean writeTSV(File file) { - try { - writeTSV(new PrintWriter(new FileWriter(file))); - } catch (IOException e) { - e.printStackTrace(); - return false; - } - return true; - } - - // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . @@ -1115,15 +902,15 @@ public class Table { } - public void removeColumn(String dead) { - removeColumn(getColumnIndex(dead)); + public void removeColumn(String columnName) { + removeColumn(getColumnIndex(columnName)); } - public void removeColumn(int index) { + public void removeColumn(int column) { Object[] temp = new Object[columns.length + 1]; - System.arraycopy(columns, 0, temp, 0, index); - System.arraycopy(columns, index+1, temp, index, (columns.length - index) + 1); + System.arraycopy(columns, 0, temp, 0, column); + System.arraycopy(columns, column+1, temp, column, (columns.length - column) + 1); columns = temp; } @@ -1280,7 +1067,7 @@ public class Table { * Set the entire table to a specific data type. */ public void setTableType(String type) { - for (int col = 0; col < columns.length; col++) { + for (int col = 0; col < getColumnCount(); col++) { setColumnType(col, type); } } @@ -1344,21 +1131,21 @@ public class Table { } - public int getColumnIndex(String name) { - return getColumnIndex(name, true); + public int getColumnIndex(String columnName) { + return getColumnIndex(columnName, true); } /** * Get the index of a column. * @param name Name of the column. - * @param report Whether to print to System.err if the column wasn't found. + * @param report Whether to throw an exception if the column wasn't found. * @return index of the found column, or -1 if not found. */ protected int getColumnIndex(String name, boolean report) { if (columnTitles == null) { if (report) { - System.err.println("Can't get column indices because no column titles are set."); + throw new IllegalArgumentException("This table has no header, so no column titles are set."); } return -1; } @@ -1373,7 +1160,9 @@ public class Table { Integer index = columnIndices.get(name); if (index == null) { if (report) { - System.err.println("No column named '" + name + "' was found."); + // Throws an exception here because the name is known and therefore most useful. + // (Rather than waiting for it to fail inside, say, getInt()) + throw new IllegalArgumentException("This table has no column named '" + name + "'"); } return -1; } @@ -1439,17 +1228,19 @@ public class Table { } - public void addRow() { + public TableRow addRow() { setRowCount(rowCount + 1); + return new RowPointer(this, rowCount - 1); } - public void addRow(String[] columns) { - setRow(getRowCount(), columns); + public TableRow addRow(String[] columnData) { + setRow(getRowCount(), columnData); + return new RowPointer(this, rowCount - 1); } - public void insertRow(int insert, String[] data) { + public void insertRow(int insert, String[] columnData) { for (int col = 0; col < columns.length; col++) { switch (columnTypes[col]) { case CATEGORICAL: @@ -1490,7 +1281,7 @@ public class Table { } } } - setRow(insert, data); + setRow(insert, columnData); rowCount++; } @@ -1600,7 +1391,85 @@ public class Table { } - public void convertRow(DataOutputStream output, String[] pieces) throws IOException { + // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + + + // converts a TSV or CSV file to binary.. do not use + protected void convertBasic(BufferedReader reader, boolean tsv, + File outputFile) throws IOException { + FileOutputStream fos = new FileOutputStream(outputFile); + BufferedOutputStream bos = new BufferedOutputStream(fos, 16384); + DataOutputStream output = new DataOutputStream(bos); + output.writeInt(0); // come back for row count + output.writeInt(getColumnCount()); + if (columnTitles != null) { + output.writeBoolean(true); + for (String title : columnTitles) { + output.writeUTF(title); + } + } else { + output.writeBoolean(false); + } + for (int type : columnTypes) { + output.writeInt(type); + } + + String line = null; + //setRowCount(1); + int prev = -1; + int row = 0; + while ((line = reader.readLine()) != null) { + convertRow(output, tsv ? PApplet.split(line, '\t') : splitLineCSV(line)); + row++; + + if (row % 10000 == 0) { + if (row < rowCount) { + int pct = (100 * row) / rowCount; + if (pct != prev) { + System.out.println(pct + "%"); + prev = pct; + } + } +// try { +// Thread.sleep(5); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } + } + } + // shorten or lengthen based on what's left +// if (row != getRowCount()) { +// setRowCount(row); +// } + + // has to come afterwards, since these tables get built out during the conversion + int col = 0; + for (HashMapBlows hmb : columnCategories) { + if (hmb == null) { + output.writeInt(0); + } else { + hmb.write(output); + hmb.writeln(PApplet.createWriter(new File(columnTitles[col] + ".categories"))); +// output.writeInt(hmb.size()); +// for (Map.Entry e : hmb.entrySet()) { +// output.writeUTF(e.getKey()); +// output.writeInt(e.getValue()); +// } + } + col++; + } + + output.flush(); + output.close(); + + // come back and write the row count + RandomAccessFile raf = new RandomAccessFile(outputFile, "rw"); + raf.writeInt(rowCount); + raf.close(); + } + + + protected void convertRow(DataOutputStream output, String[] pieces) throws IOException { if (pieces.length > getColumnCount()) { throw new IllegalArgumentException("Row with too many columns: " + PApplet.join(pieces, ",")); @@ -1662,7 +1531,8 @@ public class Table { } - protected void convertRowCol(DataOutputStream output, int row, int col, String piece) { + /* + private void convertRowCol(DataOutputStream output, int row, int col, String piece) { switch (columnTypes[col]) { case STRING: String[] stringData = (String[]) columns[col]; @@ -1696,6 +1566,7 @@ public class Table { throw new IllegalArgumentException("That's not a valid column type."); } } + */ // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . @@ -1703,101 +1574,142 @@ public class Table { protected RowIterator rowIterator; + /** + * Note that this one iterator instance is shared by any calls to iterate + * the rows of this table. This is very efficient, but not thread-safe. + * If you want to iterate in a multi-threaded manner, don't use the iterator. + */ public Iterable getRows() { return new Iterable() { public Iterator iterator() { - if (rowIterator == null) { - rowIterator = new RowIterator(); - } - rowIterator.reset(); - return rowIterator; + if (rowIterator == null) { + rowIterator = new RowIterator(Table.this); + } else { + rowIterator.reset(); + } + return rowIterator; } }; } - /** - * Note that this one iterator instance is shared by any calls to iterate the - * rows of this table. This is very efficient, but not very thread-safe. If - * you want to iterate in a multi-threaded manner, use createIterator(). - */ -// public Iterator iterator() { -// if (rowIterator == null) { -// rowIterator = new RowIterator(); -// } -// rowIterator.reset(); -// return rowIterator; -// } - - - public Iterator createIterator() { - return new RowIterator(); - } - - - // temporary objects inside loop! garbage collection! argh! -// public Iterator iterator() { -// return new RowIterator(); -// } - - - class RowIterator implements Iterator { + static class RowPointer implements TableRow { + Table table; int row; - TableRow tableRow = new TableRow() { - public String getString(int column) { - return Table.this.getString(row, column); - } - public String getString(String columnName) { - return Table.this.getString(row, columnName); - } + public RowPointer(Table table, int row) { + this.table = table; + this.row = row; + } - public int getInt(int column) { - return Table.this.getInt(row, column); - } + public void setRow(int row) { + this.row = row; + } - public int getInt(String columnName) { - return Table.this.getInt(row, columnName); - } + public String getString(int column) { + return table.getString(row, column); + } - public long getLong(int column) { - return Table.this.getLong(row, column); - } + public String getString(String columnName) { + return table.getString(row, columnName); + } - public long getLong(String columnName) { - return Table.this.getLong(row, columnName); - } + public int getInt(int column) { + return table.getInt(row, column); + } - public float getFloat(int column) { - return Table.this.getFloat(row, column); - } + public int getInt(String columnName) { + return table.getInt(row, columnName); + } - public float getFloat(String columnName) { - return Table.this.getFloat(row, columnName); - } + public long getLong(int column) { + return table.getLong(row, column); + } - public double getDouble(int column) { - return Table.this.getDouble(row, column); - } + public long getLong(String columnName) { + return table.getLong(row, columnName); + } - public double getDouble(String columnName) { - return Table.this.getDouble(row, columnName); - } - }; + public float getFloat(int column) { + return table.getFloat(row, column); + } + + public float getFloat(String columnName) { + return table.getFloat(row, columnName); + } + + public double getDouble(int column) { + return table.getDouble(row, column); + } + + public double getDouble(String columnName) { + return table.getDouble(row, columnName); + } + + public void setString(int column, String value) { + table.setString(row, column, value); + } + + public void setString(String columnName, String value) { + table.setString(row, columnName, value); + } + + public void setInt(int column, int value) { + table.setInt(row, column, value); + } + + public void setInt(String columnName, int value) { + table.setInt(row, columnName, value); + } + + public void setLong(int column, long value) { + table.setLong(row, column, value); + } + + public void setLong(String columnName, long value) { + table.setLong(row, columnName, value); + } + + public void setFloat(int column, float value) { + table.setFloat(row, column, value); + } + + public void setFloat(String columnName, float value) { + table.setFloat(row, columnName, value); + } + + public void setDouble(int column, double value) { + table.setDouble(row, column, value); + } + + @Override + public void setDouble(String columnName, double value) { + table.setDouble(row, columnName, value); + } + } + + + static class RowIterator implements Iterator { + int row; + RowPointer rp; + Table table; + + public RowIterator(Table table) { + row = -1; + rp = new RowPointer(table, row); + } public void remove() { - removeRow(row); + table.removeRow(row); } public TableRow next() { - ++row; -// iteratorRow.setRow(row); -// return iteratorRow; - return tableRow; + rp.setRow(++row); + return rp; } public boolean hasNext() { - return row+1 < getRowCount(); + return row+1 < table.getRowCount(); } public void reset() { @@ -1911,6 +1823,22 @@ public class Table { throw new RuntimeException(e); } } + + public void setString(int column, String value) { immutable(); } + public void setString(String columnName, String value) { immutable(); } + public void setInt(int column, int value) { immutable(); } + public void setInt(String columnName, int value) { immutable(); } + public void setLong(int column, long value) { immutable(); } + public void setLong(String columnName, long value) { immutable(); } + public void setFloat(int column, float value) { immutable(); } + public void setFloat(String columnName, float value) { immutable(); } + public void setDouble(int column, double value) { immutable(); } + public void setDouble(String columnName, double value) { immutable(); } + + private void immutable() { + throw new IllegalArgumentException("This TableRow cannot be modified."); + } + }; } @@ -1946,9 +1874,9 @@ public class Table { } - public void setInt(int row, int column, int what) { + public void setInt(int row, int column, int value) { if (columnTypes[column] == STRING) { - setString(row, column, String.valueOf(what)); + setString(row, column, String.valueOf(value)); } else { checkSize(row, column); @@ -1956,11 +1884,17 @@ public class Table { throw new IllegalArgumentException("Column " + column + " is not an int column."); } int[] intData = (int[]) columns[column]; - intData[row] = what; + intData[row] = value; } } + public void setInt(int row, String columnName, int value) { + setInt(row, getColumnIndex(columnName), value); + } + + + public int[] getIntColumn(String name) { int col = getColumnIndex(name); return (col == -1) ? null : getIntColumn(col); @@ -2016,9 +1950,9 @@ public class Table { } - public void setLong(int row, int column, long what) { + public void setLong(int row, int column, long value) { if (columnTypes[column] == STRING) { - setString(row, column, String.valueOf(what)); + setString(row, column, String.valueOf(value)); } else { checkSize(row, column); @@ -2026,11 +1960,16 @@ public class Table { throw new IllegalArgumentException("Column " + column + " is not a 'long' column."); } long[] longData = (long[]) columns[column]; - longData[row] = what; + longData[row] = value; } } + public void setLong(int row, String columnName, long value) { + setLong(row, getColumnIndex(columnName), value); + } + + public long[] getLongColumn(String name) { int col = getColumnIndex(name); return (col == -1) ? null : getLongColumn(col); @@ -2087,9 +2026,9 @@ public class Table { } - public void setFloat(int row, int column, float what) { + public void setFloat(int row, int column, float value) { if (columnTypes[column] == STRING) { - setString(row, column, String.valueOf(what)); + setString(row, column, String.valueOf(value)); } else { checkSize(row, column); @@ -2097,11 +2036,16 @@ public class Table { throw new IllegalArgumentException("Column " + column + " is not a float column."); } float[] longData = (float[]) columns[column]; - longData[row] = what; + longData[row] = value; } } + public void setFloat(int row, String columnName, float value) { + setFloat(row, getColumnIndex(columnName), value); + } + + public float[] getFloatColumn(String name) { int col = getColumnIndex(name); return (col == -1) ? null : getFloatColumn(col); @@ -2157,9 +2101,9 @@ public class Table { } - public void setDouble(int row, int column, double what) { + public void setDouble(int row, int column, double value) { if (columnTypes[column] == STRING) { - setString(row, column, String.valueOf(what)); + setString(row, column, String.valueOf(value)); } else { checkSize(row, column); @@ -2167,11 +2111,16 @@ public class Table { throw new IllegalArgumentException("Column " + column + " is not a 'double' column."); } double[] doubleData = (double[]) columns[column]; - doubleData[row] = what; + doubleData[row] = value; } } + public void setDouble(int row, String columnName, double value) { + setDouble(row, getColumnIndex(columnName), value); + } + + public double[] getDoubleColumn(String name) { int col = getColumnIndex(name); return (col == -1) ? null : getDoubleColumn(col); @@ -2285,19 +2234,19 @@ public class Table { } - public void setString(int row, int column, String what) { + public void setString(int row, int column, String value) { checkSize(row, column); if (columnTypes[column] != STRING) { throw new IllegalArgumentException("Column " + column + " is not a String column."); } String[] stringData = (String[]) columns[column]; - stringData[row] = what; + stringData[row] = value; } - public void setString(int row, String columnName, String what) { + public void setString(int row, String columnName, String value) { int column = checkColumnIndex(columnName); - setString(row, column, what); + setString(row, column, value); } @@ -2328,40 +2277,40 @@ public class Table { // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . - /** - * Set all 'null' entries to "" (zero length String objects). - * If columns are typed, then this will only apply to String columns. - */ - public void makeNullEmpty() { - for (int col = 0; col < columns.length; col++) { - if (columnTypes[col] == STRING) { - String[] stringData = (String[]) columns[col]; - for (int row = 0; row < rowCount; row++) { - if (stringData[row] == null) { - stringData[row] = ""; - } - } - } - } - } - - - /** - * Set all "" entries (zero length String objects) to null values. - * If columns are typed, then this will only apply to String columns. - */ - public void makeEmptyNull() { - for (int col = 0; col < columns.length; col++) { - if (columnTypes[col] == STRING) { - String[] stringData = (String[]) columns[col]; - for (int row = 0; row < rowCount; row++) { - if (stringData[row] != null && stringData[row].length() == 0) { - stringData[row] = null; - } - } - } - } - } +// /** +// * Set all 'null' entries to "" (zero length String objects). +// * If columns are typed, then this will only apply to String columns. +// */ +// public void makeNullEmpty() { +// for (int col = 0; col < columns.length; col++) { +// if (columnTypes[col] == STRING) { +// String[] stringData = (String[]) columns[col]; +// for (int row = 0; row < rowCount; row++) { +// if (stringData[row] == null) { +// stringData[row] = ""; +// } +// } +// } +// } +// } +// +// +// /** +// * Set all "" entries (zero length String objects) to null values. +// * If columns are typed, then this will only apply to String columns. +// */ +// public void makeEmptyNull() { +// for (int col = 0; col < columns.length; col++) { +// if (columnTypes[col] == STRING) { +// String[] stringData = (String[]) columns[col]; +// for (int row = 0; row < rowCount; row++) { +// if (stringData[row] != null && stringData[row].length() == 0) { +// stringData[row] = null; +// } +// } +// } +// } +// } // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . @@ -2433,37 +2382,30 @@ public class Table { } - public void removeTokens(String tokens, String column) { - removeTokens(tokens, getColumnIndex(column)); + public void removeTokens(String tokens, String columnName) { + removeTokens(tokens, getColumnIndex(columnName)); } - // TODO this isn't i18n correct, and it's a dumb implementation -// public void removeLetters(int column) { -// String alphabet = "abcdefghijklmnopqrstuvwxyz"; -// removeTokens(alphabet + alphabet.toUpperCase(), column); -// } - - // . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . /** * Return the row that contains the first String that matches. - * @param what the String to match + * @param value the String to match * @param column the column to search */ - public int findRow(String what, int column) { + public int findRowIndex(String value, int column) { checkBounds(-1, column); if (columnTypes[column] == STRING) { String[] stringData = (String[]) columns[column]; - if (what == null) { + if (value == null) { for (int row = 0; row < rowCount; row++) { if (stringData[row] == null) return row; } } else { for (int row = 0; row < rowCount; row++) { - if (stringData[row] != null && stringData[row].equals(what)) { + if (stringData[row] != null && stringData[row].equals(value)) { return row; } } @@ -2472,10 +2414,10 @@ public class Table { for (int row = 0; row < rowCount; row++) { String str = getString(row, column); if (str == null) { - if (what == null) { + if (value == null) { return row; } - } else if (str.equals(what)) { + } else if (str.equals(value)) { return row; } } @@ -2486,28 +2428,28 @@ public class Table { /** * Return the row that contains the first String that matches. - * @param what the String to match + * @param value the String to match * @param columnName the column to search */ - public int findRow(String what, String columnName) { - return findRow(what, getColumnIndex(columnName)); + public int findRowIndex(String value, String columnName) { + return findRowIndex(value, getColumnIndex(columnName)); } /** * Return a list of rows that contain the String passed in. If there are no * matches, a zero length array will be returned (not a null array). - * @param what the String to match + * @param value the String to match * @param column the column to search */ - public int[] findRows(String what, int column) { + public int[] findRowIndices(String value, int column) { int[] outgoing = new int[rowCount]; int count = 0; checkBounds(-1, column); if (columnTypes[column] == STRING) { String[] stringData = (String[]) columns[column]; - if (what == null) { + if (value == null) { for (int row = 0; row < rowCount; row++) { if (stringData[row] == null) { outgoing[count++] = row; @@ -2515,7 +2457,7 @@ public class Table { } } else { for (int row = 0; row < rowCount; row++) { - if (stringData[row] != null && stringData[row].equals(what)) { + if (stringData[row] != null && stringData[row].equals(value)) { outgoing[count++] = row; } } @@ -2524,10 +2466,10 @@ public class Table { for (int row = 0; row < rowCount; row++) { String str = getString(row, column); if (str == null) { - if (what == null) { + if (value == null) { outgoing[count++] = row; } - } else if (str.equals(what)) { + } else if (str.equals(value)) { outgoing[count++] = row; } } @@ -2539,11 +2481,11 @@ public class Table { /** * Return a list of rows that contain the String passed in. If there are no * matches, a zero length array will be returned (not a null array). - * @param what the String to match + * @param value the String to match * @param columnName the column to search */ - public int[] findRows(String what, String columnName) { - return findRows(what, getColumnIndex(columnName)); + public int[] findRowIndices(String value, String columnName) { + return findRowIndices(value, getColumnIndex(columnName)); } @@ -2552,10 +2494,10 @@ public class Table { /** * Return the row that contains the first String that matches. - * @param what the String to match + * @param regexp the String to match * @param column the column to search */ - public int matchRow(String regexp, int column) { + public int matchRowIndex(String regexp, int column) { checkBounds(-1, column); if (columnTypes[column] == STRING) { String[] stringData = (String[]) columns[column]; @@ -2583,8 +2525,8 @@ public class Table { * @param what the String to match * @param columnName the column to search */ - public int matchRow(String what, String columnName) { - return matchRow(what, getColumnIndex(columnName)); + public int matchRowIndex(String what, String columnName) { + return matchRowIndex(what, getColumnIndex(columnName)); } @@ -2594,7 +2536,7 @@ public class Table { * @param what the String to match * @param column the column to search */ - public int[] matchRows(String regexp, int column) { + public int[] matchRowIndices(String regexp, int column) { int[] outgoing = new int[rowCount]; int count = 0; @@ -2626,8 +2568,8 @@ public class Table { * @param what the String to match * @param columnName the column to search */ - public int[] matchRows(String what, String columnName) { - return matchRows(what, getColumnIndex(columnName)); + public int[] matchRowIndices(String what, String columnName) { + return matchRowIndices(what, getColumnIndex(columnName)); } @@ -2635,11 +2577,68 @@ public class Table { /** - * Return a list of rows that contain the String passed in. If there are no - * matches, a zero length array will be returned (not a null array). - * @param what the String to match - * @param column the column to search + * Replace a String with another. Set empty entries null by using + * replace("", null) or use replace(null, "") to go the other direction. + * If this is a typed table, only String columns will be modified. + * @param orig + * @param replacement */ + public void replace(String orig, String replacement) { + for (int col = 0; col < columns.length; col++) { + replace(orig, replacement, col); + } + } + + + public void replace(String orig, String replacement, int col) { + if (columnTypes[col] == STRING) { + String[] stringData = (String[]) columns[col]; + for (int row = 0; row < rowCount; row++) { + if (stringData[row].equals(orig)) { + stringData[row] = replacement; + } + } + } + } + + + public void replace(String orig, String replacement, String colName) { + replace(orig, replacement, getColumnIndex(colName)); + } + + +// public void makeNullEmpty() { +// for (int col = 0; col < columns.length; col++) { +// if (columnTypes[col] == STRING) { +// String[] stringData = (String[]) columns[col]; +// for (int row = 0; row < rowCount; row++) { +// if (stringData[row] == null) { +// stringData[row] = ""; +// } +// } +// } +// } +// } +// +// +// /** +// * Set all "" entries (zero length String objects) to null values. +// * If columns are typed, then this will only apply to String columns. +// */ +// public void makeEmptyNull() { +// for (int col = 0; col < columns.length; col++) { +// if (columnTypes[col] == STRING) { +// String[] stringData = (String[]) columns[col]; +// for (int row = 0; row < rowCount; row++) { +// if (stringData[row] != null && stringData[row].length() == 0) { +// stringData[row] = null; +// } +// } +// } +// } +// } + + public void replaceAll(String regex, String replacement, int column) { checkBounds(-1, column); if (columnTypes[column] == STRING) { diff --git a/core/src/processing/data/TableRow.java b/core/src/processing/data/TableRow.java index 5a10e3aa4..d01dcd1e7 100644 --- a/core/src/processing/data/TableRow.java +++ b/core/src/processing/data/TableRow.java @@ -12,4 +12,15 @@ public interface TableRow { public float getFloat(String columnName); public double getDouble(int column); public double getDouble(String columnName); + + public void setString(int column, String value); + public void setString(String columnName, String value); + public void setInt(int column, int value); + public void setInt(String columnName, int value); + public void setLong(int column, long value); + public void setLong(String columnName, long value); + public void setFloat(int column, float value); + public void setFloat(String columnName, float value); + public void setDouble(int column, double value); + public void setDouble(String columnName, double value); } diff --git a/core/todo.txt b/core/todo.txt index 96d35f9f4..5b7933564 100644 --- a/core/todo.txt +++ b/core/todo.txt @@ -1,6 +1,10 @@ 0216 core X Add clear() to replace background(0, 0, 0, 0) X http://code.google.com/p/processing/issues/detail?id=1446 +o add loadType() and saveType()... get working with shapes, etc +X heading2D()? weird.. changed to heading() +X http://toxiclibs.org/docs/core/toxi/geom/Vec3D.html +X http://code.google.com/p/processing/issues/detail?id=987 PShape s = createShape(); @@ -131,12 +135,8 @@ _ http://code.google.com/p/processing/issues/detail?id=1407 _ add "CGAffineTransformInvert: singular matrix" problem to the Wiki _ http://code.google.com/p/processing/issues/detail?id=1363 -_ heading2D()? weird.. angle? -_ http://toxiclibs.org/docs/core/toxi/geom/Vec3D.html -_ http://code.google.com/p/processing/issues/detail?id=987 - _ "translate, or this variation of it" when using text(s, x, y, z) accidentally -_ change to be th text command +_ change to be the text command _ OutOfMemory in image() _ http://code.google.com/p/processing/issues/detail?id=1353 @@ -146,7 +146,7 @@ _ http://code.google.com/p/processing/issues/detail?id=1262 _ hint(OPENGL_ERRORS) should be the opposite to enable the reporting, no? -_ fillMode(NONZERO) and fillMode(ODD) to replace sold(boolean)? +_ fillMode(NONZERO) and fillMode(ODD) to replace solid(boolean)? keeping these public for now get/setNormal @@ -187,8 +187,6 @@ _ toString(-1)? a new method? _ format(2), format(4)... format() -> default to 2 params _ get back to PhiLho once finished -_ add loadType() and saveType()... get working with shapes, etc - api to be fixed/removed _ remove PImage.delete() and friends from PImage, Movie, etc. _ delete()/dispose() being used in the movie @@ -620,6 +618,7 @@ _ before rewriting all of createInput() LATER (post 2.0) +_ create PVec2D and PVec3D as separate classes _ gui priorities _ + Label _ + Button diff --git a/todo.txt b/todo.txt index 06a4cdadb..9f373f749 100644 --- a/todo.txt +++ b/todo.txt @@ -16,6 +16,9 @@ X make sure javadoc builds and uploads properly X error checker broken in experimental mode (fix from Manindra) X http://code.google.com/p/processing/issues/detail?id=1449 +_ Color coding for if/else in Processing IDE doesn't match +_ http://code.google.com/p/processing/issues/detail?id=1457 + earlier X include debug mode as 'experimental'? X DebugMode throwing exception about breakpoints when trying to save