From: James Ahlborn Date: Sun, 12 Nov 2006 19:41:03 +0000 (+0000) Subject: better import/copy column setup; added ability for users to further customize import... X-Git-Tag: rel_1_1_8~6 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=13c39138cfd5e54238994893d25a40042c6740e8;p=jackcess.git better import/copy column setup; added ability for users to further customize import/copy behavior through ImportFilter (fix #1593465) git-svn-id: https://svn.code.sf.net/p/jackcess/code/jackcess/trunk@134 f203690c-595d-4dc9-a70b-905162fa7fd2 --- diff --git a/src/java/com/healthmarketscience/jackcess/DataType.java b/src/java/com/healthmarketscience/jackcess/DataType.java index 0d4e648..5cc9a2f 100644 --- a/src/java/com/healthmarketscience/jackcess/DataType.java +++ b/src/java/com/healthmarketscience/jackcess/DataType.java @@ -292,11 +292,11 @@ public enum DataType { if(rtn.isVariableLength()) { // make sure size is reasonable - if(lengthInUnits > (rtn.getMaxSize() * rtn.getUnitSize())) { + if((lengthInUnits * rtn.getUnitSize()) > rtn.getMaxSize()) { // try alternate types DataType altRtn = ALT_SQL_TYPES.get(sqlType); if(altRtn != null) { - if(lengthInUnits <= (altRtn.getMaxSize() * altRtn.getUnitSize())) { + if((lengthInUnits * altRtn.getUnitSize()) <= altRtn.getMaxSize()) { // use alternate type rtn = altRtn; } diff --git a/src/java/com/healthmarketscience/jackcess/Database.java b/src/java/com/healthmarketscience/jackcess/Database.java index 3529e62..30a835b 100644 --- a/src/java/com/healthmarketscience/jackcess/Database.java +++ b/src/java/com/healthmarketscience/jackcess/Database.java @@ -616,60 +616,53 @@ public class Database * @param name Name of the new table to create * @param source ResultSet to copy from */ - public void copyTable(String name, ResultSet source) throws SQLException, IOException { + public void copyTable(String name, ResultSet source) + throws SQLException, IOException + { + copyTable(name, source, SimpleImportFilter.INSTANCE); + } + + /** + * Copy an existing JDBC ResultSet into a new table in this database + * @param name Name of the new table to create + * @param source ResultSet to copy from + * @param filter valid import filter + */ + public void copyTable(String name, ResultSet source, ImportFilter filter) + throws SQLException, IOException + { ResultSetMetaData md = source.getMetaData(); List columns = new LinkedList(); - int textCount = 0; - int totalSize = 0; - // FIXME, there is some ugly (and broken) logic here... - for (int i = 1; i <= md.getColumnCount(); i++) { - DataType accessColumnType = DataType.fromSQLType(md.getColumnType(i)); - switch (accessColumnType) { - case BYTE: - case INT: - case LONG: - case MONEY: - case FLOAT: - case NUMERIC: - totalSize += 4; - break; - case DOUBLE: - case SHORT_DATE_TIME: - totalSize += 8; - break; - case BINARY: - case TEXT: - case OLE: - case MEMO: - textCount++; - break; - } - } - short textSize = 0; - if (textCount > 0) { - textSize = (short) ((JetFormat.MAX_RECORD_SIZE - totalSize) / textCount); - if (textSize > JetFormat.TEXT_FIELD_MAX_LENGTH) { - textSize = JetFormat.TEXT_FIELD_MAX_LENGTH; - } - } for (int i = 1; i <= md.getColumnCount(); i++) { Column column = new Column(); column.setName(escape(md.getColumnName(i))); - column.setType(DataType.fromSQLType(md.getColumnType(i))); - if (column.getType() == DataType.TEXT) { - column.setLength(textSize); + int lengthInUnits = md.getColumnDisplaySize(i); + column.setSQLType(md.getColumnType(i), lengthInUnits); + DataType type = column.getType(); + if(type.isVariableLength()) { + column.setLengthInUnits((short)lengthInUnits); + } + if(type.getHasScalePrecision()) { + int scale = md.getScale(i); + int precision = md.getPrecision(i); + if(type.isValidScale(scale)) { + column.setScale((byte)scale); + } + if(type.isValidPrecision(precision)) { + column.setPrecision((byte)precision); + } } columns.add(column); } - createTable(escape(name), columns); + createTable(escape(name), filter.filterColumns(columns, md)); Table table = getTable(escape(name)); - List rows = new ArrayList(); + List rows = new ArrayList(COPY_TABLE_BATCH_SIZE); while (source.next()) { Object[] row = new Object[md.getColumnCount()]; for (int i = 0; i < row.length; i++) { row[i] = source.getObject(i + 1); } - rows.add(row); + rows.add(filter.filterRow(row)); if (rows.size() == COPY_TABLE_BATCH_SIZE) { table.addRows(rows); rows.clear(); @@ -687,12 +680,26 @@ public class Database * @param delim Regular expression representing the delimiter string. */ public void importFile(String name, File f, String delim) - throws IOException + throws IOException + { + importFile(name, f, delim, SimpleImportFilter.INSTANCE); + } + + /** + * Copy a delimited text file into a new table in this database + * @param name Name of the new table to create + * @param f Source file to import + * @param delim Regular expression representing the delimiter string. + * @param filter valid import filter + */ + public void importFile(String name, File f, String delim, + ImportFilter filter) + throws IOException { BufferedReader in = null; try { in = new BufferedReader(new FileReader(f)); - importReader(name, in, delim); + importReader(name, in, delim, filter); } finally { if (in != null) { try { @@ -704,7 +711,6 @@ public class Database } } - /** * Copy a delimited text file into a new table in this database * @param name Name of the new table to create @@ -712,7 +718,21 @@ public class Database * @param delim Regular expression representing the delimiter string. */ public void importReader(String name, BufferedReader in, String delim) - throws IOException + throws IOException + { + importReader(name, in, delim, SimpleImportFilter.INSTANCE); + } + + /** + * Copy a delimited text file into a new table in this database + * @param name Name of the new table to create + * @param in Source reader to import + * @param delim Regular expression representing the delimiter string. + * @param filter valid import filter + */ + public void importReader(String name, BufferedReader in, String delim, + ImportFilter filter) + throws IOException { String line = in.readLine(); if (line == null || line.trim().length() == 0) { @@ -728,41 +748,40 @@ public class Database List columns = new LinkedList(); String[] columnNames = line.split(delim); - short textSize = (short) ((JetFormat.MAX_RECORD_SIZE) / columnNames.length); - if (textSize > JetFormat.TEXT_FIELD_MAX_LENGTH) { - textSize = JetFormat.TEXT_FIELD_MAX_LENGTH; - } - for (int i = 0; i < columnNames.length; i++) { Column column = new Column(); column.setName(escape(columnNames[i])); column.setType(DataType.TEXT); - column.setLength(textSize); + column.setLength((short)DataType.TEXT.getMaxSize()); columns.add(column); } + + try { + createTable(tableName, filter.filterColumns(columns, null)); + Table table = getTable(tableName); + List rows = new ArrayList(COPY_TABLE_BATCH_SIZE); - createTable(tableName, columns); - Table table = getTable(tableName); - List rows = new ArrayList(); - - while ((line = in.readLine()) != null) - { - // - // Handle the situation where the end of the line - // may have null fields. We always want to add the - // same number of columns to the table each time. - // - String[] data = new String[columnNames.length]; - String[] splitData = line.split(delim); - System.arraycopy(splitData, 0, data, 0, splitData.length); - rows.add(data); - if (rows.size() == COPY_TABLE_BATCH_SIZE) { + while ((line = in.readLine()) != null) + { + // + // Handle the situation where the end of the line + // may have null fields. We always want to add the + // same number of columns to the table each time. + // + String[] data = new String[columnNames.length]; + String[] splitData = line.split(delim); + System.arraycopy(splitData, 0, data, 0, splitData.length); + rows.add(filter.filterRow(data)); + if (rows.size() == COPY_TABLE_BATCH_SIZE) { + table.addRows(rows); + rows.clear(); + } + } + if (rows.size() > 0) { table.addRows(rows); - rows.clear(); } - } - if (rows.size() > 0) { - table.addRows(rows); + } catch(SQLException e) { + throw (IOException)new IOException(e.getMessage()).initCause(e); } } diff --git a/src/java/com/healthmarketscience/jackcess/ImportFilter.java b/src/java/com/healthmarketscience/jackcess/ImportFilter.java new file mode 100644 index 0000000..f791639 --- /dev/null +++ b/src/java/com/healthmarketscience/jackcess/ImportFilter.java @@ -0,0 +1,39 @@ +// Copyright (c) 2006 Health Market Science, Inc. + +package com.healthmarketscience.jackcess; + +import java.io.IOException; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.List; + +/** + * Interface which allows customization of the behavior of the + * Database< import/copy methods. + * + * @author James Ahlborn + */ +public interface ImportFilter { + + /** + * The columns that should be used to create the imported table. + * @param destColumns the columns as determined by the import code, may be + * directly modified and returned + * @param srcColumns the sql metadata, only available if importing from a + * JDBC source + * @return the columns to use when creating the import table + */ + public List filterColumns(List destColumns, + ResultSetMetaData srcColumns) + throws SQLException, IOException; + + /** + * The desired values for the row. + * @param row the row data as determined by the import code, may be directly + * modified + * @return the row data as it should be written to the import table + */ + public Object[] filterRow(Object[] row) + throws SQLException, IOException; + +} diff --git a/src/java/com/healthmarketscience/jackcess/JetFormat.java b/src/java/com/healthmarketscience/jackcess/JetFormat.java index b1f0c25..8b01fc0 100644 --- a/src/java/com/healthmarketscience/jackcess/JetFormat.java +++ b/src/java/com/healthmarketscience/jackcess/JetFormat.java @@ -42,9 +42,11 @@ public abstract class JetFormat { /** Maximum size of a record minus OLE objects and Memo fields */ public static final int MAX_RECORD_SIZE = 1900; //2kb minus some overhead - + + /** the "unit" size for text fields */ + public static final short TEXT_FIELD_UNIT_SIZE = 2; /** Maximum size of a text field */ - public static final short TEXT_FIELD_MAX_LENGTH = 255 * 2; + public static final short TEXT_FIELD_MAX_LENGTH = 255 * TEXT_FIELD_UNIT_SIZE; /** Offset in the file that holds the byte describing the Jet format version */ private static final long OFFSET_VERSION = 20L; diff --git a/src/java/com/healthmarketscience/jackcess/SimpleImportFilter.java b/src/java/com/healthmarketscience/jackcess/SimpleImportFilter.java new file mode 100644 index 0000000..c604da5 --- /dev/null +++ b/src/java/com/healthmarketscience/jackcess/SimpleImportFilter.java @@ -0,0 +1,36 @@ +// Copyright (c) 2006 Health Market Science, Inc. + +package com.healthmarketscience.jackcess; + +import java.io.IOException; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.List; + +/** + * Simple concrete implementation of ImportFilter which just returns the given + * values. + * + * @author James Ahlborn + */ +public class SimpleImportFilter implements ImportFilter { + + public static final SimpleImportFilter INSTANCE = new SimpleImportFilter(); + + public SimpleImportFilter() { + } + + public List filterColumns(List destColumns, + ResultSetMetaData srcColumns) + throws SQLException, IOException + { + return destColumns; + } + + public Object[] filterRow(Object[] row) + throws SQLException, IOException + { + return row; + } + +}