From abd574dd1afe6bf7bb33f5e05c9ecfec624401bd Mon Sep 17 00:00:00 2001 From: James Ahlborn Date: Fri, 8 Sep 2006 18:48:32 +0000 Subject: [PATCH] clean up lots of cruft around datatypes; add more sanity checking on table creation; fix free space calculations git-svn-id: https://svn.code.sf.net/p/jackcess/code/jackcess/trunk@105 f203690c-595d-4dc9-a70b-905162fa7fd2 --- .../healthmarketscience/jackcess/Column.java | 275 +++++++++++------- .../jackcess/DataType.java | 121 +++++++- .../jackcess/Database.java | 25 +- .../healthmarketscience/jackcess/Index.java | 50 +++- .../jackcess/JetFormat.java | 6 +- .../healthmarketscience/jackcess/Table.java | 78 +++-- .../jackcess/UsageMap.java | 4 +- .../jackcess/DatabaseTest.java | 91 +++++- .../jackcess/TableTest.java | 3 +- 9 files changed, 478 insertions(+), 175 deletions(-) diff --git a/src/java/com/healthmarketscience/jackcess/Column.java b/src/java/com/healthmarketscience/jackcess/Column.java index 3f779c1..6650048 100644 --- a/src/java/com/healthmarketscience/jackcess/Column.java +++ b/src/java/com/healthmarketscience/jackcess/Column.java @@ -81,19 +81,14 @@ public class Column implements Comparable { private static final Pattern GUID_PATTERN = Pattern.compile("\\s*[{]([\\p{XDigit}]{8})-([\\p{XDigit}]{4})-([\\p{XDigit}]{4})-([\\p{XDigit}]{4})-([\\p{XDigit}]{12})[}]\\s*"); - /** default precision value for new numeric columns */ - public static final byte DEFAULT_PRECISION = 18; - /** default scale value for new numeric columns */ - public static final byte DEFAULT_SCALE = 0; - /** For text columns, whether or not they are compressed */ private boolean _compressedUnicode = false; /** Whether or not the column is of variable length */ private boolean _variableLength; /** Numeric precision */ - private byte _precision = DEFAULT_PRECISION; + private byte _precision; /** Numeric scale */ - private byte _scale = DEFAULT_SCALE; + private byte _scale; /** Data type */ private DataType _type; /** Format that the containing database is in */ @@ -136,7 +131,7 @@ public class Column implements Comparable { setType(DataType.fromByte(buffer.get(offset + format.OFFSET_COLUMN_TYPE))); _columnNumber = buffer.getShort(offset + format.OFFSET_COLUMN_NUMBER); _columnLength = buffer.getShort(offset + format.OFFSET_COLUMN_LENGTH); - if (_type == DataType.NUMERIC) { + if (_type.getHasScalePrecision()) { _precision = buffer.get(offset + format.OFFSET_COLUMN_PRECISION); _scale = buffer.get(offset + format.OFFSET_COLUMN_SCALE); } @@ -171,12 +166,22 @@ public class Column implements Comparable { } /** - * Also sets the length and the variable length flag, inferred from the type + * Also sets the length and the variable length flag, inferred from the + * type. For types with scale/precision, sets the scale and precision to + * default values. */ public void setType(DataType type) { _type = type; - setLength((short) size()); - setVariableLength(type.isVariableLength()); + if(!type.isVariableLength()) { + setLength((short)type.getFixedSize()); + } else if(!type.isLongValue()) { + setLength((short)type.getDefaultSize()); + } + setVariableLength(type.isVariableLength()); + if(type.getHasScalePrecision()) { + setScale((byte)type.getDefaultScale()); + setPrecision((byte)type.getDefaultPrecision()); + } } public DataType getType() { return _type; @@ -199,9 +204,6 @@ public class Column implements Comparable { } public void setPrecision(byte newPrecision) { - if((newPrecision < 1) || (newPrecision > 28)) { - throw new IllegalArgumentException("Precision must be from 1 to 28 inclusive"); - } _precision = newPrecision; } @@ -210,9 +212,6 @@ public class Column implements Comparable { } public void setScale(byte newScale) { - if((newScale < 1) || (newScale > 28)) { - throw new IllegalArgumentException("Scale must be from 0 to 28 inclusive"); - } _scale = newScale; } @@ -230,6 +229,48 @@ public class Column implements Comparable { public int getFixedDataOffset() { return _fixedDataOffset; } + + /** + * Checks that this column definition is valid. + * + * @throw IllegalArgumentException if this column definition is invalid. + */ + public void validate() { + if(getType() == null) { + throw new IllegalArgumentException("must have type"); + } + if((getName() == null) || (getName().trim().length() == 0)) { + throw new IllegalArgumentException("must have valid name"); + } + if(isVariableLength() != getType().isVariableLength()) { + throw new IllegalArgumentException("invalid variable length setting"); + } + + if(!isVariableLength()) { + if(getLength() != getType().getFixedSize()) { + throw new IllegalArgumentException("invalid fixed length size"); + } + } else if(!getType().isLongValue()) { + if((getLength() < 0) || (getLength() > getType().getMaxSize())) { + throw new IllegalArgumentException("var length out of range"); + } + } + + if(getType().getHasScalePrecision()) { + if((getScale() < getType().getMinScale()) || + (getScale() > getType().getMaxScale())) { + throw new IllegalArgumentException( + "Scale must be from " + getType().getMinScale() + " to " + + getType().getMaxScale() + " inclusive"); + } + if((getPrecision() < getType().getMinPrecision()) || + (getPrecision() > getType().getMaxPrecision())) { + throw new IllegalArgumentException( + "Precision must be from " + getType().getMinPrecision() + " to " + + getType().getMaxPrecision() + " inclusive"); + } + } + } /** * Deserialize a raw byte value for this column into an Object @@ -601,7 +642,14 @@ public class Column implements Comparable { * @param value Value of the LVAL column * @return A buffer containing the LVAL definition and the column value */ - public ByteBuffer writeLongValue(byte[] value) throws IOException { + public ByteBuffer writeLongValue(byte[] value, + int remainingRowLength) throws IOException + { + // FIXME, take remainingRowLength into account (don't always write inline) + + if(value.length > getType().getMaxSize()) { + throw new IOException("value too big for column"); + } ByteBuffer def = ByteBuffer.allocate(_format.SIZE_LONG_VALUE_DEF + value.length); def.order(ByteOrder.LITTLE_ENDIAN); ByteUtil.put3ByteInt(def, value.length); @@ -619,7 +667,8 @@ public class Column implements Comparable { * @param value Value of the LVAL column * @return A buffer containing the LVAL definition */ - public ByteBuffer writeLongValueInNewPage(byte[] value) throws IOException { + // FIXME, unused? + private ByteBuffer writeLongValueInNewPage(byte[] value) throws IOException { ByteBuffer lvalPage = _pageChannel.createPageBuffer(); lvalPage.put(PageTypes.DATA); //Page type lvalPage.put((byte) 1); //Unknown @@ -651,8 +700,10 @@ public class Column implements Comparable { * @param obj Object to serialize * @return A buffer containing the bytes */ - public ByteBuffer write(Object obj) throws IOException { - return write(obj, ByteOrder.LITTLE_ENDIAN); + public ByteBuffer write(Object obj, int remainingRowLength) + throws IOException + { + return write(obj, remainingRowLength, ByteOrder.LITTLE_ENDIAN); } /** @@ -661,62 +712,114 @@ public class Column implements Comparable { * @param order Order in which to serialize * @return A buffer containing the bytes */ - public ByteBuffer write(Object obj, ByteOrder order) throws IOException { - int size = size(); - if (_type == DataType.OLE) { - size += ((byte[]) obj).length; - } else if(_type == DataType.MEMO) { - byte[] encodedData = encodeUncompressedText(toCharSequence(obj)).array(); - size += encodedData.length; - obj = encodedData; - } else if(_type == DataType.TEXT) { - size = getLength(); + public ByteBuffer write(Object obj, int remainingRowLength, ByteOrder order) + throws IOException + { + if(!isVariableLength()) { + return writeFixedLengthField(obj, order); } + + // var length column + if(!getType().isLongValue()) { + + // FIXME, take remainingRowLength into account? overflow pages? + + // this is an "inline" var length field + switch(getType()) { + case TEXT: + CharSequence text = toCharSequence(obj); + int maxChars = getLength() / 2; + if (text.length() > maxChars) { + throw new IOException("Text is too big for column"); + } + byte[] encodedData = encodeUncompressedText(text).array(); + obj = encodedData; + break; + case BINARY: + // should already be "encoded" + break; + default: + throw new RuntimeException("unexpected inline var length type: " + + getType()); + } + + ByteBuffer buffer = ByteBuffer.wrap((byte[])obj); + buffer.order(order); + return buffer; + } + + // var length, long value column + switch(getType()) { + case OLE: + // should already be "encoded" + break; + case MEMO: + obj = encodeUncompressedText(toCharSequence(obj)).array(); + break; + default: + throw new RuntimeException("unexpected var length, long value type: " + + getType()); + } + + // create long value buffer + return writeLongValue((byte[]) obj, remainingRowLength); + } + + /** + * Serialize an Object into a raw byte value for this column + * @param obj Object to serialize + * @param order Order in which to serialize + * @return A buffer containing the bytes + */ + public ByteBuffer writeFixedLengthField(Object obj, ByteOrder order) + throws IOException + { + int size = getType().getFixedSize(); + + // create buffer for data ByteBuffer buffer = ByteBuffer.allocate(size); buffer.order(order); - if (obj instanceof Boolean) { - obj = ((Boolean) obj) ? 1 : 0; - } - if (_type == DataType.BOOLEAN) { + + obj = booleanToInteger(obj); + + switch(getType()) { + case BOOLEAN: //Do nothing - } else if (_type == DataType.BYTE) { + break; + case BYTE: buffer.put(obj != null ? ((Number) obj).byteValue() : (byte) 0); - } else if (_type == DataType.INT) { + break; + case INT: buffer.putShort(obj != null ? ((Number) obj).shortValue() : (short) 0); - } else if (_type == DataType.LONG) { + break; + case LONG: buffer.putInt(obj != null ? ((Number) obj).intValue() : 0); - } else if (_type == DataType.DOUBLE) { + break; + case DOUBLE: buffer.putDouble(obj != null ? ((Number) obj).doubleValue() : (double) 0); - } else if (_type == DataType.FLOAT) { + break; + case FLOAT: buffer.putFloat(obj != null ? ((Number) obj).floatValue() : (float) 0); - } else if (_type == DataType.SHORT_DATE_TIME) { + break; + case SHORT_DATE_TIME: writeDateValue(buffer, obj); - } else if (_type == DataType.BINARY) { - buffer.put((byte[]) obj); - } else if (_type == DataType.TEXT) { - CharSequence text = toCharSequence(obj); - int maxChars = size / 2; - if (text.length() > maxChars) { - throw new IOException("Text is too big for column"); - } - buffer.put(encodeUncompressedText(text)); - } else if (_type == DataType.MONEY) { + break; + case MONEY: writeCurrencyValue(buffer, obj); - } else if (_type == DataType.OLE) { - buffer.put(writeLongValue((byte[]) obj)); - } else if (_type == DataType.MEMO) { - buffer.put(writeLongValue((byte[]) obj)); - } else if (_type == DataType.NUMERIC) { + break; + case NUMERIC: writeNumericValue(buffer, obj); - } else if (_type == DataType.GUID) { + break; + case GUID: writeGUIDValue(buffer, obj); - } else { - throw new IOException("Unsupported data type: " + _type); + break; + default: + throw new IOException("Unsupported data type: " + getType()); } buffer.flip(); return buffer; } - + /** * Decodes a compressed or uncompressed text value. */ @@ -826,45 +929,7 @@ public class Column implements Comparable { return _format.CHARSET.decode(ByteBuffer.wrap(textBytes, startPost, length)); } - - /** - * @return Number of bytes that should be read for this column - * (applies to fixed-width columns) - */ - public int size() { - if (_type == DataType.BOOLEAN) { - return 0; - } else if (_type == DataType.BYTE) { - return 1; - } else if (_type == DataType.INT) { - return 2; - } else if (_type == DataType.LONG) { - return 4; - } else if (_type == DataType.MONEY || _type == DataType.DOUBLE) { - return 8; - } else if (_type == DataType.FLOAT) { - return 4; - } else if (_type == DataType.SHORT_DATE_TIME) { - return 8; - } else if (_type == DataType.BINARY) { - return 255; - } else if (_type == DataType.TEXT) { - return 50 * 2; - } else if (_type == DataType.OLE) { - return _format.SIZE_LONG_VALUE_DEF; - } else if (_type == DataType.MEMO) { - return _format.SIZE_LONG_VALUE_DEF; - } else if (_type == DataType.NUMERIC) { - return 17; - } else if (_type == DataType.GUID) { - return 16; - } else if (_type == DataType.UNKNOWN_0D) { - throw new IllegalArgumentException("FIX ME"); - } else { - throw new IllegalArgumentException("Unrecognized data type: " + _type); - } - } - + public String toString() { StringBuilder rtn = new StringBuilder(); rtn.append("\tName: " + _name); @@ -926,7 +991,7 @@ public class Column implements Comparable { /** * @return an appropriate CharSequence representation of the given object. */ - private static CharSequence toCharSequence(Object value) + public static CharSequence toCharSequence(Object value) { if(value == null) { return null; @@ -953,5 +1018,15 @@ public class Column implements Comparable { bytes[idx + 2] = b; } } + + /** + * Treat booleans as integers (C-style). + */ + private Object booleanToInteger(Object obj) { + if (obj instanceof Boolean) { + obj = ((Boolean) obj) ? 1 : 0; + } + return obj; + } } diff --git a/src/java/com/healthmarketscience/jackcess/DataType.java b/src/java/com/healthmarketscience/jackcess/DataType.java index d3b1711..4050958 100644 --- a/src/java/com/healthmarketscience/jackcess/DataType.java +++ b/src/java/com/healthmarketscience/jackcess/DataType.java @@ -47,13 +47,15 @@ public enum DataType { FLOAT((byte) 0x06, Types.FLOAT, 4), DOUBLE((byte) 0x07, Types.DOUBLE, 8), SHORT_DATE_TIME((byte) 0x08, Types.TIMESTAMP, 8), - BINARY((byte) 0x09, Types.BINARY, 255, true), - TEXT((byte) 0x0A, Types.VARCHAR, 50 * 2, true), - OLE((byte) 0x0B, Types.LONGVARBINARY, 12, true), - MEMO((byte) 0x0C, Types.LONGVARCHAR, 12, true), + BINARY((byte) 0x09, Types.BINARY, null, true, false, 255, 255), + TEXT((byte) 0x0A, Types.VARCHAR, null, true, false, 50 * 2, + (int)JetFormat.TEXT_FIELD_MAX_LENGTH), + OLE((byte) 0x0B, Types.LONGVARBINARY, null, true, true, null, 0xFFFFFF), + MEMO((byte) 0x0C, Types.LONGVARCHAR, null, true, true, null, 0xFFFFFF), UNKNOWN_0D((byte) 0x0D), GUID((byte) 0x0F, null, 16), - NUMERIC((byte) 0x10, Types.NUMERIC, 17); + NUMERIC((byte) 0x10, Types.NUMERIC, 17, false, false, null, null, + true, 0, 0, 28, 1, 18, 28); /** Map of SQL types to Access data types */ private static Map SQL_TYPES = new HashMap(); @@ -82,27 +84,76 @@ public enum DataType { /** is this a variable length field */ private boolean _variableLength; + /** is this a long value field */ + private boolean _longValue; + /** does this field have scale/precision */ + private boolean _hasScalePrecision; /** Internal Access value */ private byte _value; - /** Size in bytes */ - private Integer _size; + /** Size in bytes of fixed length columns */ + private Integer _fixedSize; + /** default size for var length columns */ + private Integer _defaultSize; + /** Max size in bytes */ + private Integer _maxSize; /** SQL type equivalent, or null if none defined */ private Integer _sqlType; + /** min scale value */ + private Integer _minScale; + /** the default scale value */ + private Integer _defaultScale; + /** max scale value */ + private Integer _maxScale; + /** min precision value */ + private Integer _minPrecision; + /** the default precision value */ + private Integer _defaultPrecision; + /** max precision value */ + private Integer _maxPrecision; private DataType(byte value) { this(value, null, null); } - private DataType(byte value, Integer sqlType, Integer size) { - this(value, sqlType, size, false); + private DataType(byte value, Integer sqlType, Integer fixedSize) { + this(value, sqlType, fixedSize, false, false, null, null); + } + + private DataType(byte value, Integer sqlType, Integer fixedSize, + boolean variableLength, + boolean longValue, + Integer defaultSize, + Integer maxSize) { + this(value, sqlType, fixedSize, variableLength, longValue, defaultSize, + maxSize, false, null, null, null, null, null, null); } - private DataType(byte value, Integer sqlType, Integer size, - boolean variableLength) { + private DataType(byte value, Integer sqlType, Integer fixedSize, + boolean variableLength, + boolean longValue, + Integer defaultSize, + Integer maxSize, + boolean hasScalePrecision, + Integer minScale, + Integer defaultScale, + Integer maxScale, + Integer minPrecision, + Integer defaultPrecision, + Integer maxPrecision) { _value = value; _sqlType = sqlType; - _size = size; + _fixedSize = fixedSize; _variableLength = variableLength; + _longValue = longValue; + _defaultSize = defaultSize; + _maxSize = maxSize; + _hasScalePrecision = hasScalePrecision; + _minScale = minScale; + _defaultScale = defaultScale; + _maxScale = maxScale; + _minPrecision = minPrecision; + _defaultPrecision = defaultPrecision; + _maxPrecision = maxPrecision; } public byte getValue() { @@ -112,14 +163,30 @@ public enum DataType { public boolean isVariableLength() { return _variableLength; } + + public boolean isLongValue() { + return _longValue; + } + + public boolean getHasScalePrecision() { + return _hasScalePrecision; + } - public int getSize() { - if (_size != null) { - return _size; + public int getFixedSize() { + if(_fixedSize != null) { + return _fixedSize; } else { throw new IllegalArgumentException("FIX ME"); } } + + public int getDefaultSize() { + return _defaultSize; + } + + public int getMaxSize() { + return _maxSize; + } public int getSQLType() throws SQLException { if (_sqlType != null) { @@ -128,6 +195,30 @@ public enum DataType { throw new SQLException("Unsupported data type: " + toString()); } } + + public int getMinScale() { + return _minScale; + } + + public int getDefaultScale() { + return _defaultScale; + } + + public int getMaxScale() { + return _maxScale; + } + + public int getMinPrecision() { + return _minPrecision; + } + + public int getDefaultPrecision() { + return _defaultPrecision; + } + + public int getMaxPrecision() { + return _maxPrecision; + } public static DataType fromByte(byte b) throws IOException { DataType rtn = DATA_TYPES.get(b); diff --git a/src/java/com/healthmarketscience/jackcess/Database.java b/src/java/com/healthmarketscience/jackcess/Database.java index b7240cf..ffed46a 100644 --- a/src/java/com/healthmarketscience/jackcess/Database.java +++ b/src/java/com/healthmarketscience/jackcess/Database.java @@ -356,6 +356,20 @@ public class Database throw new IllegalArgumentException( "Cannot create table with name of existing table"); } + if(columns.isEmpty()) { + throw new IllegalArgumentException( + "Cannot create table with no columns"); + } + + Set colNames = new HashSet(); + // next, validate the column definitions + for(Column column : columns) { + column.validate(); + if(!colNames.add(column.getName().toUpperCase())) { + throw new IllegalArgumentException("duplicate column name: " + + column.getName()); + } + } //We are creating a new page at the end of the db for the tdef. int pageNumber = _pageChannel.getPageCount(); @@ -454,7 +468,7 @@ public class Database buffer.putShort((short) 0); } buffer.putShort(columnNumber); //Column Number again - if(col.getType() == DataType.NUMERIC) { + if(col.getType().getHasScalePrecision()) { buffer.put((byte) col.getPrecision()); // numeric precision buffer.put((byte) col.getScale()); // numeric scale } else { @@ -478,9 +492,13 @@ public class Database buffer.putShort((short) 0); } else { buffer.putShort(fixedOffset); - fixedOffset += col.getType().getSize(); + fixedOffset += col.getType().getFixedSize(); + } + if(!col.getType().isLongValue()) { + buffer.putShort(col.getLength()); //Column length + } else { + buffer.putShort((short)0x0000); // unused } - buffer.putShort(col.getLength()); //Column length if (LOG.isDebugEnabled()) { LOG.debug("Creating new column def block\n" + ByteUtil.toHexString( buffer, position, _format.SIZE_COLUMN_DEF_BLOCK)); @@ -598,6 +616,7 @@ public class Database List columns = new LinkedList(); int textCount = 0; int totalSize = 0; + // FIXME, there is some ugly (and broken) logic here... for (int i = 1; i <= md.getColumnCount(); i++) { DataType accessColumnType = DataType.fromSQLType(md.getColumnType(i)); switch (accessColumnType) { diff --git a/src/java/com/healthmarketscience/jackcess/Index.java b/src/java/com/healthmarketscience/jackcess/Index.java index b532f51..07a054c 100644 --- a/src/java/com/healthmarketscience/jackcess/Index.java +++ b/src/java/com/healthmarketscience/jackcess/Index.java @@ -260,7 +260,9 @@ public class Index implements Comparable { * @param pageNumber Page number on which the row is stored * @param rowNumber Row number at which the row is stored */ - public void addRow(Object[] row, int pageNumber, byte rowNumber) { + public void addRow(Object[] row, int pageNumber, byte rowNumber) + throws IOException + { _entries.add(new Entry(row, pageNumber, rowNumber)); } @@ -285,6 +287,21 @@ public class Index implements Comparable { return 0; } } + + private static void checkColumnType(Column col) + throws IOException + { + if(col.isVariableLength() && !isTextualColumn(col)) { + throw new IOException("unsupported index column type: " + + col.getType()); + } + } + + private static boolean isTextualColumn(Column col) { + return((col.getType() == DataType.TEXT) || + (col.getType() == DataType.MEMO)); + } + /** * A single entry in an index (points to a single row) @@ -304,7 +321,8 @@ public class Index implements Comparable { * @param page Page number on which the row is stored * @param rowNumber Row number at which the row is stored */ - public Entry(Object[] values, int page, byte rowNumber) { + public Entry(Object[] values, int page, byte rowNumber) throws IOException + { _page = page; _row = rowNumber; Iterator iter = _columns.keySet().iterator(); @@ -409,12 +427,15 @@ public class Index implements Comparable { /** * Create a new EntryColumn */ - public EntryColumn(Column col, Comparable value) { + public EntryColumn(Column col, Comparable value) throws IOException { + checkColumnType(col); _column = col; _value = value; - if(_column.getType() == DataType.TEXT) { + if(isTextualColumn(_column)) { // index strings are stored as uppercase - _value = ((_value != null) ? _value.toString().toUpperCase() : null); + _value = ((_value != null) ? + Column.toCharSequence(_value).toString().toUpperCase() : + null); } } @@ -422,10 +443,11 @@ public class Index implements Comparable { * Read in an existing EntryColumn from a buffer */ public EntryColumn(Column col, ByteBuffer buffer) throws IOException { + checkColumnType(col); _column = col; byte flag = buffer.get(); if (flag != (byte) 0) { - if (col.getType() == DataType.TEXT) { + if (isTextualColumn(col)) { StringBuilder sb = new StringBuilder(); byte b; while ( (b = buffer.get()) != (byte) 1) { @@ -453,7 +475,7 @@ public class Index implements Comparable { } _value = sb.toString(); } else { - byte[] data = new byte[col.getType().getSize()]; + byte[] data = new byte[col.getType().getFixedSize()]; buffer.get(data); _value = (Comparable) col.read(data, ByteOrder.BIG_ENDIAN); //ints and shorts are stored in index as value + 2147483648 @@ -465,7 +487,7 @@ public class Index implements Comparable { } } } - + public Comparable getValue() { return _value; } @@ -475,7 +497,7 @@ public class Index implements Comparable { */ public void write(ByteBuffer buffer) throws IOException { buffer.put((byte) 0x7F); - if (_column.getType() == DataType.TEXT) { + if (isTextualColumn(_column)) { String s = (String) _value; for (int i = 0; i < s.length(); i++) { Byte b = (Byte) CODES.get(new Character(s.charAt(i))); @@ -508,16 +530,16 @@ public class Index implements Comparable { } else if (value instanceof Short) { value = new Short((short) (((Short) value).longValue() - ((long) Integer.MAX_VALUE + 1L))); } - buffer.put(_column.write(value, ByteOrder.BIG_ENDIAN)); + buffer.put(_column.write(value, 0, ByteOrder.BIG_ENDIAN)); } } public int size() { if (_value == null) { return 0; - } else if (_value instanceof String) { + } else if(isTextualColumn(_column)) { int rtn = 3; - String s = (String) _value; + String s = (String)_value; for (int i = 0; i < s.length(); i++) { rtn++; if (s.charAt(i) == '^' || s.charAt(i) == '_' || s.charAt(i) == '{' || @@ -531,8 +553,8 @@ public class Index implements Comparable { rtn += _extraBytes.length; } return rtn; - } else { - return _column.getType().getSize(); + } else { + return _column.getType().getFixedSize(); } } diff --git a/src/java/com/healthmarketscience/jackcess/JetFormat.java b/src/java/com/healthmarketscience/jackcess/JetFormat.java index ec5c73e..22b03c8 100644 --- a/src/java/com/healthmarketscience/jackcess/JetFormat.java +++ b/src/java/com/healthmarketscience/jackcess/JetFormat.java @@ -98,7 +98,6 @@ public abstract class JetFormat { public final int OFFSET_REFERENCE_MAP_PAGE_NUMBERS; public final int OFFSET_FREE_SPACE; - public final int OFFSET_DATA_ROW_LOCATION_BLOCK; public final int OFFSET_NUM_ROWS_ON_DATA_PAGE; public final int OFFSET_LVAL_ROW_LOCATION_BLOCK; @@ -182,7 +181,6 @@ public abstract class JetFormat { OFFSET_REFERENCE_MAP_PAGE_NUMBERS = defineOffsetReferenceMapPageNumbers(); OFFSET_FREE_SPACE = defineOffsetFreeSpace(); - OFFSET_DATA_ROW_LOCATION_BLOCK = defineOffsetDataRowLocationBlock(); OFFSET_NUM_ROWS_ON_DATA_PAGE = defineOffsetNumRowsOnDataPage(); OFFSET_LVAL_ROW_LOCATION_BLOCK = defineOffsetLvalRowLocationBlock(); @@ -245,7 +243,6 @@ public abstract class JetFormat { protected abstract int defineOffsetReferenceMapPageNumbers(); protected abstract int defineOffsetFreeSpace(); - protected abstract int defineOffsetDataRowLocationBlock(); protected abstract int defineOffsetNumRowsOnDataPage(); protected abstract int defineOffsetLvalRowLocationBlock(); @@ -271,7 +268,7 @@ public abstract class JetFormat { protected int definePageSize() { return 4096; } - protected int defineMaxRowSize() { return PAGE_SIZE - 18; } + protected int defineMaxRowSize() { return PAGE_SIZE - 16; } protected int defineOffsetNextTableDefPage() { return 4; } protected int defineOffsetNumRows() { return 16; } @@ -309,7 +306,6 @@ public abstract class JetFormat { protected int defineOffsetReferenceMapPageNumbers() { return 1; } protected int defineOffsetFreeSpace() { return 2; } - protected int defineOffsetDataRowLocationBlock() { return 14; } protected int defineOffsetNumRowsOnDataPage() { return 12; } protected int defineOffsetLvalRowLocationBlock() { return 10; } diff --git a/src/java/com/healthmarketscience/jackcess/Table.java b/src/java/com/healthmarketscience/jackcess/Table.java index 6fe49d0..22caf23 100644 --- a/src/java/com/healthmarketscience/jackcess/Table.java +++ b/src/java/com/healthmarketscience/jackcess/Table.java @@ -54,6 +54,10 @@ public class Table private static final Log LOG = LogFactory.getLog(Table.class); private static final short OFFSET_MASK = (short)0x1FFF; + + private static final short DELETED_ROW_MASK = (short)0x4000; + + private static final short OVERFLOW_ROW_MASK = (short)0x8000; /** Table type code for system tables */ public static final byte TYPE_SYSTEM = 0x53; @@ -203,9 +207,9 @@ public class Table if (_currentRowInPage == 0) { throw new IllegalStateException("Must call getNextRow first"); } - int index = _format.OFFSET_DATA_ROW_LOCATION_BLOCK + (_currentRowInPage - 1) * - _format.SIZE_ROW_LOCATION + 1; - _buffer.put(index, (byte) (_buffer.get(index) | 0xc0)); + int index = getRowStartOffset(_currentRowInPage - 1, _format); + _buffer.putShort(index, (short) (_buffer.getShort(index) + | DELETED_ROW_MASK | OVERFLOW_ROW_MASK)); _pageChannel.writePage(_buffer, _ownedPages.getCurrentPageNumber()); } @@ -273,7 +277,7 @@ public class Table { // find fixed length column data colDataPos = dataStart + column.getFixedDataOffset(); - colDataLen = column.getLength(); + colDataLen = column.getType().getFixedSize(); } else { @@ -319,15 +323,15 @@ public class Table _currentRowInPage = 0; _lastRowStart = (short) _format.PAGE_SIZE; } - _rowStart = _buffer.getShort(_format.OFFSET_DATA_ROW_LOCATION_BLOCK + - _currentRowInPage * _format.SIZE_ROW_LOCATION); + _rowStart = _buffer.getShort(getRowStartOffset(_currentRowInPage, + _format)); _currentRowInPage++; _rowsLeftOnPage--; // FIXME, mdbtools seems to be confused as to which flag is which, this // code follows the actual code, which disagrees with the HACKING doc - boolean deletedRow = ((_rowStart & 0x4000) != 0); - boolean overflowRow = ((_rowStart & 0x8000) != 0); + boolean deletedRow = ((_rowStart & DELETED_ROW_MASK) != 0); + boolean overflowRow = ((_rowStart & OVERFLOW_ROW_MASK) != 0); if(deletedRow ^ overflowRow) { if(LOG.isDebugEnabled()) { @@ -514,7 +518,7 @@ public class Table ByteBuffer[] rowData = new ByteBuffer[rows.size()]; Iterator iter = rows.iterator(); for (int i = 0; iter.hasNext(); i++) { - rowData[i] = createRow((Object[]) iter.next()); + rowData[i] = createRow((Object[]) iter.next(), _format.MAX_ROW_SIZE); } List pageNumbers = _ownedPages.getPageNumbers(); int pageNumber; @@ -533,7 +537,7 @@ public class Table short freeSpaceInPage = dataPage.getShort(_format.OFFSET_FREE_SPACE); if (freeSpaceInPage < (rowSize + _format.SIZE_ROW_LOCATION)) { //Last data page is full. Create a new one. - if (rowSize + _format.SIZE_ROW_LOCATION > _format.MAX_ROW_SIZE) { + if (rowSize > _format.MAX_ROW_SIZE) { throw new IOException("Row size " + rowSize + " is too large"); } _pageChannel.writePage(dataPage, pageNumber); @@ -548,18 +552,9 @@ public class Table //Increment row count record. short rowCount = dataPage.getShort(_format.OFFSET_NUM_ROWS_ON_DATA_PAGE); dataPage.putShort(_format.OFFSET_NUM_ROWS_ON_DATA_PAGE, (short) (rowCount + 1)); - short rowLocation = (short) _format.PAGE_SIZE; - if (rowCount > 0) { - rowLocation = dataPage.getShort(_format.OFFSET_DATA_ROW_LOCATION_BLOCK + - (rowCount - 1) * _format.SIZE_ROW_LOCATION); - if (rowLocation < 0) { - // Deleted row - rowLocation &= ~0xc000; - } - } + short rowLocation = findRowEnd(dataPage, rowCount, _format); rowLocation -= rowSize; - dataPage.putShort(_format.OFFSET_DATA_ROW_LOCATION_BLOCK + - rowCount * _format.SIZE_ROW_LOCATION, rowLocation); + dataPage.putShort(getRowStartOffset(rowCount, _format), rowLocation); dataPage.position(rowLocation); dataPage.put(rowData[i]); Iterator indIter = _indexes.iterator(); @@ -594,8 +589,7 @@ public class Table } dataPage.put(PageTypes.DATA); //Page type dataPage.put((byte) 1); //Unknown - dataPage.putShort((short) (_format.PAGE_SIZE - _format.OFFSET_DATA_ROW_LOCATION_BLOCK - - (rowData.limit() - 1) - _format.SIZE_ROW_LOCATION)); //Free space in this page + dataPage.putShort((short)_format.MAX_ROW_SIZE); //Free space in this page dataPage.putInt(_tableDefPageNumber); //Page pointer to table definition dataPage.putInt(0); //Unknown dataPage.putInt(0); //Number of records on this page @@ -608,7 +602,7 @@ public class Table /** * Serialize a row of Objects into a byte buffer */ - ByteBuffer createRow(Object[] rowArray) throws IOException { + ByteBuffer createRow(Object[] rowArray, int maxRowSize) throws IOException { ByteBuffer buffer = _pageChannel.createPageBuffer(); buffer.putShort((short) _columns.size()); NullMask nullMask = new NullMask(_columns.size()); @@ -625,8 +619,9 @@ public class Table for (iter = _columns.iterator(); iter.hasNext() && index < row.size(); index++) { col = (Column) iter.next(); if (!col.isVariableLength()) { - //Fixed length column data comes first - buffer.put(col.write(row.get(index))); + //Fixed length column data comes first (remainingRowLength is ignored + //when writing fixed length data + buffer.put(col.write(row.get(index), 0)); } if (col.getType() == DataType.BOOLEAN) { if (row.get(index) != null) { @@ -639,17 +634,28 @@ public class Table nullMask.markNull(index); } } + int varLengthCount = Column.countVariableLength(_columns); + + // figure out how much space remains for var length data. first, account + // for already written space + maxRowSize -= buffer.position(); + // now, account for trailer space + maxRowSize -= (nullMask.byteSize() + 4 + (varLengthCount * 2)); + short[] varColumnOffsets = new short[varLengthCount]; index = 0; int varColumnOffsetsIndex = 0; //Now write out variable length column data - for (iter = _columns.iterator(); iter.hasNext() && index < row.size(); index++) { + for (iter = _columns.iterator(); iter.hasNext() && index < row.size(); + index++) { col = (Column) iter.next(); short offset = (short) buffer.position(); if (col.isVariableLength()) { if (row.get(index) != null) { - buffer.put(col.write(row.get(index))); + ByteBuffer varDataBuf = col.write(row.get(index), maxRowSize); + maxRowSize -= varDataBuf.remaining(); + buffer.put(varDataBuf); } varColumnOffsets[varColumnOffsetsIndex++] = offset; } @@ -748,21 +754,29 @@ public class Table public static short findRowStart(ByteBuffer buffer, int rowNum, JetFormat format) { - return (short)(buffer.getShort(format.OFFSET_ROW_START + - (format.SIZE_ROW_LOCATION * rowNum)) + return (short)(buffer.getShort(getRowStartOffset(rowNum, format)) & OFFSET_MASK); } + + public static int getRowStartOffset(int rowNum, JetFormat format) + { + return format.OFFSET_ROW_START + (format.SIZE_ROW_LOCATION * rowNum); + } public static short findRowEnd(ByteBuffer buffer, int rowNum, JetFormat format) { return (short)((rowNum == 0) ? format.PAGE_SIZE : - (buffer.getShort(format.OFFSET_ROW_START + - (format.SIZE_ROW_LOCATION * (rowNum - 1))) + (buffer.getShort(getRowEndOffset(rowNum, format)) & OFFSET_MASK)); } + public static int getRowEndOffset(int rowNum, JetFormat format) + { + return format.OFFSET_ROW_START + (format.SIZE_ROW_LOCATION * (rowNum - 1)); + } + /** * Row iterator for this table, supports modification. */ diff --git a/src/java/com/healthmarketscience/jackcess/UsageMap.java b/src/java/com/healthmarketscience/jackcess/UsageMap.java index 5a4adfc..59b6326 100644 --- a/src/java/com/healthmarketscience/jackcess/UsageMap.java +++ b/src/java/com/healthmarketscience/jackcess/UsageMap.java @@ -99,8 +99,8 @@ public abstract class UsageMap { * @param format Format of the database that contains this usage map * @param rowStart Offset at which the declaration starts in the buffer */ - public UsageMap(PageChannel pageChannel, ByteBuffer dataBuffer, int pageNum, - JetFormat format, short rowStart) + protected UsageMap(PageChannel pageChannel, ByteBuffer dataBuffer, + int pageNum, JetFormat format, short rowStart) throws IOException { _pageChannel = pageChannel; diff --git a/test/src/java/com/healthmarketscience/jackcess/DatabaseTest.java b/test/src/java/com/healthmarketscience/jackcess/DatabaseTest.java index 18d3b47..ccefd0f 100644 --- a/test/src/java/com/healthmarketscience/jackcess/DatabaseTest.java +++ b/test/src/java/com/healthmarketscience/jackcess/DatabaseTest.java @@ -9,6 +9,7 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -35,6 +36,63 @@ public class DatabaseTest extends TestCase { return Database.create(tmp); } + public void testInvalidTableDefs() throws Exception { + Database db = create(); + + try { + db.createTable("test", Collections.emptyList()); + fail("created table with no columns?"); + } catch(IllegalArgumentException e) { + // success + } + + List columns = new ArrayList(); + Column col = new Column(); + col.setName("A"); + col.setType(DataType.TEXT); + columns.add(col); + col = new Column(); + col.setName("a"); + col.setType(DataType.MEMO); + columns.add(col); + + try { + db.createTable("test", columns); + fail("created table with duplicate column names?"); + } catch(IllegalArgumentException e) { + // success + } + + columns = new ArrayList(); + col = new Column(); + col.setName("A"); + col.setType(DataType.TEXT); + col.setLength((short)(352 * 2)); + columns.add(col); + + try { + db.createTable("test", columns); + fail("created table with invalid column length?"); + } catch(IllegalArgumentException e) { + // success + } + + columns = new ArrayList(); + col = new Column(); + col.setName("A"); + col.setType(DataType.TEXT); + columns.add(col); + db.createTable("test", columns); + + try { + db.createTable("Test", columns); + fail("create duplicate tables?"); + } catch(IllegalArgumentException e) { + // success + } + + } + public void testReadDeletedRows() throws Exception { Table table = Database.open(new File("test/data/delTest.mdb")).getTable("Table"); int rows = 0; @@ -153,10 +211,33 @@ public class DatabaseTest extends TestCase { } public void testDeleteCurrentRow() throws Exception { + + // make sure correct row is deleted Database db = create(); createTestTable(db); - Object[] row = createTestRow(); + Object[] row1 = createTestRow("Tim1"); + Object[] row2 = createTestRow("Tim2"); + Object[] row3 = createTestRow("Tim3"); Table table = db.getTable("Test"); + table.addRows(Arrays.asList(row1, row2, row3)); + + table.reset(); + table.getNextRow(); + table.getNextRow(); + table.deleteCurrentRow(); + + table.reset(); + + Map outRow = table.getNextRow(); + assertEquals("Tim1", outRow.get("A")); + outRow = table.getNextRow(); + assertEquals("Tim3", outRow.get("A")); + + // test multi row delete/add + db = create(); + createTestTable(db); + Object[] row = createTestRow(); + table = db.getTable("Test"); for (int i = 0; i < 10; i++) { row[3] = i; table.addRow(row); @@ -445,10 +526,14 @@ public class DatabaseTest extends TestCase { assertEquals(89, columns.size()); } - private Object[] createTestRow() { - return new Object[] {"Tim", "R", "McCune", 1234, (byte) 0xad, 555.66d, + private Object[] createTestRow(String col1Val) { + return new Object[] {col1Val, "R", "McCune", 1234, (byte) 0xad, 555.66d, 777.88f, (short) 999, new Date()}; } + + private Object[] createTestRow() { + return createTestRow("Tim"); + } private void createTestTable(Database db) throws Exception { List columns = new ArrayList(); diff --git a/test/src/java/com/healthmarketscience/jackcess/TableTest.java b/test/src/java/com/healthmarketscience/jackcess/TableTest.java index ecb02e8..29178f9 100644 --- a/test/src/java/com/healthmarketscience/jackcess/TableTest.java +++ b/test/src/java/com/healthmarketscience/jackcess/TableTest.java @@ -18,6 +18,7 @@ public class TableTest extends TestCase { } public void testCreateRow() throws Exception { + JetFormat format = JetFormat.VERSION_4; Table table = new Table(); List columns = new ArrayList(); Column col = new Column(); @@ -33,7 +34,7 @@ public class TableTest extends TestCase { row[0] = new Short((short) 9); row[1] = "Tim"; row[2] = "McCune"; - ByteBuffer buffer = table.createRow(row); + ByteBuffer buffer = table.createRow(row, format.MAX_ROW_SIZE); assertEquals((short) colCount, buffer.getShort()); assertEquals((short) 9, buffer.getShort()); assertEquals((byte) 'T', buffer.get()); -- 2.39.5