From 40842f747e387e5584744845f7ee37eb689105ec Mon Sep 17 00:00:00 2001 From: James Ahlborn Date: Sun, 7 Sep 2014 00:43:04 +0000 Subject: [PATCH] initial support for reading and writing calculated columns (issue #105) git-svn-id: https://svn.code.sf.net/p/jackcess/code/jackcess/trunk@868 f203690c-595d-4dc9-a70b-905162fa7fd2 --- TODO.txt | 15 + .../healthmarketscience/jackcess/Column.java | 8 + .../jackcess/PropertyMap.java | 2 + .../jackcess/impl/ByteUtil.java | 53 +- .../jackcess/impl/CalculatedColumnUtil.java | 327 ++++++++ .../jackcess/impl/ColumnImpl.java | 710 ++++++------------ .../jackcess/impl/ComplexColumnImpl.java | 10 +- .../jackcess/impl/JetFormat.java | 10 +- .../jackcess/impl/LongValueColumnImpl.java | 389 +++++++++- .../jackcess/impl/MemoColumnImpl.java | 19 +- .../jackcess/impl/NumericColumnImpl.java | 13 +- .../jackcess/impl/PropertyMaps.java | 2 +- .../jackcess/impl/TableImpl.java | 31 +- .../jackcess/impl/TextColumnImpl.java | 17 +- .../jackcess/impl/UnsupportedColumnImpl.java | 18 +- .../jackcess/DatabaseTest.java | 2 +- .../jackcess/TableTest.java | 7 +- .../jackcess/util/ErrorHandlerTest.java | 7 +- .../jackcess/util/RowFilterTest.java | 3 +- 19 files changed, 1057 insertions(+), 586 deletions(-) create mode 100644 src/main/java/com/healthmarketscience/jackcess/impl/CalculatedColumnUtil.java diff --git a/TODO.txt b/TODO.txt index 8fcbb21..3c955d5 100644 --- a/TODO.txt +++ b/TODO.txt @@ -16,6 +16,21 @@ Missing pieces: * EASY - figure out how msaccess manages page/row locks * MEDIUM + +- calculated fields + - v2010+ + - no indexes + - no unicode compression + - double/int/longint/single/repid/decimal/text/date/memo/currency/bool, no ole/hyperlink + - read/write, create column + - only uses in-table columns (need to force update on every row update?) + - numeric data has embedded precision/scale, something else? only last 8 + bytes is data? implicit precision of 18 for "pure" numeric? implicit + precision of 15 for double-ish numeric? + +- add properties on table/column creation + +- calculated fields in queries? (2003+), w/ aliases? Refactor goals: - tweak lookup apis (specify column vs column name) diff --git a/src/main/java/com/healthmarketscience/jackcess/Column.java b/src/main/java/com/healthmarketscience/jackcess/Column.java index 383f605..fc60f32 100644 --- a/src/main/java/com/healthmarketscience/jackcess/Column.java +++ b/src/main/java/com/healthmarketscience/jackcess/Column.java @@ -133,6 +133,14 @@ public interface Column */ public boolean isHyperlink(); + /** + * Returns whether or not this is a calculated column. Note that jackess + * won't interpret the calculation expression (but the field can be + * written directly). + * @usage _general_method_ + */ + public boolean isCalculated(); + /** * Returns extended functionality for "complex" columns. * @usage _general_method_ diff --git a/src/main/java/com/healthmarketscience/jackcess/PropertyMap.java b/src/main/java/com/healthmarketscience/jackcess/PropertyMap.java index a651811..779d92b 100644 --- a/src/main/java/com/healthmarketscience/jackcess/PropertyMap.java +++ b/src/main/java/com/healthmarketscience/jackcess/PropertyMap.java @@ -45,6 +45,8 @@ public interface PropertyMap extends Iterable public static final String VALIDATION_TEXT_PROP = "ValidationText"; public static final String GUID_PROP = "GUID"; public static final String DESCRIPTION_PROP = "Description"; + public static final String RESULT_TYPE_PROP = "ResultType"; + public static final String EXPRESSION_PROP = "Expression"; public String getName(); diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/ByteUtil.java b/src/main/java/com/healthmarketscience/jackcess/impl/ByteUtil.java index 663ff95..67f0724 100644 --- a/src/main/java/com/healthmarketscience/jackcess/impl/ByteUtil.java +++ b/src/main/java/com/healthmarketscience/jackcess/impl/ByteUtil.java @@ -512,7 +512,7 @@ public final class ByteUtil { } for(int i = 0; i < hexChars.length; i += 2) { String tmpStr = new String(hexChars, i, 2); - buffer.put((byte)Long.parseLong(tmpStr, 16)); + buffer.put((byte)Integer.parseInt(tmpStr, 16)); } } @@ -548,6 +548,20 @@ public final class ByteUtil { return s & 0xFFFF; } + /** + * Swaps the 8 bytes (changes endianness) of the bytes at the given offset. + * + * @param bytes buffer containing bytes to swap + * @param offset offset of the first byte of the bytes to swap + */ + public static void swap8Bytes(byte[] bytes, int offset) + { + swapBytesAt(bytes, offset + 0, offset + 7); + swapBytesAt(bytes, offset + 1, offset + 6); + swapBytesAt(bytes, offset + 2, offset + 5); + swapBytesAt(bytes, offset + 3, offset + 4); + } + /** * Swaps the 4 bytes (changes endianness) of the bytes at the given offset. * @@ -556,12 +570,8 @@ public final class ByteUtil { */ public static void swap4Bytes(byte[] bytes, int offset) { - byte b = bytes[offset + 0]; - bytes[offset + 0] = bytes[offset + 3]; - bytes[offset + 3] = b; - b = bytes[offset + 1]; - bytes[offset + 1] = bytes[offset + 2]; - bytes[offset + 2] = b; + swapBytesAt(bytes, offset + 0, offset + 3); + swapBytesAt(bytes, offset + 1, offset + 2); } /** @@ -572,9 +582,17 @@ public final class ByteUtil { */ public static void swap2Bytes(byte[] bytes, int offset) { - byte b = bytes[offset + 0]; - bytes[offset + 0] = bytes[offset + 1]; - bytes[offset + 1] = b; + swapBytesAt(bytes, offset + 0, offset + 1); + } + + /** + * Swaps the bytes at the given positions. + */ + private static void swapBytesAt(byte[] bytes, int p1, int p2) + { + byte b = bytes[p1]; + bytes[p1] = bytes[p2]; + bytes[p2] = b; } /** @@ -594,7 +612,7 @@ public final class ByteUtil { */ public static byte[] copyOf(byte[] arr, int newLength) { - return copyOf(arr, 0, newLength); + return copyOf(arr, 0, newLength, 0); } /** @@ -602,10 +620,21 @@ public final class ByteUtil { * given position. */ public static byte[] copyOf(byte[] arr, int offset, int newLength) + { + return copyOf(arr, offset, newLength, 0); + } + + /** + * Returns a copy of the given array of the given length starting at the + * given position. + */ + public static byte[] copyOf(byte[] arr, int offset, int newLength, + int dstOffset) { byte[] newArr = new byte[newLength]; int srcLen = arr.length - offset; - System.arraycopy(arr, offset, newArr, 0, Math.min(srcLen, newLength)); + int dstLen = newLength - dstOffset; + System.arraycopy(arr, offset, newArr, dstOffset, Math.min(srcLen, dstLen)); return newArr; } diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/CalculatedColumnUtil.java b/src/main/java/com/healthmarketscience/jackcess/impl/CalculatedColumnUtil.java new file mode 100644 index 0000000..930a0d7 --- /dev/null +++ b/src/main/java/com/healthmarketscience/jackcess/impl/CalculatedColumnUtil.java @@ -0,0 +1,327 @@ +/* +Copyright (c) 2014 James Ahlborn + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 +USA +*/ + +package com.healthmarketscience.jackcess.impl; + +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + + +/** + * Utility code for dealing with calculated columns. + *

+ * These are the currently possible calculated types: FLOAT, DOUBLE, INT, + * LONG, GUID, SHORT_DATE_TIME, MONEY, BOOLEAN, NUMERIC, TEXT, MEMO. + * + * @author James Ahlborn + */ +class CalculatedColumnUtil +{ + private static final int CALC_DATA_LEN_OFFSET = 16; + private static final int CALC_DATA_OFFSET = CALC_DATA_LEN_OFFSET + 4; + private static final int CALC_EXTRA_DATA_LEN = 23; + + private static final byte[] CALC_BOOL_TRUE = wrapCalculatedValue( + new byte[]{(byte)0xFF}); + private static final byte[] CALC_BOOL_FALSE = wrapCalculatedValue( + new byte[]{0}); + + /** + * Creates the appropriate ColumnImpl class for a calculated column and + * reads a column definition in from a buffer + * + * @param table owning table + * @param buffer Buffer containing column definition + * @param offset Offset in the buffer at which the column definition starts + * @usage _advanced_method_ + */ + static ColumnImpl create(ColumnImpl.InitArgs args) throws IOException + { + switch(args.type) { + case BOOLEAN: + return new CalcBooleanColImpl(args); + case TEXT: + return new CalcTextColImpl(args); + case MEMO: + return new CalcMemoColImpl(args); + default: + // fall through + } + + if(args.type.getHasScalePrecision()) { + return new CalcNumericColImpl(args); + } + + return new CalcColImpl(args); + } + + private static byte[] unwrapCalculatedValue(byte[] data) { + if(data.length < CALC_DATA_OFFSET) { + return data; + } + + ByteBuffer buffer = PageChannel.wrap(data); + buffer.position(CALC_DATA_LEN_OFFSET); + int dataLen = buffer.getInt(); + byte[] newData = new byte[Math.min(buffer.remaining(), dataLen)]; + buffer.get(newData); + return newData; + } + + private static ByteBuffer wrapCalculatedValue(ByteBuffer buffer) { + int dataLen = buffer.remaining(); + byte[] data = new byte[dataLen + CALC_EXTRA_DATA_LEN]; + buffer.get(data, CALC_DATA_OFFSET, dataLen); + buffer = PageChannel.wrap(data); + buffer.putInt(CALC_DATA_LEN_OFFSET, dataLen); + return buffer; + } + + private static byte[] wrapCalculatedValue(byte[] data) { + int dataLen = data.length; + data = ByteUtil.copyOf(data, 0, dataLen + CALC_EXTRA_DATA_LEN, + CALC_DATA_OFFSET); + PageChannel.wrap(data).putInt(CALC_DATA_LEN_OFFSET, dataLen); + return data; + } + + private static ByteBuffer prepareWrappedCalcValue(int dataLen, ByteOrder order) + { + ByteBuffer buffer = ByteBuffer.allocate( + dataLen + CALC_EXTRA_DATA_LEN).order(order); + buffer.putInt(CALC_DATA_LEN_OFFSET, dataLen); + buffer.position(CALC_DATA_OFFSET); + return buffer; + } + + + private static class CalcColImpl extends ColumnImpl + { + CalcColImpl(InitArgs args) throws IOException { + super(args); + } + + @Override + public Object read(byte[] data, ByteOrder order) throws IOException { + return super.read(unwrapCalculatedValue(data), order); + } + + @Override + protected ByteBuffer writeRealData(Object obj, int remainingRowLength, + ByteOrder order) + throws IOException + { + // we should only be working with fixed length types + return writeFixedLengthField( + obj, prepareWrappedCalcValue(getType().getFixedSize(), order)); + } + } + + private static class CalcBooleanColImpl extends ColumnImpl + { + CalcBooleanColImpl(InitArgs args) throws IOException { + super(args); + } + + @Override + public boolean storeInNullMask() { + // calculated booleans are _not_ stored in null mask + return false; + } + + @Override + public Object read(byte[] data, ByteOrder order) throws IOException { + data = unwrapCalculatedValue(data); + return ((data[0] != 0) ? Boolean.TRUE : Boolean.FALSE); + } + + @Override + protected ByteBuffer writeRealData(Object obj, int remainingRowLength, + ByteOrder order) + throws IOException + { + return ByteBuffer.wrap( + toBooleanValue(obj) ? CALC_BOOL_TRUE : CALC_BOOL_FALSE).order(order); + } + } + + private static class CalcTextColImpl extends TextColumnImpl + { + CalcTextColImpl(InitArgs args) throws IOException { + super(args); + } + + @Override + public Object read(byte[] data, ByteOrder order) throws IOException { + return decodeTextValue(unwrapCalculatedValue(data)); + } + + @Override + protected ByteBuffer writeRealData(Object obj, int remainingRowLength, + ByteOrder order) + throws IOException + { + int maxChars = getType().toUnitSize(getLength() - CALC_EXTRA_DATA_LEN); + return wrapCalculatedValue(encodeTextValue(obj, 0, maxChars, false)); + } + } + + private static class CalcMemoColImpl extends MemoColumnImpl + { + CalcMemoColImpl(InitArgs args) throws IOException { + super(args); + } + + @Override + protected byte[] readLongValue(byte[] lvalDefinition) + throws IOException + { + return unwrapCalculatedValue(super.readLongValue(lvalDefinition)); + } + + @Override + protected ByteBuffer writeLongValue(byte[] value, int remainingRowLength) + throws IOException + { + return super.writeLongValue(wrapCalculatedValue(value), remainingRowLength); + } + } + + private static class CalcNumericColImpl extends NumericColumnImpl + { + CalcNumericColImpl(InitArgs args) throws IOException { + super(args); + } + + @Override + public byte getPrecision() { + return (byte)getType().getMaxPrecision(); + } + + @Override + public Object read(byte[] data, ByteOrder order) throws IOException { + data = unwrapCalculatedValue(data); + return readCalcNumericValue(ByteBuffer.wrap(data).order(order)); + } + + @Override + protected ByteBuffer writeRealData(Object obj, int remainingRowLength, + ByteOrder order) + throws IOException + { + int totalDataLen = Math.min(CALC_EXTRA_DATA_LEN + 16 + 4, getLength()); + int dataLen = totalDataLen - CALC_EXTRA_DATA_LEN; + ByteBuffer buffer = prepareWrappedCalcValue(dataLen, order); + + writeCalcNumericValue(buffer, obj, dataLen); + + buffer.flip(); + + return buffer; + } + + private static BigDecimal readCalcNumericValue(ByteBuffer buffer) + { + short totalLen = buffer.getShort(); + // numeric bytes need to be a multiple of 4 and we currently handle at + // most 16 bytes + int numByteLen = ((totalLen > 0) ? totalLen : buffer.remaining()) - 2; + numByteLen = Math.min((numByteLen / 4) * 4, 16); + byte scale = buffer.get(); + boolean negate = (buffer.get() != 0); + byte[] tmpArr = ByteUtil.getBytes(buffer, numByteLen); + + if(buffer.order() != ByteOrder.BIG_ENDIAN) { + fixNumericByteOrder(tmpArr); + } + + return toBigDecimal(tmpArr, negate, scale); + } + + private void writeCalcNumericValue(ByteBuffer buffer, Object value, + int dataLen) + throws IOException + { + Object inValue = value; + try { + BigDecimal decVal = toBigDecimal(value); + inValue = decVal; + + int signum = decVal.signum(); + if(signum < 0) { + decVal = decVal.negate(); + } + + int maxScale = getType().getMaxScale(); + if(decVal.scale() > maxScale) { + // adjust scale according to max (will cause the an + // ArithmeticException if number has too many decimal places) + decVal = decVal.setScale(maxScale); + } + int scale = decVal.scale(); + + // check precision + if(decVal.precision() > getType().getMaxPrecision()) { + throw new IOException( + "Numeric value is too big for specified precision " + + getType().getMaxPrecision() + ": " + decVal); + } + + // convert to unscaled BigInteger, big-endian bytes + byte[] intValBytes = toUnscaledByteArray(decVal, dataLen - 4); + + if(buffer.order() != ByteOrder.BIG_ENDIAN) { + fixNumericByteOrder(intValBytes); + } + + buffer.putShort((short)(dataLen - 2)); + buffer.put((byte)scale); + // write sign byte + buffer.put(signum < 0 ? (byte)0x80 : (byte)0); + buffer.put(intValBytes); + + } catch(ArithmeticException e) { + throw (IOException) + new IOException("Numeric value '" + inValue + "' out of range") + .initCause(e); + } + } + + private static void fixNumericByteOrder(byte[] bytes) { + + // this is a little weird. it looks like they decided to truncate + // leading 0 bytes and _then_ swapp endian, which ends up kind of odd. + int pos = 0; + if((bytes.length % 8) != 0) { + // leading 4 bytes are swapped + ByteUtil.swap4Bytes(bytes, 0); + pos += 4; + } + + // then fix endianness of each 8 byte segment + for(; pos < bytes.length; pos+=8) { + ByteUtil.swap8Bytes(bytes, pos); + } + } + + } + +} diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/ColumnImpl.java b/src/main/java/com/healthmarketscience/jackcess/impl/ColumnImpl.java index ce8ff63..c9c67e7 100644 --- a/src/main/java/com/healthmarketscience/jackcess/impl/ColumnImpl.java +++ b/src/main/java/com/healthmarketscience/jackcess/impl/ColumnImpl.java @@ -77,7 +77,7 @@ import org.apache.commons.logging.LogFactory; */ public class ColumnImpl implements Column, Comparable { - private static final Log LOG = LogFactory.getLog(ColumnImpl.class); + protected static final Log LOG = LogFactory.getLog(ColumnImpl.class); /** * Placeholder object for adding rows which indicates that the caller wants @@ -101,27 +101,6 @@ public class ColumnImpl implements Column, Comparable { private static final long MILLIS_BETWEEN_EPOCH_AND_1900 = 25569L * (long)MILLISECONDS_PER_DAY; - /** - * Long value (LVAL) type that indicates that the value is stored on the - * same page - */ - private static final byte LONG_VALUE_TYPE_THIS_PAGE = (byte) 0x80; - /** - * Long value (LVAL) type that indicates that the value is stored on another - * page - */ - private static final byte LONG_VALUE_TYPE_OTHER_PAGE = (byte) 0x40; - /** - * Long value (LVAL) type that indicates that the value is stored on - * multiple other pages - */ - private static final byte LONG_VALUE_TYPE_OTHER_PAGES = (byte) 0x00; - /** - * Mask to apply the long length in order to get the flag bits (only the - * first 2 bits are type flags). - */ - private static final int LONG_VALUE_TYPE_MASK = 0xC0000000; - /** * mask for the fixed len bit * @usage _advanced_field_ @@ -154,7 +133,9 @@ public class ColumnImpl implements Column, Comparable { // some other flags? // 0x10: replication related field (or hidden?) - // 0x80: hyperlink (some memo based thing) + + protected static final byte COMPRESSED_UNICODE_EXT_FLAG_MASK = (byte)0x01; + private static final byte CALCULATED_EXT_FLAG_MASK = (byte)0xC0; /** the value for the "general" sort order */ private static final short GENERAL_SORT_ORDER_VALUE = 1033; @@ -188,6 +169,8 @@ public class ColumnImpl implements Column, Comparable { private final boolean _variableLength; /** Whether or not the column is an autonumber column */ private final boolean _autoNumber; + /** Whether or not the column is a calculated column */ + private final boolean _calculated; /** Data type */ private final DataType _type; /** Maximum column length */ @@ -199,7 +182,7 @@ public class ColumnImpl implements Column, Comparable { /** display index of the data for this column */ private final int _displayIndex; /** Column name */ - private String _name; + private final String _name; /** the offset of the fixed data in the row */ private final int _fixedDataOffset; /** the index of the variable length data in the var len offset table */ @@ -214,9 +197,10 @@ public class ColumnImpl implements Column, Comparable { /** * @usage _advanced_method_ */ - protected ColumnImpl(TableImpl table, DataType type, int colNumber, - int fixedOffset, int varLenIndex) { + protected ColumnImpl(TableImpl table, String name, DataType type, + int colNumber, int fixedOffset, int varLenIndex) { _table = table; + _name = name; _type = type; if(!_type.isVariableLength()) { @@ -226,6 +210,7 @@ public class ColumnImpl implements Column, Comparable { } _variableLength = type.isVariableLength(); _autoNumber = false; + _calculated = false; _autoNumberGenerator = null; _columnNumber = (short)colNumber; _columnIndex = colNumber; @@ -241,32 +226,37 @@ public class ColumnImpl implements Column, Comparable { * @param offset Offset in the buffer at which the column definition starts * @usage _advanced_method_ */ - ColumnImpl(TableImpl table, ByteBuffer buffer, int offset, int displayIndex, - DataType type, byte flags) + ColumnImpl(InitArgs args) throws IOException { - _table = table; - _displayIndex = displayIndex; - _type = type; + _table = args.table; + _name = args.name; + _displayIndex = args.displayIndex; + _type = args.type; - _columnNumber = buffer.getShort(offset + getFormat().OFFSET_COLUMN_NUMBER); - _columnLength = buffer.getShort(offset + getFormat().OFFSET_COLUMN_LENGTH); + _columnNumber = args.buffer.getShort( + args.offset + getFormat().OFFSET_COLUMN_NUMBER); + _columnLength = args.buffer.getShort( + args.offset + getFormat().OFFSET_COLUMN_LENGTH); - _variableLength = ((flags & FIXED_LEN_FLAG_MASK) == 0); - _autoNumber = ((flags & (AUTO_NUMBER_FLAG_MASK | AUTO_NUMBER_GUID_FLAG_MASK)) - != 0); + _variableLength = ((args.flags & FIXED_LEN_FLAG_MASK) == 0); + _autoNumber = ((args.flags & + (AUTO_NUMBER_FLAG_MASK | AUTO_NUMBER_GUID_FLAG_MASK)) != 0); + _calculated = ((args.extFlags & CALCULATED_EXT_FLAG_MASK) != 0); _autoNumberGenerator = createAutoNumberGenerator(); if(_variableLength) { - _varLenTableIndex = buffer.getShort(offset + getFormat().OFFSET_COLUMN_VARIABLE_TABLE_INDEX); + _varLenTableIndex = args.buffer.getShort( + args.offset + getFormat().OFFSET_COLUMN_VARIABLE_TABLE_INDEX); _fixedDataOffset = 0; } else { - _fixedDataOffset = buffer.getShort(offset + getFormat().OFFSET_COLUMN_FIXED_DATA_OFFSET); + _fixedDataOffset = args.buffer.getShort( + args.offset + getFormat().OFFSET_COLUMN_FIXED_DATA_OFFSET); _varLenTableIndex = 0; } } - + /** * Creates the appropriate ColumnImpl class and reads a column definition in * from a buffer @@ -275,49 +265,56 @@ public class ColumnImpl implements Column, Comparable { * @param offset Offset in the buffer at which the column definition starts * @usage _advanced_method_ */ - public static ColumnImpl create(TableImpl table, ByteBuffer buffer, int offset, - int displayIndex) + public static ColumnImpl create(TableImpl table, ByteBuffer buffer, + int offset, String name, int displayIndex) throws IOException { - byte colType = buffer.get(offset + table.getFormat().OFFSET_COLUMN_TYPE); - byte flags = buffer.get(offset + table.getFormat().OFFSET_COLUMN_FLAGS); - - DataType type = null; + InitArgs args = new InitArgs(table, buffer, offset, name, displayIndex); + + boolean calculated = ((args.extFlags & CALCULATED_EXT_FLAG_MASK) != 0); + byte colType = args.colType; + if(calculated) { + // "real" data type is in the "result type" property + PropertyMap colProps = table.getPropertyMaps().get(name); + Byte resultType = (Byte)colProps.getValue(PropertyMap.RESULT_TYPE_PROP); + if(resultType != null) { + colType = resultType; + } + } + try { - type = DataType.fromByte(colType); + args.type = DataType.fromByte(colType); } catch(IOException e) { LOG.warn("Unsupported column type " + colType); - boolean variableLength = ((flags & FIXED_LEN_FLAG_MASK) == 0); - type = (variableLength ? DataType.UNSUPPORTED_VARLEN : - DataType.UNSUPPORTED_FIXEDLEN); - return new UnsupportedColumnImpl(table, buffer, offset, displayIndex, type, - flags, colType); + boolean variableLength = ((args.flags & FIXED_LEN_FLAG_MASK) == 0); + args.type = (variableLength ? DataType.UNSUPPORTED_VARLEN : + DataType.UNSUPPORTED_FIXEDLEN); + return new UnsupportedColumnImpl(args); } - switch(type) { + if(calculated) { + return CalculatedColumnUtil.create(args); + } + + switch(args.type) { case TEXT: - return new TextColumnImpl(table, buffer, offset, displayIndex, type, - flags); + return new TextColumnImpl(args); case MEMO: - return new MemoColumnImpl(table, buffer, offset, displayIndex, type, - flags); + return new MemoColumnImpl(args); case COMPLEX_TYPE: - return new ComplexColumnImpl(table, buffer, offset, displayIndex, type, - flags); + return new ComplexColumnImpl(args); default: // fall through } - if(type.getHasScalePrecision()) { - return new NumericColumnImpl(table, buffer, offset, displayIndex, type, - flags); + if(args.type.getHasScalePrecision()) { + return new NumericColumnImpl(args); } - if(type.isLongValue()) { - return new LongValueColumnImpl(table, buffer, offset, displayIndex, type, - flags); + if(args.type.isLongValue()) { + return new LongValueColumnImpl(args); } - return new ColumnImpl(table, buffer, offset, displayIndex, type, flags); + return new ColumnImpl(args); } /** @@ -359,13 +356,6 @@ public class ColumnImpl implements Column, Comparable { public String getName() { return _name; } - - /** - * @usage _advanced_method_ - */ - public void setName(String name) { - _name = name; - } public boolean isVariableLength() { return _variableLength; @@ -441,6 +431,10 @@ public class ColumnImpl implements Column, Comparable { public short getLengthInUnits() { return (short)getType().toUnitSize(getLength()); } + + public boolean isCalculated() { + return _calculated; + } /** * @usage _advanced_method_ @@ -525,10 +519,6 @@ public class ColumnImpl implements Column, Comparable { byte getOriginalDataType() { return _type.getValue(); } - - LongValueBufferHolder getLongValueBufferHolder() { - return null; - } private AutoNumberGenerator createAutoNumberGenerator() { if(!_autoNumber || (_type == null)) { @@ -581,7 +571,19 @@ public class ColumnImpl implements Column, Comparable { public Object getRowValue(Map rowMap) { return rowMap.get(_name); } + + public boolean storeInNullMask() { + return (getType() == DataType.BOOLEAN); + } + public boolean writeToNullMask(Object value) { + return toBooleanValue(value); + } + + public Object readFromNullMask(boolean isNull) { + return Boolean.valueOf(!isNull); + } + /** * Deserialize a raw byte value for this column into an Object * @param data The raw byte value @@ -600,171 +602,44 @@ public class ColumnImpl implements Column, Comparable { * @usage _advanced_method_ */ public Object read(byte[] data, ByteOrder order) throws IOException { - ByteBuffer buffer = ByteBuffer.wrap(data); - buffer.order(order); - if (_type == DataType.BOOLEAN) { + ByteBuffer buffer = ByteBuffer.wrap(data).order(order); + + switch(getType()) { + case BOOLEAN: throw new IOException("Tried to read a boolean from data instead of null mask."); - } else if (_type == DataType.BYTE) { + case BYTE: return Byte.valueOf(buffer.get()); - } else if (_type == DataType.INT) { + case INT: return Short.valueOf(buffer.getShort()); - } else if (_type == DataType.LONG) { + case LONG: return Integer.valueOf(buffer.getInt()); - } else if (_type == DataType.DOUBLE) { + case DOUBLE: return Double.valueOf(buffer.getDouble()); - } else if (_type == DataType.FLOAT) { + case FLOAT: return Float.valueOf(buffer.getFloat()); - } else if (_type == DataType.SHORT_DATE_TIME) { + case SHORT_DATE_TIME: return readDateValue(buffer); - } else if (_type == DataType.BINARY) { + case BINARY: return data; - } else if (_type == DataType.TEXT) { + case TEXT: return decodeTextValue(data); - } else if (_type == DataType.MONEY) { + case MONEY: return readCurrencyValue(buffer); - } else if (_type == DataType.OLE) { - if (data.length > 0) { - return readLongValue(data); - } - return null; - } else if (_type == DataType.MEMO) { - if (data.length > 0) { - return readLongStringValue(data); - } - return null; - } else if (_type == DataType.NUMERIC) { + case NUMERIC: return readNumericValue(buffer); - } else if (_type == DataType.GUID) { + case GUID: return readGUIDValue(buffer, order); - } else if ((_type == DataType.UNKNOWN_0D) || - (_type == DataType.UNKNOWN_11)) { + case UNKNOWN_0D: + case UNKNOWN_11: // treat like "binary" data return data; - } else if (_type == DataType.COMPLEX_TYPE) { + case COMPLEX_TYPE: return new ComplexValueForeignKeyImpl(this, buffer.getInt()); - } else if(_type.isUnsupported()) { - return rawDataWrapper(data); - } else { + default: throw new IOException("Unrecognized data type: " + _type); } } - /** - * @param lvalDefinition Column value that points to an LVAL record - * @return The LVAL data - */ - private byte[] readLongValue(byte[] lvalDefinition) - throws IOException - { - ByteBuffer def = PageChannel.wrap(lvalDefinition); - int lengthWithFlags = def.getInt(); - int length = lengthWithFlags & (~LONG_VALUE_TYPE_MASK); - - byte[] rtn = new byte[length]; - byte type = (byte)((lengthWithFlags & LONG_VALUE_TYPE_MASK) >>> 24); - - if(type == LONG_VALUE_TYPE_THIS_PAGE) { - - // inline long value - def.getInt(); //Skip over lval_dp - def.getInt(); //Skip over unknown - - int rowLen = def.remaining(); - if(rowLen < length) { - // warn the caller, but return whatever we can - LOG.warn(getName() + " value may be truncated: expected length " + - length + " found " + rowLen); - rtn = new byte[rowLen]; - } - - def.get(rtn); - - } else { - - // long value on other page(s) - if (lvalDefinition.length != getFormat().SIZE_LONG_VALUE_DEF) { - throw new IOException("Expected " + getFormat().SIZE_LONG_VALUE_DEF + - " bytes in long value definition, but found " + - lvalDefinition.length); - } - - int rowNum = ByteUtil.getUnsignedByte(def); - int pageNum = ByteUtil.get3ByteInt(def, def.position()); - ByteBuffer lvalPage = getPageChannel().createPageBuffer(); - - switch (type) { - case LONG_VALUE_TYPE_OTHER_PAGE: - { - getPageChannel().readPage(lvalPage, pageNum); - - short rowStart = TableImpl.findRowStart(lvalPage, rowNum, getFormat()); - short rowEnd = TableImpl.findRowEnd(lvalPage, rowNum, getFormat()); - - int rowLen = rowEnd - rowStart; - if(rowLen < length) { - // warn the caller, but return whatever we can - LOG.warn(getName() + " value may be truncated: expected length " + - length + " found " + rowLen); - rtn = new byte[rowLen]; - } - - lvalPage.position(rowStart); - lvalPage.get(rtn); - } - break; - - case LONG_VALUE_TYPE_OTHER_PAGES: - - ByteBuffer rtnBuf = ByteBuffer.wrap(rtn); - int remainingLen = length; - while(remainingLen > 0) { - lvalPage.clear(); - getPageChannel().readPage(lvalPage, pageNum); - - short rowStart = TableImpl.findRowStart(lvalPage, rowNum, getFormat()); - short rowEnd = TableImpl.findRowEnd(lvalPage, rowNum, getFormat()); - - // read next page information - lvalPage.position(rowStart); - rowNum = ByteUtil.getUnsignedByte(lvalPage); - pageNum = ByteUtil.get3ByteInt(lvalPage); - - // update rowEnd and remainingLen based on chunkLength - int chunkLength = (rowEnd - rowStart) - 4; - if(chunkLength > remainingLen) { - rowEnd = (short)(rowEnd - (chunkLength - remainingLen)); - chunkLength = remainingLen; - } - remainingLen -= chunkLength; - - lvalPage.limit(rowEnd); - rtnBuf.put(lvalPage); - } - - break; - - default: - throw new IOException("Unrecognized long value type: " + type); - } - } - - return rtn; - } - - /** - * @param lvalDefinition Column value that points to an LVAL record - * @return The LVAL data - */ - private String readLongStringValue(byte[] lvalDefinition) - throws IOException - { - byte[] binData = readLongValue(lvalDefinition); - if(binData == null) { - return null; - } - return decodeTextValue(binData); - } - /** * Decodes "Currency" values. * @@ -820,11 +695,22 @@ public class ColumnImpl implements Column, Comparable { fixNumericByteOrder(tmpArr); } - BigInteger intVal = new BigInteger(tmpArr); + return toBigDecimal(tmpArr, negate, getScale()); + } + + static BigDecimal toBigDecimal(byte[] bytes, boolean negate, int scale) + { + if((bytes[0] & 0x80) != 0) { + // the data is effectively unsigned, but the BigInteger handles it as + // signed twos complement. we need to add an extra byte to the input so + // that it will be treated as unsigned + bytes = ByteUtil.copyOf(bytes, 0, bytes.length + 1, 1); + } + BigInteger intVal = new BigInteger(bytes); if(negate) { intVal = intVal.negate(); } - return new BigDecimal(intVal, getScale()); + return new BigDecimal(intVal, scale); } /** @@ -838,13 +724,13 @@ public class ColumnImpl implements Column, Comparable { BigDecimal decVal = toBigDecimal(value); inValue = decVal; - boolean negative = (decVal.compareTo(BigDecimal.ZERO) < 0); - if(negative) { + int signum = decVal.signum(); + if(signum < 0) { decVal = decVal.negate(); } // write sign byte - buffer.put(negative ? (byte)0x80 : (byte)0); + buffer.put(signum < 0 ? (byte)0x80 : (byte)0); // adjust scale according to this column type (will cause the an // ArithmeticException if number has too many decimal places) @@ -858,18 +744,8 @@ public class ColumnImpl implements Column, Comparable { } // convert to unscaled BigInteger, big-endian bytes - byte[] intValBytes = decVal.unscaledValue().toByteArray(); - int maxByteLen = getType().getFixedSize() - 1; - if(intValBytes.length > maxByteLen) { - throw new IOException("Too many bytes for valid BigInteger?"); - } - if(intValBytes.length < maxByteLen) { - byte[] tmpBytes = new byte[maxByteLen]; - System.arraycopy(intValBytes, 0, tmpBytes, - (maxByteLen - intValBytes.length), - intValBytes.length); - intValBytes = tmpBytes; - } + byte[] intValBytes = toUnscaledByteArray( + decVal, getType().getFixedSize() - 1); if(buffer.order() != ByteOrder.BIG_ENDIAN) { fixNumericByteOrder(intValBytes); } @@ -881,6 +757,27 @@ public class ColumnImpl implements Column, Comparable { } } + static byte[] toUnscaledByteArray(BigDecimal decVal, int maxByteLen) + throws IOException + { + // convert to unscaled BigInteger, big-endian bytes + byte[] intValBytes = decVal.unscaledValue().toByteArray(); + if(intValBytes.length > maxByteLen) { + if((intValBytes[0] == 0) && ((intValBytes.length - 1) == maxByteLen)) { + // in order to not return a negative two's complement value, + // toByteArray() may return an extra leading 0 byte. we are working + // with unsigned values, so we can drop the extra leading 0 + intValBytes = ByteUtil.copyOf(intValBytes, 1, maxByteLen); + } else { + throw new IOException("Too many bytes for valid BigInteger?"); + } + } else if(intValBytes.length < maxByteLen) { + intValBytes = ByteUtil.copyOf(intValBytes, 0, maxByteLen, + (maxByteLen - intValBytes.length)); + } + return intValBytes; + } + /** * Decodes a date value. */ @@ -1008,8 +905,7 @@ public class ColumnImpl implements Column, Comparable { /** * Writes a GUID value. */ - private static void writeGUIDValue(ByteBuffer buffer, Object value, - ByteOrder order) + private static void writeGUIDValue(ByteBuffer buffer, Object value) throws IOException { Matcher m = GUID_PATTERN.matcher(toCharSequence(value)); @@ -1019,7 +915,7 @@ public class ColumnImpl implements Column, Comparable { ByteBuffer origBuffer = null; byte[] tmpBuf = null; - if(order != ByteOrder.BIG_ENDIAN) { + if(buffer.order() != ByteOrder.BIG_ENDIAN) { // write to a temp buf so we can do some swapping below origBuffer = buffer; tmpBuf = new byte[16]; @@ -1048,151 +944,6 @@ public class ColumnImpl implements Column, Comparable { static boolean isGUIDValue(Object value) throws IOException { return GUID_PATTERN.matcher(toCharSequence(value)).matches(); } - - /** - * Write an LVAL column into a ByteBuffer inline if it fits, otherwise in - * other data page(s). - * @param value Value of the LVAL column - * @return A buffer containing the LVAL definition and (possibly) the column - * value (unless written to other pages) - * @usage _advanced_method_ - */ - public ByteBuffer writeLongValue(byte[] value, - int remainingRowLength) throws IOException - { - if(value.length > getType().getMaxSize()) { - throw new IOException("value too big for column, max " + - getType().getMaxSize() + ", got " + - value.length); - } - - // determine which type to write - byte type = 0; - int lvalDefLen = getFormat().SIZE_LONG_VALUE_DEF; - if(((getFormat().SIZE_LONG_VALUE_DEF + value.length) <= remainingRowLength) - && (value.length <= getFormat().MAX_INLINE_LONG_VALUE_SIZE)) { - type = LONG_VALUE_TYPE_THIS_PAGE; - lvalDefLen += value.length; - } else if(value.length <= getFormat().MAX_LONG_VALUE_ROW_SIZE) { - type = LONG_VALUE_TYPE_OTHER_PAGE; - } else { - type = LONG_VALUE_TYPE_OTHER_PAGES; - } - - ByteBuffer def = getPageChannel().createBuffer(lvalDefLen); - // take length and apply type to first byte - int lengthWithFlags = value.length | (type << 24); - def.putInt(lengthWithFlags); - - if(type == LONG_VALUE_TYPE_THIS_PAGE) { - // write long value inline - def.putInt(0); - def.putInt(0); //Unknown - def.put(value); - } else { - - ByteBuffer lvalPage = null; - int firstLvalPageNum = PageChannel.INVALID_PAGE_NUMBER; - byte firstLvalRow = 0; - LongValueBufferHolder lvalBufferH = getLongValueBufferHolder(); - - // write other page(s) - switch(type) { - case LONG_VALUE_TYPE_OTHER_PAGE: - lvalPage = lvalBufferH.getLongValuePage(value.length); - firstLvalPageNum = lvalBufferH.getPageNumber(); - firstLvalRow = (byte)TableImpl.addDataPageRow(lvalPage, value.length, - getFormat(), 0); - lvalPage.put(value); - getPageChannel().writePage(lvalPage, firstLvalPageNum); - break; - - case LONG_VALUE_TYPE_OTHER_PAGES: - - ByteBuffer buffer = ByteBuffer.wrap(value); - int remainingLen = buffer.remaining(); - buffer.limit(0); - lvalPage = lvalBufferH.getLongValuePage(remainingLen); - firstLvalPageNum = lvalBufferH.getPageNumber(); - firstLvalRow = (byte)TableImpl.getRowsOnDataPage(lvalPage, getFormat()); - int lvalPageNum = firstLvalPageNum; - ByteBuffer nextLvalPage = null; - int nextLvalPageNum = 0; - int nextLvalRowNum = 0; - while(remainingLen > 0) { - lvalPage.clear(); - - // figure out how much we will put in this page (we need 4 bytes for - // the next page pointer) - int chunkLength = Math.min(getFormat().MAX_LONG_VALUE_ROW_SIZE - 4, - remainingLen); - - // figure out if we will need another page, and if so, allocate it - if(chunkLength < remainingLen) { - // force a new page to be allocated for the chunk after this - lvalBufferH.clear(); - nextLvalPage = lvalBufferH.getLongValuePage( - (remainingLen - chunkLength) + 4); - nextLvalPageNum = lvalBufferH.getPageNumber(); - nextLvalRowNum = TableImpl.getRowsOnDataPage(nextLvalPage, - getFormat()); - } else { - nextLvalPage = null; - nextLvalPageNum = 0; - nextLvalRowNum = 0; - } - - // add row to this page - TableImpl.addDataPageRow(lvalPage, chunkLength + 4, getFormat(), 0); - - // write next page info - lvalPage.put((byte)nextLvalRowNum); // row number - ByteUtil.put3ByteInt(lvalPage, nextLvalPageNum); // page number - - // write this page's chunk of data - buffer.limit(buffer.limit() + chunkLength); - lvalPage.put(buffer); - remainingLen -= chunkLength; - - // write new page to database - getPageChannel().writePage(lvalPage, lvalPageNum); - - // move to next page - lvalPage = nextLvalPage; - lvalPageNum = nextLvalPageNum; - } - break; - - default: - throw new IOException("Unrecognized long value type: " + type); - } - - // update def - def.put(firstLvalRow); - ByteUtil.put3ByteInt(def, firstLvalPageNum); - def.putInt(0); //Unknown - - } - - def.flip(); - return def; - } - - /** - * Writes the header info for a long value page. - */ - private void writeLongValueHeader(ByteBuffer lvalPage) - { - lvalPage.put(PageTypes.DATA); //Page type - lvalPage.put((byte) 1); //Unknown - lvalPage.putShort((short)getFormat().DATA_PAGE_INITIAL_FREE_SPACE); //Free space - lvalPage.put((byte) 'L'); - lvalPage.put((byte) 'V'); - lvalPage.put((byte) 'A'); - lvalPage.put((byte) 'L'); - lvalPage.putInt(0); //unknown - lvalPage.putShort((short)0); // num rows in page - } /** * Passes the given obj through the currently configured validator for this @@ -1230,60 +981,43 @@ public class ColumnImpl implements Column, Comparable { return ByteBuffer.wrap(((RawData)obj).getBytes()); } + return writeRealData(obj, remainingRowLength, order); + } + + protected ByteBuffer writeRealData(Object obj, int remainingRowLength, + ByteOrder order) + throws IOException + { if(!isVariableLength() || !getType().isVariableLength()) { return writeFixedLengthField(obj, order); } - // var length column - if(!getType().isLongValue()) { - - // this is an "inline" var length field - switch(getType()) { - case NUMERIC: - // don't ask me why numerics are "var length" columns... - ByteBuffer buffer = getPageChannel().createBuffer( - getType().getFixedSize(), order); - writeNumericValue(buffer, obj); - buffer.flip(); - return buffer; - - case TEXT: - byte[] encodedData = encodeTextValue( - obj, 0, getLengthInUnits(), false).array(); - obj = encodedData; - break; - - case BINARY: - case UNKNOWN_0D: - case UNSUPPORTED_VARLEN: - // should already be "encoded" - break; - default: - throw new RuntimeException("unexpected inline var length type: " + - getType()); - } - - ByteBuffer buffer = ByteBuffer.wrap(toByteArray(obj)); - buffer.order(order); + // this is an "inline" var length field + switch(getType()) { + case NUMERIC: + // don't ask me why numerics are "var length" columns... + ByteBuffer buffer = getPageChannel().createBuffer( + getType().getFixedSize(), order); + writeNumericValue(buffer, obj); + buffer.flip(); return buffer; - } - // var length, long value column - switch(getType()) { - case OLE: + case TEXT: + return encodeTextValue( + obj, 0, getLengthInUnits(), false).order(order); + + case BINARY: + case UNKNOWN_0D: + case UNSUPPORTED_VARLEN: // should already be "encoded" break; - case MEMO: - int maxMemoChars = DataType.MEMO.toUnitSize(DataType.MEMO.getMaxSize()); - obj = encodeTextValue(obj, 0, maxMemoChars, false).array(); - break; default: - throw new RuntimeException("unexpected var length, long value type: " + + throw new RuntimeException("unexpected inline var length type: " + getType()); - } + } - // create long value buffer - return writeLongValue(toByteArray(obj), remainingRowLength); + ByteBuffer buffer = ByteBuffer.wrap(toByteArray(obj)).order(order); + return buffer; } /** @@ -1293,14 +1027,18 @@ public class ColumnImpl implements Column, Comparable { * @return A buffer containing the bytes * @usage _advanced_method_ */ - public ByteBuffer writeFixedLengthField(Object obj, ByteOrder order) + protected ByteBuffer writeFixedLengthField(Object obj, ByteOrder order) throws IOException { int size = getType().getFixedSize(_columnLength); - // create buffer for data - ByteBuffer buffer = getPageChannel().createBuffer(size, order); + return writeFixedLengthField( + obj, getPageChannel().createBuffer(size, order)); + } + protected ByteBuffer writeFixedLengthField(Object obj, ByteBuffer buffer) + throws IOException + { // since booleans are not written by this method, it's safe to convert any // incoming boolean into an integer. obj = booleanToInteger(obj); @@ -1338,7 +1076,7 @@ public class ColumnImpl implements Column, Comparable { buffer.put(encodeTextValue(obj, numChars, numChars, true)); break; case GUID: - writeGUIDValue(buffer, obj, order); + writeGUIDValue(buffer, obj); break; case NUMERIC: // yes, that's right, occasionally numeric values are written as fixed @@ -1369,7 +1107,7 @@ public class ColumnImpl implements Column, Comparable { /** * Decodes a compressed or uncompressed text value. */ - private String decodeTextValue(byte[] data) + String decodeTextValue(byte[] data) throws IOException { try { @@ -1464,8 +1202,8 @@ public class ColumnImpl implements Column, Comparable { /** * Encodes a text value, possibly compressing. */ - private ByteBuffer encodeTextValue(Object obj, int minChars, int maxChars, - boolean forceUncompressed) + ByteBuffer encodeTextValue(Object obj, int minChars, int maxChars, + boolean forceUncompressed) throws IOException { CharSequence text = toCharSequence(obj); @@ -1556,7 +1294,10 @@ public class ColumnImpl implements Column, Comparable { " (" + _type + ")") .append("number", _columnNumber) .append("length", _columnLength) - .append("variableLength", _variableLength); + .append("variableLength", _variableLength); + if(_calculated) { + sb.append("calculated", _calculated); + } if(_type.isTextual()) { sb.append("compressedUnicode", isCompressedUnicode()) .append("textSortOrder", getTextSortOrder()); @@ -1569,7 +1310,11 @@ public class ColumnImpl implements Column, Comparable { if(isHyperlink()) { sb.append("hyperlink", isHyperlink()); } - } + } + if(_type.getHasScalePrecision()) { + sb.append("precision", getPrecision()) + .append("scale", getScale()); + } if(_autoNumber) { sb.append("lastAutoNumber", _autoNumberGenerator.getLast()); } @@ -1657,7 +1402,7 @@ public class ColumnImpl implements Column, Comparable { * null is returned as 0 and Numbers are converted * using their double representation. */ - private static BigDecimal toBigDecimal(Object value) + static BigDecimal toBigDecimal(Object value) { if(value == null) { return BigDecimal.ZERO; @@ -1770,8 +1515,8 @@ public class ColumnImpl implements Column, Comparable { private static void fixNumericByteOrder(byte[] bytes) { // fix endianness of each 4 byte segment - for(int i = 0; i < 4; ++i) { - ByteUtil.swap4Bytes(bytes, i * 4); + for(int i = 0; i < bytes.length; i+=4) { + ByteUtil.swap4Bytes(bytes, i); } } @@ -1909,6 +1654,15 @@ public class ColumnImpl implements Column, Comparable { int cpOffset = format.OFFSET_COLUMN_CODE_PAGE; return ((cpOffset >= 0) ? buffer.getShort(offset + cpOffset) : 0); } + + /** + * Read the extra flags field for a column definition. + */ + static byte readExtraFlags(ByteBuffer buffer, int offset, JetFormat format) + { + int extFlagsOffset = format.OFFSET_COLUMN_EXT_FLAGS; + return ((extFlagsOffset >= 0) ? buffer.get(offset + extFlagsOffset) : 0); + } /** * Writes the sort order info to the given buffer at the current position. @@ -1933,7 +1687,7 @@ public class ColumnImpl implements Column, Comparable { // for now, the only mutable value this class returns is byte[] return !(value instanceof byte[]); } - + /** * Date subclass which stashes the original date bits, in case we attempt to * re-write the value (will not lose precision). @@ -2192,55 +1946,31 @@ public class ColumnImpl implements Column, Comparable { } /** - * Manages secondary page buffers for long value writing. + * Utility struct for passing params through ColumnImpl constructors. */ - abstract class LongValueBufferHolder + static final class InitArgs { - /** - * Returns a long value data page with space for data of the given length. - */ - public ByteBuffer getLongValuePage(int dataLength) throws IOException { - - TempPageHolder lvalBufferH = getBufferHolder(); - dataLength = Math.min(dataLength, getFormat().MAX_LONG_VALUE_ROW_SIZE); - - ByteBuffer lvalPage = null; - if(lvalBufferH.getPageNumber() != PageChannel.INVALID_PAGE_NUMBER) { - lvalPage = lvalBufferH.getPage(getPageChannel()); - if(TableImpl.rowFitsOnDataPage(dataLength, lvalPage, getFormat())) { - // the current page has space - return lvalPage; - } - } - - // need new page - return findNewPage(dataLength); - } - - protected ByteBuffer findNewPage(int dataLength) throws IOException { - ByteBuffer lvalPage = getBufferHolder().setNewPage(getPageChannel()); - writeLongValueHeader(lvalPage); - return lvalPage; - } - - public int getOwnedPageCount() { - return 0; - } - - /** - * Returns the page number of the current long value data page. - */ - public int getPageNumber() { - return getBufferHolder().getPageNumber(); - } - - /** - * Discards the current the current long value data page. - */ - public void clear() throws IOException { - getBufferHolder().clear(); + public final TableImpl table; + public final ByteBuffer buffer; + public final int offset; + public final String name; + public final int displayIndex; + public final byte colType; + public final byte flags; + public final byte extFlags; + public DataType type; + + InitArgs(TableImpl table, ByteBuffer buffer, int offset, String name, + int displayIndex) { + this.table = table; + this.buffer = buffer; + this.offset = offset; + this.name = name; + this.displayIndex = displayIndex; + + this.colType = buffer.get(offset + table.getFormat().OFFSET_COLUMN_TYPE); + this.flags = buffer.get(offset + table.getFormat().OFFSET_COLUMN_FLAGS); + this.extFlags = readExtraFlags(buffer, offset, table.getFormat()); } - - protected abstract TempPageHolder getBufferHolder(); } } diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/ComplexColumnImpl.java b/src/main/java/com/healthmarketscience/jackcess/impl/ComplexColumnImpl.java index 85036a6..61edbaa 100644 --- a/src/main/java/com/healthmarketscience/jackcess/impl/ComplexColumnImpl.java +++ b/src/main/java/com/healthmarketscience/jackcess/impl/ComplexColumnImpl.java @@ -20,9 +20,7 @@ USA package com.healthmarketscience.jackcess.impl; import java.io.IOException; -import java.nio.ByteBuffer; -import com.healthmarketscience.jackcess.DataType; import com.healthmarketscience.jackcess.complex.ComplexColumnInfo; import com.healthmarketscience.jackcess.complex.ComplexValue; import com.healthmarketscience.jackcess.impl.complex.ComplexColumnInfoImpl; @@ -38,12 +36,10 @@ class ComplexColumnImpl extends ColumnImpl /** additional information specific to complex columns */ private final ComplexColumnInfo _complexInfo; - ComplexColumnImpl(TableImpl table, ByteBuffer buffer, int offset, - int displayIndex, DataType type, byte flags) - throws IOException + ComplexColumnImpl(InitArgs args) throws IOException { - super(table, buffer, offset, displayIndex, type, flags); - _complexInfo = ComplexColumnSupport.create(this, buffer, offset); + super(args); + _complexInfo = ComplexColumnSupport.create(this, args.buffer, args.offset); } @Override diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/JetFormat.java b/src/main/java/com/healthmarketscience/jackcess/impl/JetFormat.java index ad5eb28..ebc9081 100644 --- a/src/main/java/com/healthmarketscience/jackcess/impl/JetFormat.java +++ b/src/main/java/com/healthmarketscience/jackcess/impl/JetFormat.java @@ -207,7 +207,7 @@ public abstract class JetFormat { public final int OFFSET_COLUMN_CODE_PAGE; public final int OFFSET_COLUMN_COMPLEX_ID; public final int OFFSET_COLUMN_FLAGS; - public final int OFFSET_COLUMN_COMPRESSED_UNICODE; + public final int OFFSET_COLUMN_EXT_FLAGS; public final int OFFSET_COLUMN_LENGTH; public final int OFFSET_COLUMN_VARIABLE_TABLE_INDEX; public final int OFFSET_COLUMN_FIXED_DATA_OFFSET; @@ -342,7 +342,7 @@ public abstract class JetFormat { OFFSET_COLUMN_CODE_PAGE = defineOffsetColumnCodePage(); OFFSET_COLUMN_COMPLEX_ID = defineOffsetColumnComplexId(); OFFSET_COLUMN_FLAGS = defineOffsetColumnFlags(); - OFFSET_COLUMN_COMPRESSED_UNICODE = defineOffsetColumnCompressedUnicode(); + OFFSET_COLUMN_EXT_FLAGS = defineOffsetColumnExtFlags(); OFFSET_COLUMN_LENGTH = defineOffsetColumnLength(); OFFSET_COLUMN_VARIABLE_TABLE_INDEX = defineOffsetColumnVariableTableIndex(); OFFSET_COLUMN_FIXED_DATA_OFFSET = defineOffsetColumnFixedDataOffset(); @@ -445,7 +445,7 @@ public abstract class JetFormat { protected abstract int defineOffsetColumnCodePage(); protected abstract int defineOffsetColumnComplexId(); protected abstract int defineOffsetColumnFlags(); - protected abstract int defineOffsetColumnCompressedUnicode(); + protected abstract int defineOffsetColumnExtFlags(); protected abstract int defineOffsetColumnLength(); protected abstract int defineOffsetColumnVariableTableIndex(); protected abstract int defineOffsetColumnFixedDataOffset(); @@ -611,7 +611,7 @@ public abstract class JetFormat { @Override protected int defineOffsetColumnFlags() { return 13; } @Override - protected int defineOffsetColumnCompressedUnicode() { return 16; } + protected int defineOffsetColumnExtFlags() { return -1; } @Override protected int defineOffsetColumnLength() { return 16; } @Override @@ -836,7 +836,7 @@ public abstract class JetFormat { @Override protected int defineOffsetColumnFlags() { return 15; } @Override - protected int defineOffsetColumnCompressedUnicode() { return 16; } + protected int defineOffsetColumnExtFlags() { return 16; } @Override protected int defineOffsetColumnLength() { return 23; } @Override diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/LongValueColumnImpl.java b/src/main/java/com/healthmarketscience/jackcess/impl/LongValueColumnImpl.java index 3ac1093..1f89b38 100644 --- a/src/main/java/com/healthmarketscience/jackcess/impl/LongValueColumnImpl.java +++ b/src/main/java/com/healthmarketscience/jackcess/impl/LongValueColumnImpl.java @@ -21,6 +21,7 @@ package com.healthmarketscience.jackcess.impl; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import com.healthmarketscience.jackcess.DataType; @@ -32,19 +33,34 @@ import com.healthmarketscience.jackcess.DataType; */ class LongValueColumnImpl extends ColumnImpl { + /** + * Long value (LVAL) type that indicates that the value is stored on the + * same page + */ + private static final byte LONG_VALUE_TYPE_THIS_PAGE = (byte) 0x80; + /** + * Long value (LVAL) type that indicates that the value is stored on another + * page + */ + private static final byte LONG_VALUE_TYPE_OTHER_PAGE = (byte) 0x40; + /** + * Long value (LVAL) type that indicates that the value is stored on + * multiple other pages + */ + private static final byte LONG_VALUE_TYPE_OTHER_PAGES = (byte) 0x00; + /** + * Mask to apply the long length in order to get the flag bits (only the + * first 2 bits are type flags). + */ + private static final int LONG_VALUE_TYPE_MASK = 0xC0000000; + + /** Holds additional info for writing long values */ private LongValueBufferHolder _lvalBufferH; - LongValueColumnImpl(TableImpl table, ByteBuffer buffer, int offset, - int displayIndex, DataType type, byte flags) - throws IOException + LongValueColumnImpl(InitArgs args) throws IOException { - super(table, buffer, offset, displayIndex, type, flags); - } - - @Override - LongValueBufferHolder getLongValueBufferHolder() { - return _lvalBufferH; + super(args); } @Override @@ -64,7 +80,362 @@ class LongValueColumnImpl extends ColumnImpl } super.postTableLoadInit(); } + + @Override + public Object read(byte[] data, ByteOrder order) throws IOException { + switch(getType()) { + case OLE: + if (data.length > 0) { + return readLongValue(data); + } + return null; + case MEMO: + if (data.length > 0) { + return readLongStringValue(data); + } + return null; + default: + throw new RuntimeException("unexpected var length, long value type: " + + getType()); + } + } + + @Override + protected ByteBuffer writeRealData(Object obj, int remainingRowLength, + ByteOrder order) + throws IOException + { + switch(getType()) { + case OLE: + // should already be "encoded" + break; + case MEMO: + int maxMemoChars = DataType.MEMO.toUnitSize(DataType.MEMO.getMaxSize()); + obj = encodeTextValue(obj, 0, maxMemoChars, false).array(); + break; + default: + throw new RuntimeException("unexpected var length, long value type: " + + getType()); + } + + // create long value buffer + return writeLongValue(toByteArray(obj), remainingRowLength); + } + + /** + * @param lvalDefinition Column value that points to an LVAL record + * @return The LVAL data + */ + protected byte[] readLongValue(byte[] lvalDefinition) + throws IOException + { + ByteBuffer def = PageChannel.wrap(lvalDefinition); + int lengthWithFlags = def.getInt(); + int length = lengthWithFlags & (~LONG_VALUE_TYPE_MASK); + + byte[] rtn = new byte[length]; + byte type = (byte)((lengthWithFlags & LONG_VALUE_TYPE_MASK) >>> 24); + + if(type == LONG_VALUE_TYPE_THIS_PAGE) { + + // inline long value + def.getInt(); //Skip over lval_dp + def.getInt(); //Skip over unknown + + int rowLen = def.remaining(); + if(rowLen < length) { + // warn the caller, but return whatever we can + LOG.warn(getName() + " value may be truncated: expected length " + + length + " found " + rowLen); + rtn = new byte[rowLen]; + } + + def.get(rtn); + + } else { + + // long value on other page(s) + if (lvalDefinition.length != getFormat().SIZE_LONG_VALUE_DEF) { + throw new IOException("Expected " + getFormat().SIZE_LONG_VALUE_DEF + + " bytes in long value definition, but found " + + lvalDefinition.length); + } + + int rowNum = ByteUtil.getUnsignedByte(def); + int pageNum = ByteUtil.get3ByteInt(def, def.position()); + ByteBuffer lvalPage = getPageChannel().createPageBuffer(); + + switch (type) { + case LONG_VALUE_TYPE_OTHER_PAGE: + { + getPageChannel().readPage(lvalPage, pageNum); + + short rowStart = TableImpl.findRowStart(lvalPage, rowNum, getFormat()); + short rowEnd = TableImpl.findRowEnd(lvalPage, rowNum, getFormat()); + + int rowLen = rowEnd - rowStart; + if(rowLen < length) { + // warn the caller, but return whatever we can + LOG.warn(getName() + " value may be truncated: expected length " + + length + " found " + rowLen); + rtn = new byte[rowLen]; + } + + lvalPage.position(rowStart); + lvalPage.get(rtn); + } + break; + + case LONG_VALUE_TYPE_OTHER_PAGES: + + ByteBuffer rtnBuf = ByteBuffer.wrap(rtn); + int remainingLen = length; + while(remainingLen > 0) { + lvalPage.clear(); + getPageChannel().readPage(lvalPage, pageNum); + + short rowStart = TableImpl.findRowStart(lvalPage, rowNum, getFormat()); + short rowEnd = TableImpl.findRowEnd(lvalPage, rowNum, getFormat()); + + // read next page information + lvalPage.position(rowStart); + rowNum = ByteUtil.getUnsignedByte(lvalPage); + pageNum = ByteUtil.get3ByteInt(lvalPage); + + // update rowEnd and remainingLen based on chunkLength + int chunkLength = (rowEnd - rowStart) - 4; + if(chunkLength > remainingLen) { + rowEnd = (short)(rowEnd - (chunkLength - remainingLen)); + chunkLength = remainingLen; + } + remainingLen -= chunkLength; + + lvalPage.limit(rowEnd); + rtnBuf.put(lvalPage); + } + + break; + + default: + throw new IOException("Unrecognized long value type: " + type); + } + } + + return rtn; + } + /** + * @param lvalDefinition Column value that points to an LVAL record + * @return The LVAL data + */ + private String readLongStringValue(byte[] lvalDefinition) + throws IOException + { + byte[] binData = readLongValue(lvalDefinition); + if((binData == null) || (binData.length == 0)) { + return null; + } + return decodeTextValue(binData); + } + + /** + * Write an LVAL column into a ByteBuffer inline if it fits, otherwise in + * other data page(s). + * @param value Value of the LVAL column + * @return A buffer containing the LVAL definition and (possibly) the column + * value (unless written to other pages) + * @usage _advanced_method_ + */ + protected ByteBuffer writeLongValue(byte[] value, int remainingRowLength) + throws IOException + { + if(value.length > getType().getMaxSize()) { + throw new IOException("value too big for column, max " + + getType().getMaxSize() + ", got " + + value.length); + } + + // determine which type to write + byte type = 0; + int lvalDefLen = getFormat().SIZE_LONG_VALUE_DEF; + if(((getFormat().SIZE_LONG_VALUE_DEF + value.length) <= remainingRowLength) + && (value.length <= getFormat().MAX_INLINE_LONG_VALUE_SIZE)) { + type = LONG_VALUE_TYPE_THIS_PAGE; + lvalDefLen += value.length; + } else if(value.length <= getFormat().MAX_LONG_VALUE_ROW_SIZE) { + type = LONG_VALUE_TYPE_OTHER_PAGE; + } else { + type = LONG_VALUE_TYPE_OTHER_PAGES; + } + + ByteBuffer def = getPageChannel().createBuffer(lvalDefLen); + // take length and apply type to first byte + int lengthWithFlags = value.length | (type << 24); + def.putInt(lengthWithFlags); + + if(type == LONG_VALUE_TYPE_THIS_PAGE) { + // write long value inline + def.putInt(0); + def.putInt(0); //Unknown + def.put(value); + } else { + + ByteBuffer lvalPage = null; + int firstLvalPageNum = PageChannel.INVALID_PAGE_NUMBER; + byte firstLvalRow = 0; + + // write other page(s) + switch(type) { + case LONG_VALUE_TYPE_OTHER_PAGE: + lvalPage = _lvalBufferH.getLongValuePage(value.length); + firstLvalPageNum = _lvalBufferH.getPageNumber(); + firstLvalRow = (byte)TableImpl.addDataPageRow(lvalPage, value.length, + getFormat(), 0); + lvalPage.put(value); + getPageChannel().writePage(lvalPage, firstLvalPageNum); + break; + + case LONG_VALUE_TYPE_OTHER_PAGES: + + ByteBuffer buffer = ByteBuffer.wrap(value); + int remainingLen = buffer.remaining(); + buffer.limit(0); + lvalPage = _lvalBufferH.getLongValuePage(remainingLen); + firstLvalPageNum = _lvalBufferH.getPageNumber(); + firstLvalRow = (byte)TableImpl.getRowsOnDataPage(lvalPage, getFormat()); + int lvalPageNum = firstLvalPageNum; + ByteBuffer nextLvalPage = null; + int nextLvalPageNum = 0; + int nextLvalRowNum = 0; + while(remainingLen > 0) { + lvalPage.clear(); + + // figure out how much we will put in this page (we need 4 bytes for + // the next page pointer) + int chunkLength = Math.min(getFormat().MAX_LONG_VALUE_ROW_SIZE - 4, + remainingLen); + + // figure out if we will need another page, and if so, allocate it + if(chunkLength < remainingLen) { + // force a new page to be allocated for the chunk after this + _lvalBufferH.clear(); + nextLvalPage = _lvalBufferH.getLongValuePage( + (remainingLen - chunkLength) + 4); + nextLvalPageNum = _lvalBufferH.getPageNumber(); + nextLvalRowNum = TableImpl.getRowsOnDataPage(nextLvalPage, + getFormat()); + } else { + nextLvalPage = null; + nextLvalPageNum = 0; + nextLvalRowNum = 0; + } + + // add row to this page + TableImpl.addDataPageRow(lvalPage, chunkLength + 4, getFormat(), 0); + + // write next page info + lvalPage.put((byte)nextLvalRowNum); // row number + ByteUtil.put3ByteInt(lvalPage, nextLvalPageNum); // page number + + // write this page's chunk of data + buffer.limit(buffer.limit() + chunkLength); + lvalPage.put(buffer); + remainingLen -= chunkLength; + + // write new page to database + getPageChannel().writePage(lvalPage, lvalPageNum); + + // move to next page + lvalPage = nextLvalPage; + lvalPageNum = nextLvalPageNum; + } + break; + + default: + throw new IOException("Unrecognized long value type: " + type); + } + + // update def + def.put(firstLvalRow); + ByteUtil.put3ByteInt(def, firstLvalPageNum); + def.putInt(0); //Unknown + + } + + def.flip(); + return def; + } + + /** + * Writes the header info for a long value page. + */ + private void writeLongValueHeader(ByteBuffer lvalPage) + { + lvalPage.put(PageTypes.DATA); //Page type + lvalPage.put((byte) 1); //Unknown + lvalPage.putShort((short)getFormat().DATA_PAGE_INITIAL_FREE_SPACE); //Free space + lvalPage.put((byte) 'L'); + lvalPage.put((byte) 'V'); + lvalPage.put((byte) 'A'); + lvalPage.put((byte) 'L'); + lvalPage.putInt(0); //unknown + lvalPage.putShort((short)0); // num rows in page + } + + + /** + * Manages secondary page buffers for long value writing. + */ + private abstract class LongValueBufferHolder + { + /** + * Returns a long value data page with space for data of the given length. + */ + public ByteBuffer getLongValuePage(int dataLength) throws IOException { + + TempPageHolder lvalBufferH = getBufferHolder(); + dataLength = Math.min(dataLength, getFormat().MAX_LONG_VALUE_ROW_SIZE); + + ByteBuffer lvalPage = null; + if(lvalBufferH.getPageNumber() != PageChannel.INVALID_PAGE_NUMBER) { + lvalPage = lvalBufferH.getPage(getPageChannel()); + if(TableImpl.rowFitsOnDataPage(dataLength, lvalPage, getFormat())) { + // the current page has space + return lvalPage; + } + } + + // need new page + return findNewPage(dataLength); + } + + protected ByteBuffer findNewPage(int dataLength) throws IOException { + ByteBuffer lvalPage = getBufferHolder().setNewPage(getPageChannel()); + writeLongValueHeader(lvalPage); + return lvalPage; + } + + public int getOwnedPageCount() { + return 0; + } + + /** + * Returns the page number of the current long value data page. + */ + public int getPageNumber() { + return getBufferHolder().getPageNumber(); + } + + /** + * Discards the current the current long value data page. + */ + public void clear() throws IOException { + getBufferHolder().clear(); + } + + protected abstract TempPageHolder getBufferHolder(); + } + /** * Manages a common, shared extra page for long values. This is legacy * behavior from before it was understood that there were additional usage diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/MemoColumnImpl.java b/src/main/java/com/healthmarketscience/jackcess/impl/MemoColumnImpl.java index 8a9c742..dbd4023 100644 --- a/src/main/java/com/healthmarketscience/jackcess/impl/MemoColumnImpl.java +++ b/src/main/java/com/healthmarketscience/jackcess/impl/MemoColumnImpl.java @@ -20,8 +20,6 @@ USA package com.healthmarketscience.jackcess.impl; import java.io.IOException; -import com.healthmarketscience.jackcess.DataType; -import java.nio.ByteBuffer; /** * ColumnImpl subclass which is used for Memo data types. @@ -44,22 +42,21 @@ class MemoColumnImpl extends LongValueColumnImpl of type MEMO) */ private boolean _hyperlink; - MemoColumnImpl(TableImpl table, ByteBuffer buffer, int offset, - int displayIndex, DataType type, byte flags) - throws IOException + MemoColumnImpl(InitArgs args) throws IOException { - super(table, buffer, offset, displayIndex, type, flags); + super(args); // co-located w/ precision/scale _sortOrder = readSortOrder( - buffer, offset + getFormat().OFFSET_COLUMN_SORT_ORDER, getFormat()); - _codePage = readCodePage(buffer, offset, getFormat()); + args.buffer, args.offset + getFormat().OFFSET_COLUMN_SORT_ORDER, + getFormat()); + _codePage = readCodePage(args.buffer, args.offset, getFormat()); - _compressedUnicode = ((buffer.get(offset + - getFormat().OFFSET_COLUMN_COMPRESSED_UNICODE) & 1) == 1); + _compressedUnicode = + ((args.extFlags & COMPRESSED_UNICODE_EXT_FLAG_MASK) != 0); // only memo fields can be hyperlinks - _hyperlink = ((flags & HYPERLINK_FLAG_MASK) != 0); + _hyperlink = ((args.flags & HYPERLINK_FLAG_MASK) != 0); } @Override diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/NumericColumnImpl.java b/src/main/java/com/healthmarketscience/jackcess/impl/NumericColumnImpl.java index 667315a..cb19a42 100644 --- a/src/main/java/com/healthmarketscience/jackcess/impl/NumericColumnImpl.java +++ b/src/main/java/com/healthmarketscience/jackcess/impl/NumericColumnImpl.java @@ -20,8 +20,6 @@ USA package com.healthmarketscience.jackcess.impl; import java.io.IOException; -import com.healthmarketscience.jackcess.DataType; -import java.nio.ByteBuffer; /** * ColumnImpl subclass which is used for numeric data types. @@ -36,14 +34,13 @@ class NumericColumnImpl extends ColumnImpl /** Numeric scale */ private final byte _scale; - NumericColumnImpl(TableImpl table, ByteBuffer buffer, int offset, - int displayIndex, DataType type, byte flags) - throws IOException + NumericColumnImpl(InitArgs args) throws IOException { - super(table, buffer, offset, displayIndex, type, flags); + super(args); - _precision = buffer.get(offset + getFormat().OFFSET_COLUMN_PRECISION); - _scale = buffer.get(offset + getFormat().OFFSET_COLUMN_SCALE); + _precision = args.buffer.get( + args.offset + getFormat().OFFSET_COLUMN_PRECISION); + _scale = args.buffer.get(args.offset + getFormat().OFFSET_COLUMN_SCALE); } @Override diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/PropertyMaps.java b/src/main/java/com/healthmarketscience/jackcess/impl/PropertyMaps.java index 50712bc..bff4472 100644 --- a/src/main/java/com/healthmarketscience/jackcess/impl/PropertyMaps.java +++ b/src/main/java/com/healthmarketscience/jackcess/impl/PropertyMaps.java @@ -448,7 +448,7 @@ public class PropertyMaps implements Iterable private class PropColumn extends ColumnImpl { private PropColumn(DataType type) { - super(null, type, 0, 0, 0); + super(null, null, type, 0, 0, 0); } @Override diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/TableImpl.java b/src/main/java/com/healthmarketscience/jackcess/impl/TableImpl.java index 3f22847..3ab142c 100644 --- a/src/main/java/com/healthmarketscience/jackcess/impl/TableImpl.java +++ b/src/main/java/com/healthmarketscience/jackcess/impl/TableImpl.java @@ -692,11 +692,11 @@ public class TableImpl implements Table NullMask nullMask = rowState.getNullMask(rowBuffer); boolean isNull = nullMask.isNull(column); - if(column.getType() == DataType.BOOLEAN) { + if(column.storeInNullMask()) { // Boolean values are stored in the null mask. see note about // caching below return rowState.setRowCacheValue(column.getColumnIndex(), - Boolean.valueOf(!isNull)); + column.readFromNullMask(isNull)); } else if(isNull) { // well, that's easy! (no need to update cache w/ null) return null; @@ -995,8 +995,8 @@ public class TableImpl implements Table // now, create the table definition PageChannel pageChannel = creator.getPageChannel(); - ByteBuffer buffer = pageChannel .createBuffer(Math.max(totalTableDefSize, - format.PAGE_SIZE)); + ByteBuffer buffer = pageChannel.createBuffer(Math.max(totalTableDefSize, + format.PAGE_SIZE)); writeTableDefinitionHeader(creator, buffer, totalTableDefSize); if(creator.hasIndexes()) { @@ -1303,10 +1303,19 @@ public class TableImpl implements Table { int colOffset = getFormat().OFFSET_INDEX_DEF_BLOCK + _indexCount * getFormat().SIZE_INDEX_DEFINITION; + + tableBuffer.position(colOffset + + (columnCount * getFormat().SIZE_COLUMN_HEADER)); + List colNames = new ArrayList(columnCount); + for (int i = 0; i < columnCount; i++) { + colNames.add(readName(tableBuffer)); + } + int dispIndex = 0; for (int i = 0; i < columnCount; i++) { ColumnImpl column = ColumnImpl.create(this, tableBuffer, - colOffset + (i * getFormat().SIZE_COLUMN_HEADER), dispIndex++); + colOffset + (i * getFormat().SIZE_COLUMN_HEADER), colNames.get(i), + dispIndex++); _columns.add(column); if(column.isVariableLength()) { // also shove it in the variable columns list, which is ordered @@ -1314,12 +1323,7 @@ public class TableImpl implements Table _varColumns.add(column); } } - tableBuffer.position(colOffset + - (columnCount * getFormat().SIZE_COLUMN_HEADER)); - for (int i = 0; i < columnCount; i++) { - ColumnImpl column = _columns.get(i); - column.setName(readName(tableBuffer)); - } + Collections.sort(_columns); getAutoNumberColumns(); @@ -2056,10 +2060,9 @@ public class TableImpl implements Table Object rowValue = col.getRowValue(rowArray); - if (col.getType() == DataType.BOOLEAN) { + if (col.storeInNullMask()) { - if(ColumnImpl.toBooleanValue(rowValue)) { - //Booleans are stored in the null mask + if(col.writeToNullMask(rowValue)) { nullMask.markNotNull(col); } rowValue = null; diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/TextColumnImpl.java b/src/main/java/com/healthmarketscience/jackcess/impl/TextColumnImpl.java index 0966d5b..d7d169f 100644 --- a/src/main/java/com/healthmarketscience/jackcess/impl/TextColumnImpl.java +++ b/src/main/java/com/healthmarketscience/jackcess/impl/TextColumnImpl.java @@ -20,8 +20,6 @@ USA package com.healthmarketscience.jackcess.impl; import java.io.IOException; -import com.healthmarketscience.jackcess.DataType; -import java.nio.ByteBuffer; /** * ColumnImpl subclass which is used for Text data types. @@ -38,19 +36,18 @@ class TextColumnImpl extends ColumnImpl /** the code page for a text field (for certain db versions) */ private final short _codePage; - TextColumnImpl(TableImpl table, ByteBuffer buffer, int offset, - int displayIndex, DataType type, byte flags) - throws IOException + TextColumnImpl(InitArgs args) throws IOException { - super(table, buffer, offset, displayIndex, type, flags); + super(args); // co-located w/ precision/scale _sortOrder = readSortOrder( - buffer, offset + getFormat().OFFSET_COLUMN_SORT_ORDER, getFormat()); - _codePage = readCodePage(buffer, offset, getFormat()); + args.buffer, args.offset + getFormat().OFFSET_COLUMN_SORT_ORDER, + getFormat()); + _codePage = readCodePage(args.buffer, args.offset, getFormat()); - _compressedUnicode = ((buffer.get(offset + - getFormat().OFFSET_COLUMN_COMPRESSED_UNICODE) & 1) == 1); + _compressedUnicode = + ((args.extFlags & COMPRESSED_UNICODE_EXT_FLAG_MASK) != 0); } @Override diff --git a/src/main/java/com/healthmarketscience/jackcess/impl/UnsupportedColumnImpl.java b/src/main/java/com/healthmarketscience/jackcess/impl/UnsupportedColumnImpl.java index 5165a53..6418ba9 100644 --- a/src/main/java/com/healthmarketscience/jackcess/impl/UnsupportedColumnImpl.java +++ b/src/main/java/com/healthmarketscience/jackcess/impl/UnsupportedColumnImpl.java @@ -20,9 +20,8 @@ USA package com.healthmarketscience.jackcess.impl; import java.io.IOException; -import java.nio.ByteBuffer; +import java.nio.ByteOrder; -import com.healthmarketscience.jackcess.DataType; /** * ColumnImpl subclass which is used for unknown/unsupported data types. @@ -34,18 +33,19 @@ class UnsupportedColumnImpl extends ColumnImpl { private final byte _originalType; - UnsupportedColumnImpl(TableImpl table, ByteBuffer buffer, int offset, - int displayIndex, DataType type, byte flags, - byte originalType) - throws IOException + UnsupportedColumnImpl(InitArgs args) throws IOException { - super(table, buffer, offset, displayIndex, type, flags); - _originalType = originalType; + super(args); + _originalType = args.colType; } @Override byte getOriginalDataType() { return _originalType; } - + + @Override + public Object read(byte[] data, ByteOrder order) throws IOException { + return rawDataWrapper(data); + } } diff --git a/src/test/java/com/healthmarketscience/jackcess/DatabaseTest.java b/src/test/java/com/healthmarketscience/jackcess/DatabaseTest.java index e146aef..a9178fe 100644 --- a/src/test/java/com/healthmarketscience/jackcess/DatabaseTest.java +++ b/src/test/java/com/healthmarketscience/jackcess/DatabaseTest.java @@ -1443,7 +1443,7 @@ public class DatabaseTest extends TestCase private static void doTestTimeZone(final TimeZone tz) throws Exception { - ColumnImpl col = new ColumnImpl(null, DataType.SHORT_DATE_TIME, 0, 0, 0) { + ColumnImpl col = new ColumnImpl(null, null, DataType.SHORT_DATE_TIME, 0, 0, 0) { @Override protected Calendar getCalendar() { return Calendar.getInstance(tz); } }; diff --git a/src/test/java/com/healthmarketscience/jackcess/TableTest.java b/src/test/java/com/healthmarketscience/jackcess/TableTest.java index 29408ef..0ee410a 100644 --- a/src/test/java/com/healthmarketscience/jackcess/TableTest.java +++ b/src/test/java/com/healthmarketscience/jackcess/TableTest.java @@ -87,7 +87,7 @@ public class TableTest extends TestCase { public void testUnicodeCompression() throws Exception { reset(); newTestColumn(DataType.TEXT, false); - newTestColumn(DataType.MEMO, false); + newTestColumn(DataType.TEXT, false); newTestTable(); String small = "this is a string"; @@ -100,7 +100,7 @@ public class TableTest extends TestCase { reset(); newTestColumn(DataType.TEXT, true); - newTestColumn(DataType.MEMO, true); + newTestColumn(DataType.TEXT, true); newTestTable(); ByteBuffer[] bufCmp1 = encodeColumns(small, large); @@ -177,7 +177,8 @@ public class TableTest extends TestCase { _fixedOffset += type.getFixedSize(); } - ColumnImpl col = new ColumnImpl(null, type, nextColIdx, nextFixedOff, nextVarLenIdx) { + ColumnImpl col = new ColumnImpl(null, null, type, nextColIdx, nextFixedOff, + nextVarLenIdx) { @Override public TableImpl getTable() { return _testTable; diff --git a/src/test/java/com/healthmarketscience/jackcess/util/ErrorHandlerTest.java b/src/test/java/com/healthmarketscience/jackcess/util/ErrorHandlerTest.java index 6431ad8..00ff159 100644 --- a/src/test/java/com/healthmarketscience/jackcess/util/ErrorHandlerTest.java +++ b/src/test/java/com/healthmarketscience/jackcess/util/ErrorHandlerTest.java @@ -160,8 +160,7 @@ public class ErrorHandlerTest extends TestCase List cols = (List)colsField.get(t); Column srcCol = null; - ColumnImpl destCol = new BogusColumn(t); - destCol.setName(colName); + ColumnImpl destCol = new BogusColumn(t, colName); for(int i = 0; i < cols.size(); ++i) { srcCol = cols.get(i); if(srcCol.getName().equals(colName)) { @@ -182,8 +181,8 @@ public class ErrorHandlerTest extends TestCase private static class BogusColumn extends ColumnImpl { - private BogusColumn(Table table) { - super((TableImpl)table, DataType.LONG, 1, 0, 0); + private BogusColumn(Table table, String name) { + super((TableImpl)table, name, DataType.LONG, 1, 0, 0); } @Override diff --git a/src/test/java/com/healthmarketscience/jackcess/util/RowFilterTest.java b/src/test/java/com/healthmarketscience/jackcess/util/RowFilterTest.java index 7808a08..e6d79db 100644 --- a/src/test/java/com/healthmarketscience/jackcess/util/RowFilterTest.java +++ b/src/test/java/com/healthmarketscience/jackcess/util/RowFilterTest.java @@ -65,8 +65,7 @@ public class RowFilterTest extends TestCase List rows = Arrays.asList(row0, row1, row2, row3, row4, row5); - ColumnImpl testCol = new ColumnImpl(null, DataType.TEXT, 0, 0, 0) {}; - testCol.setName(COL1); + ColumnImpl testCol = new ColumnImpl(null, COL1, DataType.TEXT, 0, 0, 0) {}; assertEquals(Arrays.asList(row0, row2, row4), toList(RowFilter.matchPattern(testCol, "foo").apply(rows))); -- 2.39.5