From: James Ahlborn Date: Fri, 13 Jul 2007 01:19:09 +0000 (+0000) Subject: add support for writing large table definitions; move table definition code into... X-Git-Tag: rel_1_1_9~3 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=c06862149bc2c2e09db8a1912abb625e3bb60463;p=jackcess.git add support for writing large table definitions; move table definition code into Table (out of Database) git-svn-id: https://svn.code.sf.net/p/jackcess/code/jackcess/trunk@159 f203690c-595d-4dc9-a70b-905162fa7fd2 --- diff --git a/src/java/com/healthmarketscience/jackcess/Column.java b/src/java/com/healthmarketscience/jackcess/Column.java index b9dc208..8612e2c 100644 --- a/src/java/com/healthmarketscience/jackcess/Column.java +++ b/src/java/com/healthmarketscience/jackcess/Column.java @@ -115,7 +115,15 @@ public class Column implements Comparable { public Column(JetFormat format) { _format = format; } - + + /** + * Only used by unit tests + */ + Column(boolean testing) { + _format = JetFormat.VERSION_4; + _pageChannel = new PageChannel(testing); + } + /** * Read a column definition in from a buffer * @param buffer Buffer containing column definition @@ -696,8 +704,7 @@ public class Column implements Comparable { type = LONG_VALUE_TYPE_OTHER_PAGES; } - ByteBuffer def = ByteBuffer.allocate(lvalDefLen); - def.order(ByteOrder.LITTLE_ENDIAN); + ByteBuffer def = _pageChannel.createBuffer(lvalDefLen); ByteUtil.put3ByteInt(def, value.length); def.put(type); @@ -833,8 +840,7 @@ public class Column implements Comparable { switch(getType()) { case NUMERIC: // don't ask me why numerics are "var length" columns... - ByteBuffer buffer = ByteBuffer.allocate(getLength()); - buffer.order(order); + ByteBuffer buffer = _pageChannel.createBuffer(getLength(), order); writeNumericValue(buffer, obj); buffer.flip(); return buffer; @@ -891,8 +897,7 @@ public class Column implements Comparable { int size = getType().getFixedSize(); // create buffer for data - ByteBuffer buffer = ByteBuffer.allocate(size); - buffer.order(order); + ByteBuffer buffer = _pageChannel.createBuffer(size, order); obj = booleanToInteger(obj); diff --git a/src/java/com/healthmarketscience/jackcess/Database.java b/src/java/com/healthmarketscience/jackcess/Database.java index 7c68f21..450d51a 100644 --- a/src/java/com/healthmarketscience/jackcess/Database.java +++ b/src/java/com/healthmarketscience/jackcess/Database.java @@ -87,9 +87,6 @@ public class Database private static final int ACM = 1048319; - /** Free space left in page for new usage map definition pages */ - private static final short USAGE_MAP_DEF_FREE_SPACE = 3940; - private static final String COL_ACM = "ACM"; /** System catalog column name of the date a system object was created */ private static final String COL_DATE_CREATE = "DateCreate"; @@ -425,25 +422,9 @@ public class Database } } - // first, create the usage map page - int usageMapPageNumber = _pageChannel.writeNewPage( - createUsageMapDefinitionBuffer()); - - // now, create the table definition - ByteBuffer buffer = _pageChannel.createPageBuffer(); - writeTableDefinition(buffer, columns, usageMapPageNumber); - writeColumnDefinitions(buffer, columns); - - //End of tabledef - buffer.put((byte) 0xff); - buffer.put((byte) 0xff); - - int tableDefLen = buffer.position(); - buffer.putShort(2, (short)(_format.PAGE_SIZE - tableDefLen - 8)); // overwrite page free space - buffer.putInt(8, tableDefLen); //Overwrite length of data for this page - //Write the tdef page to disk. - int tdefPageNumber = _pageChannel.writeNewPage(buffer); + int tdefPageNumber = Table.writeTableDefinition(columns, _pageChannel, + _format); //Add this table to our internal list. addTable(name, new Integer(tdefPageNumber)); @@ -452,145 +433,7 @@ public class Database addToSystemCatalog(name, tdefPageNumber); addToAccessControlEntries(tdefPageNumber); } - - /** - * @param buffer Buffer to write to - * @param columns List of Columns in the table - * @param pageNumber Page number that this table definition will be written to - */ - private void writeTableDefinition(ByteBuffer buffer, List columns, - int usageMapPageNumber) - throws IOException { - //Start writing the tdef - buffer.put(PageTypes.TABLE_DEF); //Page type - buffer.put((byte) 0x01); //Unknown - buffer.put((byte) 0); //Unknown - buffer.put((byte) 0); //Unknown - buffer.putInt(0); //Next TDEF page pointer - buffer.putInt(0); //Length of data for this page - buffer.put((byte) 0x59); //Unknown - buffer.put((byte) 0x06); //Unknown - buffer.putShort((short) 0); //Unknown - buffer.putInt(0); //Number of rows - buffer.putInt(0); //Autonumber - for (int i = 0; i < 16; i++) { //Unknown - buffer.put((byte) 0); - } - buffer.put(Table.TYPE_USER); //Table type - buffer.putShort((short) columns.size()); //Max columns a row will have - buffer.putShort(Column.countVariableLength(columns)); //Number of variable columns in table - buffer.putShort((short) columns.size()); //Number of columns in table - buffer.putInt(0); //Number of indexes in table - buffer.putInt(0); //Number of indexes in table - buffer.put((byte) 0); //Usage map row number - ByteUtil.put3ByteInt(buffer, usageMapPageNumber); //Usage map page number - buffer.put((byte) 1); //Free map row number - ByteUtil.put3ByteInt(buffer, usageMapPageNumber); //Free map page number - if (LOG.isDebugEnabled()) { - int position = buffer.position(); - buffer.rewind(); - LOG.debug("Creating new table def block:\n" + ByteUtil.toHexString( - buffer, _format.SIZE_TDEF_BLOCK)); - buffer.position(position); - } - } - - /** - * @param buffer Buffer to write to - * @param columns List of Columns to write definitions for - */ - private void writeColumnDefinitions(ByteBuffer buffer, List columns) - throws IOException { - short columnNumber = (short) 0; - short fixedOffset = (short) 0; - short variableOffset = (short) 0; - // we specifically put the "long variable" values after the normal - // variable length values so that we have a better chance of fitting it - // all (because "long variable" values can go in separate pages) - short longVariableOffset = - (short) Column.countNonLongVariableLength(columns); - for (Column col : columns) { - int position = buffer.position(); - buffer.put(col.getType().getValue()); - buffer.put((byte) 0x59); //Unknown - buffer.put((byte) 0x06); //Unknown - buffer.putShort((short) 0); //Unknown - buffer.putShort(columnNumber); //Column Number - if (col.isVariableLength()) { - if(!col.getType().isLongValue()) { - buffer.putShort(variableOffset++); - } else { - buffer.putShort(longVariableOffset++); - } - } else { - buffer.putShort((short) 0); - } - buffer.putShort(columnNumber); //Column Number again - if(col.getType().getHasScalePrecision()) { - buffer.put((byte) col.getPrecision()); // numeric precision - buffer.put((byte) col.getScale()); // numeric scale - } else { - buffer.put((byte) 0x00); //unused - buffer.put((byte) 0x00); //unused - } - buffer.putShort((short) 0); //Unknown - if (col.isVariableLength()) { //Variable length - buffer.put((byte) 0x2); - } else { - buffer.put((byte) 0x3); - } - if (col.isCompressedUnicode()) { //Compressed - buffer.put((byte) 1); - } else { - buffer.put((byte) 0); - } - buffer.putInt(0); //Unknown, but always 0. - //Offset for fixed length columns - if (col.isVariableLength()) { - buffer.putShort((short) 0); - } else { - buffer.putShort(fixedOffset); - fixedOffset += col.getType().getFixedSize(); - } - if(!col.getType().isLongValue()) { - buffer.putShort(col.getLength()); //Column length - } else { - buffer.putShort((short)0x0000); // unused - } - columnNumber++; - if (LOG.isDebugEnabled()) { - LOG.debug("Creating new column def block\n" + ByteUtil.toHexString( - buffer, position, _format.SIZE_COLUMN_DEF_BLOCK)); - } - } - for (Column col : columns) { - ByteBuffer colName = _format.CHARSET.encode(col.getName()); - buffer.putShort((short) colName.remaining()); - buffer.put(colName); - } - } - - /** - * Create the usage map definition page buffer. - */ - private ByteBuffer createUsageMapDefinitionBuffer() throws IOException - { - ByteBuffer rtn = _pageChannel.createPageBuffer(); - rtn.put(PageTypes.DATA); - rtn.put((byte) 0x1); //Unknown - rtn.putShort(USAGE_MAP_DEF_FREE_SPACE); //Free space in page - rtn.putInt(0); //Table definition - rtn.putInt(0); //Unknown - rtn.putShort((short) 2); //Number of records on this page - rtn.putShort((short) _format.OFFSET_USED_PAGES_USAGE_MAP_DEF); //First location - rtn.putShort((short) _format.OFFSET_FREE_PAGES_USAGE_MAP_DEF); //Second location - rtn.position(_format.OFFSET_USED_PAGES_USAGE_MAP_DEF); - rtn.put((byte) UsageMap.MAP_TYPE_REFERENCE); - rtn.position(_format.OFFSET_FREE_PAGES_USAGE_MAP_DEF); - rtn.put((byte) UsageMap.MAP_TYPE_INLINE); - return rtn; - } - + /** * Add a new table to the system catalog * @param name Table name diff --git a/src/java/com/healthmarketscience/jackcess/JetFormat.java b/src/java/com/healthmarketscience/jackcess/JetFormat.java index b88c5f6..0eac460 100644 --- a/src/java/com/healthmarketscience/jackcess/JetFormat.java +++ b/src/java/com/healthmarketscience/jackcess/JetFormat.java @@ -94,7 +94,7 @@ public abstract class JetFormat { public final int OFFSET_ROW_LOCATION_BLOCK; public final int OFFSET_ROW_START; - public final int OFFSET_MAP_START; + public final int OFFSET_USAGE_MAP_START; public final int OFFSET_USAGE_MAP_PAGE_DATA; @@ -103,9 +103,6 @@ public abstract class JetFormat { public final int OFFSET_FREE_SPACE; public final int OFFSET_NUM_ROWS_ON_DATA_PAGE; - public final int OFFSET_USED_PAGES_USAGE_MAP_DEF; - public final int OFFSET_FREE_PAGES_USAGE_MAP_DEF; - public final int OFFSET_INDEX_COMPRESSED_BYTE_COUNT; public final int OFFSET_INDEX_ENTRY_MASK; public final int OFFSET_NEXT_INDEX_LEAF_PAGE; @@ -114,7 +111,8 @@ public abstract class JetFormat { public final int SIZE_COLUMN_HEADER; public final int SIZE_ROW_LOCATION; public final int SIZE_LONG_VALUE_DEF; - public final int SIZE_TDEF_BLOCK; + public final int SIZE_TDEF_HEADER; + public final int SIZE_TDEF_TRAILER; public final int SIZE_COLUMN_DEF_BLOCK; public final int SIZE_INDEX_ENTRY_MASK; @@ -178,7 +176,7 @@ public abstract class JetFormat { OFFSET_ROW_LOCATION_BLOCK = defineOffsetRowLocationBlock(); OFFSET_ROW_START = defineOffsetRowStart(); - OFFSET_MAP_START = defineOffsetMapStart(); + OFFSET_USAGE_MAP_START = defineOffsetUsageMapStart(); OFFSET_USAGE_MAP_PAGE_DATA = defineOffsetUsageMapPageData(); @@ -187,9 +185,6 @@ public abstract class JetFormat { OFFSET_FREE_SPACE = defineOffsetFreeSpace(); OFFSET_NUM_ROWS_ON_DATA_PAGE = defineOffsetNumRowsOnDataPage(); - OFFSET_USED_PAGES_USAGE_MAP_DEF = defineOffsetUsedPagesUsageMapDef(); - OFFSET_FREE_PAGES_USAGE_MAP_DEF = defineOffsetFreePagesUsageMapDef(); - OFFSET_INDEX_COMPRESSED_BYTE_COUNT = defineOffsetIndexCompressedByteCount(); OFFSET_INDEX_ENTRY_MASK = defineOffsetIndexEntryMask(); OFFSET_NEXT_INDEX_LEAF_PAGE = defineOffsetNextIndexLeafPage(); @@ -198,7 +193,8 @@ public abstract class JetFormat { SIZE_COLUMN_HEADER = defineSizeColumnHeader(); SIZE_ROW_LOCATION = defineSizeRowLocation(); SIZE_LONG_VALUE_DEF = defineSizeLongValueDef(); - SIZE_TDEF_BLOCK = defineSizeTdefBlock(); + SIZE_TDEF_HEADER = defineSizeTdefHeader(); + SIZE_TDEF_TRAILER = defineSizeTdefTrailer(); SIZE_COLUMN_DEF_BLOCK = defineSizeColumnDefBlock(); SIZE_INDEX_ENTRY_MASK = defineSizeIndexEntryMask(); @@ -240,7 +236,7 @@ public abstract class JetFormat { protected abstract int defineOffsetRowLocationBlock(); protected abstract int defineOffsetRowStart(); - protected abstract int defineOffsetMapStart(); + protected abstract int defineOffsetUsageMapStart(); protected abstract int defineOffsetUsageMapPageData(); @@ -249,9 +245,6 @@ public abstract class JetFormat { protected abstract int defineOffsetFreeSpace(); protected abstract int defineOffsetNumRowsOnDataPage(); - protected abstract int defineOffsetUsedPagesUsageMapDef(); - protected abstract int defineOffsetFreePagesUsageMapDef(); - protected abstract int defineOffsetIndexCompressedByteCount(); protected abstract int defineOffsetIndexEntryMask(); protected abstract int defineOffsetNextIndexLeafPage(); @@ -260,7 +253,8 @@ public abstract class JetFormat { protected abstract int defineSizeColumnHeader(); protected abstract int defineSizeRowLocation(); protected abstract int defineSizeLongValueDef(); - protected abstract int defineSizeTdefBlock(); + protected abstract int defineSizeTdefHeader(); + protected abstract int defineSizeTdefTrailer(); protected abstract int defineSizeColumnDefBlock(); protected abstract int defineSizeIndexEntryMask(); @@ -278,66 +272,109 @@ public abstract class JetFormat { private Jet4Format() { super("VERSION_4"); } - + + @Override protected int definePageSize() { return 4096; } + @Override protected int defineMaxRowSize() { return PAGE_SIZE - 16; } + @Override protected int defineOffsetNextTableDefPage() { return 4; } + @Override protected int defineOffsetNumRows() { return 16; } + @Override protected int defineOffsetTableType() { return 40; } + @Override protected int defineOffsetMaxCols() { return 41; } + @Override protected int defineOffsetNumVarCols() { return 43; } + @Override protected int defineOffsetNumCols() { return 45; } + @Override protected int defineOffsetNumIndexSlots() { return 47; } + @Override protected int defineOffsetNumIndexes() { return 51; } + @Override protected int defineOffsetOwnedPages() { return 55; } + @Override protected int defineOffsetFreeSpacePages() { return 59; } + @Override protected int defineOffsetIndexDefBlock() { return 63; } + @Override protected int defineOffsetIndexNumberBlock() { return 52; } + @Override protected int defineOffsetColumnType() { return 0; } + @Override protected int defineOffsetColumnNumber() { return 5; } + @Override protected int defineOffsetColumnPrecision() { return 11; } + @Override protected int defineOffsetColumnScale() { return 12; } + @Override protected int defineOffsetColumnVariable() { return 15; } + @Override protected int defineOffsetColumnCompressedUnicode() { return 16; } + @Override protected int defineOffsetColumnLength() { return 23; } + @Override protected int defineOffsetColumnVariableTableIndex() { return 7; } + @Override protected int defineOffsetColumnFixedDataOffset() { return 21; } + @Override protected int defineOffsetTableDefLocation() { return 4; } + @Override protected int defineOffsetNumRowsOnPage() { return 12; } + @Override protected int defineOffsetRowLocationBlock() { return 16; } + @Override protected int defineOffsetRowStart() { return 14; } - protected int defineOffsetMapStart() { return 5; } + @Override + protected int defineOffsetUsageMapStart() { return 5; } + @Override protected int defineOffsetUsageMapPageData() { return 4; } + @Override protected int defineOffsetReferenceMapPageNumbers() { return 1; } + @Override protected int defineOffsetFreeSpace() { return 2; } + @Override protected int defineOffsetNumRowsOnDataPage() { return 12; } - protected int defineOffsetUsedPagesUsageMapDef() { return 4027; } - protected int defineOffsetFreePagesUsageMapDef() { return 3958; } - + @Override protected int defineOffsetIndexCompressedByteCount() { return 24; } + @Override protected int defineOffsetIndexEntryMask() { return 27; } + @Override protected int defineOffsetNextIndexLeafPage() { return 16; } + @Override protected int defineSizeIndexDefinition() { return 12; } + @Override protected int defineSizeColumnHeader() { return 25; } + @Override protected int defineSizeRowLocation() { return 2; } + @Override protected int defineSizeLongValueDef() { return 12; } - protected int defineSizeTdefBlock() { return 63; } + @Override + protected int defineSizeTdefHeader() { return 63; } + @Override + protected int defineSizeTdefTrailer() { return 2; } + @Override protected int defineSizeColumnDefBlock() { return 25; } + @Override protected int defineSizeIndexEntryMask() { return 453; } + @Override protected int defineUsageMapTableByteLength() { return 64; } + @Override protected Charset defineCharset() { return Charset.forName("UTF-16LE"); } } diff --git a/src/java/com/healthmarketscience/jackcess/PageChannel.java b/src/java/com/healthmarketscience/jackcess/PageChannel.java index 9f394f0..dcd31a6 100644 --- a/src/java/com/healthmarketscience/jackcess/PageChannel.java +++ b/src/java/com/healthmarketscience/jackcess/PageChannel.java @@ -80,6 +80,15 @@ public class PageChannel implements Channel, Flushable { format, true); } } + + /** + * Only used by unit tests + */ + PageChannel(boolean testing) { + _channel = null; + _format = JetFormat.VERSION_4; + _autoSync = false; + } /** * @param buffer Buffer to read the page into @@ -163,11 +172,26 @@ public class PageChannel implements Channel, Flushable { * @return A newly-allocated buffer that can be passed to readPage */ public ByteBuffer createPageBuffer() { - ByteBuffer rtn = ByteBuffer.allocate(_format.PAGE_SIZE); - rtn.order(ByteOrder.LITTLE_ENDIAN); - return rtn; + return createBuffer(_format.PAGE_SIZE); } + /** + * @return A newly-allocated buffer of the given size and LITTLE_ENDIAN byte + * order + */ + public ByteBuffer createBuffer(int size) { + return createBuffer(size, ByteOrder.LITTLE_ENDIAN); + } + + /** + * @return A newly-allocated buffer of the given size and byte order + */ + public ByteBuffer createBuffer(int size, ByteOrder order) { + ByteBuffer rtn = ByteBuffer.allocate(size); + rtn.order(order); + return rtn; + } + public void flush() throws IOException { _channel.force(true); } diff --git a/src/java/com/healthmarketscience/jackcess/Table.java b/src/java/com/healthmarketscience/jackcess/Table.java index bc04545..968ad95 100644 --- a/src/java/com/healthmarketscience/jackcess/Table.java +++ b/src/java/com/healthmarketscience/jackcess/Table.java @@ -119,8 +119,8 @@ public class Table /** * Only used by unit tests */ - Table() throws IOException { - _pageChannel = new PageChannel(null, JetFormat.VERSION_4, true); + Table(boolean testing) throws IOException { + _pageChannel = new PageChannel(testing); } /** @@ -147,9 +147,8 @@ public class Table } _pageChannel.readPage(nextPageBuffer, nextPage); nextPage = nextPageBuffer.getInt(_format.OFFSET_NEXT_TABLE_DEF_PAGE); - ByteBuffer newBuffer = ByteBuffer.allocate(tableBuffer.capacity() + - format.PAGE_SIZE - 8); - newBuffer.order(ByteOrder.LITTLE_ENDIAN); + ByteBuffer newBuffer = _pageChannel.createBuffer( + tableBuffer.capacity() + format.PAGE_SIZE - 8); newBuffer.put(tableBuffer); newBuffer.put(nextPageBuffer.array(), 8, format.PAGE_SIZE - 8); tableBuffer = newBuffer; @@ -551,7 +550,272 @@ public class Table { return new RowIterator(columnNames); } + + /** + * Writes a new table defined by the given columns to the database. + * @return the first page of the new table's definition + */ + public static int writeTableDefinition( + List columns, PageChannel pageChannel, JetFormat format) + throws IOException + { + // first, create the usage map page + int usageMapPageNumber = pageChannel.writeNewPage( + createUsageMapDefinitionBuffer(pageChannel, format)); + + // next, determine how big the table def will be (in case it will be more + // than one page) + int totalTableDefSize = format.SIZE_TDEF_HEADER + + (format.SIZE_COLUMN_DEF_BLOCK * columns.size()) + + format.SIZE_TDEF_TRAILER; + for(Column col : columns) { + // we add the number of bytes for the column name and 2 bytes for the + // length of the column name + ByteBuffer cName = format.CHARSET.encode(col.getName()); + int nameByteLen = (col.getName().length() * + JetFormat.TEXT_FIELD_UNIT_SIZE); + totalTableDefSize += nameByteLen + 2; + } + + // now, create the table definition + ByteBuffer buffer = pageChannel.createBuffer(Math.max(totalTableDefSize, + format.PAGE_SIZE)); + writeTableDefinitionHeader(buffer, columns, usageMapPageNumber, + totalTableDefSize, format); + writeColumnDefinitions(buffer, columns, format); + + //End of tabledef + buffer.put((byte) 0xff); + buffer.put((byte) 0xff); + + // write table buffer to database + int tdefPageNumber = PageChannel.INVALID_PAGE_NUMBER; + if(totalTableDefSize <= format.PAGE_SIZE) { + + // easy case, fits on one page + buffer.putShort(format.OFFSET_FREE_SPACE, + (short)(buffer.remaining() - 8)); // overwrite page free space + // Write the tdef page to disk. + tdefPageNumber = pageChannel.writeNewPage(buffer); + + } else { + + // need to split across multiple pages + ByteBuffer partialTdef = pageChannel.createPageBuffer(); + buffer.rewind(); + int nextTdefPageNumber = PageChannel.INVALID_PAGE_NUMBER; + while(buffer.hasRemaining()) { + + // reset for next write + partialTdef.clear(); + + if(tdefPageNumber == PageChannel.INVALID_PAGE_NUMBER) { + + // this is the first page. note, the first page already has the + // page header, so no need to write it here + tdefPageNumber = pageChannel.allocateNewPage(); + nextTdefPageNumber = tdefPageNumber; + + } else { + + // write page header + writeTablePageHeader(partialTdef); + } + + // copy the next page of tdef bytes + int curTdefPageNumber = nextTdefPageNumber; + int writeLen = Math.min(partialTdef.remaining(), buffer.remaining()); + partialTdef.put(buffer.array(), buffer.position(), writeLen); + buffer.position(buffer.position() + writeLen); + + if(buffer.hasRemaining()) { + // need a next page + nextTdefPageNumber = pageChannel.allocateNewPage(); + partialTdef.putInt(format.OFFSET_NEXT_TABLE_DEF_PAGE, + nextTdefPageNumber); + } + + // update page free space + partialTdef.putShort(format.OFFSET_FREE_SPACE, + (short)(partialTdef.remaining() - 8)); // overwrite page free space + + // write partial page to disk + pageChannel.writePage(partialTdef, curTdefPageNumber); + } + + } + + return tdefPageNumber; + } + + /** + * @param buffer Buffer to write to + * @param columns List of Columns in the table + */ + private static void writeTableDefinitionHeader( + ByteBuffer buffer, List columns, + int usageMapPageNumber, int totalTableDefSize, JetFormat format) + throws IOException + { + //Start writing the tdef + writeTablePageHeader(buffer); + buffer.putInt(totalTableDefSize); //Length of table def + buffer.put((byte) 0x59); //Unknown + buffer.put((byte) 0x06); //Unknown + buffer.putShort((short) 0); //Unknown + buffer.putInt(0); //Number of rows + buffer.putInt(0); //Autonumber + for (int i = 0; i < 16; i++) { //Unknown + buffer.put((byte) 0); + } + buffer.put(Table.TYPE_USER); //Table type + buffer.putShort((short) columns.size()); //Max columns a row will have + buffer.putShort(Column.countVariableLength(columns)); //Number of variable columns in table + buffer.putShort((short) columns.size()); //Number of columns in table + buffer.putInt(0); //Number of indexes in table + buffer.putInt(0); //Number of indexes in table + buffer.put((byte) 0); //Usage map row number + ByteUtil.put3ByteInt(buffer, usageMapPageNumber); //Usage map page number + buffer.put((byte) 1); //Free map row number + ByteUtil.put3ByteInt(buffer, usageMapPageNumber); //Free map page number + if (LOG.isDebugEnabled()) { + int position = buffer.position(); + buffer.rewind(); + LOG.debug("Creating new table def block:\n" + ByteUtil.toHexString( + buffer, format.SIZE_TDEF_HEADER)); + buffer.position(position); + } + } + + /** + * Writes the page header for a table definition page + * @param buffer Buffer to write to + */ + private static void writeTablePageHeader(ByteBuffer buffer) + { + buffer.put(PageTypes.TABLE_DEF); //Page type + buffer.put((byte) 0x01); //Unknown + buffer.put((byte) 0); //Unknown + buffer.put((byte) 0); //Unknown + buffer.putInt(0); //Next TDEF page pointer + } + /** + * @param buffer Buffer to write to + * @param columns List of Columns to write definitions for + */ + private static void writeColumnDefinitions( + ByteBuffer buffer, List columns, JetFormat format) + throws IOException + { + short columnNumber = (short) 0; + short fixedOffset = (short) 0; + short variableOffset = (short) 0; + // we specifically put the "long variable" values after the normal + // variable length values so that we have a better chance of fitting it + // all (because "long variable" values can go in separate pages) + short longVariableOffset = + (short) Column.countNonLongVariableLength(columns); + for (Column col : columns) { + int position = buffer.position(); + buffer.put(col.getType().getValue()); + buffer.put((byte) 0x59); //Unknown + buffer.put((byte) 0x06); //Unknown + buffer.putShort((short) 0); //Unknown + buffer.putShort(columnNumber); //Column Number + if (col.isVariableLength()) { + if(!col.getType().isLongValue()) { + buffer.putShort(variableOffset++); + } else { + buffer.putShort(longVariableOffset++); + } + } else { + buffer.putShort((short) 0); + } + buffer.putShort(columnNumber); //Column Number again + if(col.getType().getHasScalePrecision()) { + buffer.put((byte) col.getPrecision()); // numeric precision + buffer.put((byte) col.getScale()); // numeric scale + } else { + buffer.put((byte) 0x00); //unused + buffer.put((byte) 0x00); //unused + } + buffer.putShort((short) 0); //Unknown + if (col.isVariableLength()) { //Variable length + buffer.put((byte) 0x2); + } else { + buffer.put((byte) 0x3); + } + if (col.isCompressedUnicode()) { //Compressed + buffer.put((byte) 1); + } else { + buffer.put((byte) 0); + } + buffer.putInt(0); //Unknown, but always 0. + //Offset for fixed length columns + if (col.isVariableLength()) { + buffer.putShort((short) 0); + } else { + buffer.putShort(fixedOffset); + fixedOffset += col.getType().getFixedSize(); + } + if(!col.getType().isLongValue()) { + buffer.putShort(col.getLength()); //Column length + } else { + buffer.putShort((short)0x0000); // unused + } + columnNumber++; + if (LOG.isDebugEnabled()) { + LOG.debug("Creating new column def block\n" + ByteUtil.toHexString( + buffer, position, format.SIZE_COLUMN_DEF_BLOCK)); + } + } + for (Column col : columns) { + ByteBuffer colName = format.CHARSET.encode(col.getName()); + buffer.putShort((short) colName.remaining()); + buffer.put(colName); + } + } + + /** + * Create the usage map definition page buffer. The "used pages" map is in + * row 0, the "pages with free space" map is in row 1. + */ + private static ByteBuffer createUsageMapDefinitionBuffer( + PageChannel pageChannel, JetFormat format) + throws IOException + { + // USAGE_MAP_DEF_FREE_SPACE = 3940; + int usageMapRowLength = format.OFFSET_USAGE_MAP_START + + format.USAGE_MAP_TABLE_BYTE_LENGTH; + int freeSpace = getRowSpaceUsage(format.MAX_ROW_SIZE, format) + - (2 * getRowSpaceUsage(usageMapRowLength, format)); + + ByteBuffer rtn = pageChannel.createPageBuffer(); + rtn.put(PageTypes.DATA); + rtn.put((byte) 0x1); //Unknown + rtn.putShort((short)freeSpace); //Free space in page + rtn.putInt(0); //Table definition + rtn.putInt(0); //Unknown + rtn.putShort((short) 2); //Number of records on this page + + // write two rows of usage map definitions + int rowStart = findRowEnd(rtn, 0, format) - usageMapRowLength; + for(int i = 0; i < 2; ++i) { + rtn.putShort(getRowStartOffset(i, format), (short)rowStart); + if(i == 0) { + // initial "usage pages" map definition + rtn.put(rowStart, (byte)UsageMap.MAP_TYPE_REFERENCE); + } else { + // initial "pages with free space" map definition + rtn.put(rowStart, (byte)UsageMap.MAP_TYPE_INLINE); + } + rowStart -= usageMapRowLength; + } + + return rtn; + } + /** * Read the table definition */ @@ -560,7 +824,7 @@ public class Table if (LOG.isDebugEnabled()) { tableBuffer.rewind(); LOG.debug("Table def block:\n" + ByteUtil.toHexString(tableBuffer, - _format.SIZE_TDEF_BLOCK)); + _format.SIZE_TDEF_HEADER)); } _rowCount = tableBuffer.getInt(_format.OFFSET_NUM_ROWS); _tableType = tableBuffer.get(_format.OFFSET_TABLE_TYPE); diff --git a/src/java/com/healthmarketscience/jackcess/UsageMap.java b/src/java/com/healthmarketscience/jackcess/UsageMap.java index 1ba9a6f..bf19fb4 100644 --- a/src/java/com/healthmarketscience/jackcess/UsageMap.java +++ b/src/java/com/healthmarketscience/jackcess/UsageMap.java @@ -90,7 +90,7 @@ public class UsageMap _tablePageNum = pageNum; _format = format; _rowStart = rowStart; - _tableBuffer.position((int) _rowStart + format.OFFSET_MAP_START); + _tableBuffer.position((int) _rowStart + format.OFFSET_USAGE_MAP_START); _startOffset = _tableBuffer.position(); if (LOG.isDebugEnabled()) { LOG.debug("Usage map block:\n" + ByteUtil.toHexString(_tableBuffer, _rowStart, @@ -242,7 +242,7 @@ public class UsageMap ++_modCount; // clear out the table data - int tableStart = getRowStart() + getFormat().OFFSET_MAP_START - 4; + int tableStart = getRowStart() + getFormat().OFFSET_USAGE_MAP_START - 4; int tableEnd = tableStart + getFormat().USAGE_MAP_TABLE_BYTE_LENGTH + 4; ByteUtil.clearRange(_tableBuffer, tableStart, tableEnd); } @@ -441,7 +441,9 @@ public class UsageMap if(add) { // we can ignore out-of-range page addition if we are already - // assuming out-of-range bits are "on" + // assuming out-of-range bits are "on". Note, we are leaving small + // holes in the database here (leaving behind some free pages), but + // it's not the end of the world. if(!_assumeOutOfRangeBitsOn) { // we are adding, can we shift the bits and stay inline? @@ -556,7 +558,7 @@ public class UsageMap if(firstPage == PageChannel.INVALID_PAGE_NUMBER) { // this is the common case where we left everything behind - int tableStart = getRowStart() + getFormat().OFFSET_MAP_START; + int tableStart = getRowStart() + getFormat().OFFSET_USAGE_MAP_START; int tableEnd = tableStart + getFormat().USAGE_MAP_TABLE_BYTE_LENGTH; ByteUtil.fillRange(_tableBuffer, tableStart, tableEnd); diff --git a/test/src/java/com/healthmarketscience/jackcess/DatabaseTest.java b/test/src/java/com/healthmarketscience/jackcess/DatabaseTest.java index 5083577..c46ee00 100644 --- a/test/src/java/com/healthmarketscience/jackcess/DatabaseTest.java +++ b/test/src/java/com/healthmarketscience/jackcess/DatabaseTest.java @@ -663,6 +663,42 @@ public class DatabaseTest extends TestCase { db.close(); } + + + public void testLargeTableDef() throws Exception { + final int numColumns = 90; + Database db = create(); + + List columns = new ArrayList(); + List colNames = new ArrayList(); + for(int i = 0; i < numColumns; ++i) { + String colName = "MyColumnName" + i; + colNames.add(colName); + Column col = new Column(); + col.setName(colName); + col.setType(DataType.TEXT); + columns.add(col); + } + + db.createTable("test", columns); + + Table t = db.getTable("test"); + + List row = new ArrayList(); + Map expectedRowData = new HashMap(); + for(int i = 0; i < numColumns; ++i) { + String value = "" + i + " some row data"; + row.add(value); + expectedRowData.put(colNames.get(i), value); + } + + t.addRow(row.toArray()); + + t.reset(); + assertEquals(expectedRowData, t.getNextRow()); + + db.close(); + } static Object[] createTestRow(String col1Val) { return new Object[] {col1Val, "R", "McCune", 1234, (byte) 0xad, 555.66d, @@ -713,7 +749,7 @@ public class DatabaseTest extends TestCase { columns.add(col); db.createTable("test", columns); } - + static String createString(int len) { StringBuilder builder = new StringBuilder(len); for(int i = 0; i < len; ++i) { diff --git a/test/src/java/com/healthmarketscience/jackcess/TableTest.java b/test/src/java/com/healthmarketscience/jackcess/TableTest.java index 96d33fa..041600a 100644 --- a/test/src/java/com/healthmarketscience/jackcess/TableTest.java +++ b/test/src/java/com/healthmarketscience/jackcess/TableTest.java @@ -19,15 +19,15 @@ public class TableTest extends TestCase { public void testCreateRow() throws Exception { JetFormat format = JetFormat.VERSION_4; - Table table = new Table(); + Table table = new Table(true); List columns = new ArrayList(); - Column col = new Column(); + Column col = new Column(true); col.setType(DataType.INT); columns.add(col); - col = new Column(); + col = new Column(true); col.setType(DataType.TEXT); columns.add(col); - col = new Column(); + col = new Column(true); col.setType(DataType.TEXT); columns.add(col); table.setColumns(columns);