From: James Ahlborn Date: Mon, 24 Sep 2012 01:42:23 +0000 (+0000) Subject: Added the MemFileChannel to enable working with dbs completely in memory X-Git-Tag: jackcess-1.2.9~8 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=74b23acf24e0b16f3d57654680930e55ddd697a0;p=jackcess.git Added the MemFileChannel to enable working with dbs completely in memory git-svn-id: https://svn.code.sf.net/p/jackcess/code/jackcess/trunk@642 f203690c-595d-4dc9-a70b-905162fa7fd2 --- diff --git a/src/changes/changes.xml b/src/changes/changes.xml index afd3161..acccf8b 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -13,6 +13,10 @@ Added DatabaseBuilder in for more convenient and flexible Database open/create. + + Added the MemFileChannel (and associated support in DatabaseBuilder) + to enable working with Database files completely in memory. + diff --git a/src/java/com/healthmarketscience/jackcess/Database.java b/src/java/com/healthmarketscience/jackcess/Database.java index 1c6f347..5f3505a 100644 --- a/src/java/com/healthmarketscience/jackcess/Database.java +++ b/src/java/com/healthmarketscience/jackcess/Database.java @@ -279,6 +279,11 @@ public class Database static final int SYSTEM_OBJECT_FLAGS = SYSTEM_OBJECT_FLAG | ALT_SYSTEM_OBJECT_FLAG; + /** read-only channel access mode */ + static final String RO_CHANNEL_MODE = "r"; + /** read/write channel access mode */ + static final String RW_CHANNEL_MODE = "rw"; + /** * Enum which indicates which version of Access created the database. * @usage _general_class_ @@ -857,7 +862,7 @@ public class Database static FileChannel openChannel(final File mdbFile, final boolean readOnly) throws FileNotFoundException { - final String mode = (readOnly ? "r" : "rw"); + final String mode = (readOnly ? RO_CHANNEL_MODE : RW_CHANNEL_MODE); return new RandomAccessFile(mdbFile, mode).getChannel(); } diff --git a/src/java/com/healthmarketscience/jackcess/MemFileChannel.java b/src/java/com/healthmarketscience/jackcess/MemFileChannel.java new file mode 100644 index 0000000..221cb67 --- /dev/null +++ b/src/java/com/healthmarketscience/jackcess/MemFileChannel.java @@ -0,0 +1,477 @@ +/* +Copyright (c) 2012 James Ahlborn + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 +USA +*/ + +package com.healthmarketscience.jackcess; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.NonWritableChannelException; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; + +/** + * FileChannel implementation which maintains the entire "file" in memory. + * This enables working with a Database entirely in memory (for situations + * where disk usage may not be possible or desirable). Obviously, this + * requires enough jvm heap space to fit the file data. Use one of the + * {@code newChannel()} methods to construct an instance of this class. + *

+ * In order to use this class with a Database, you must use the {@link + * DatabaseBuilder} to open/create the Database instance, passing an instance + * of this class to the {@link DatabaseBuilder#setChannel} method. + *

+ * Implementation note: this class is optimized for use with {@link Database}. + * Therefore not all methods may be implemented and individual read/write + * operations are only supported within page boundaries. + * + * @author James Ahlborn + */ +public class MemFileChannel extends FileChannel +{ + private static final byte[][] EMPTY_DATA = new byte[0][]; + + // use largest possible Jet "page size" to ensure that reads/writes will + // always be within a single chunk + private static final int CHUNK_SIZE = 4096; + // this ensures that an "empty" mdb will fit in the initial chunk table + private static final int INIT_CHUNKS = 128; + + /** current read/write position */ + private long _position; + /** current amount of actual data in the file */ + private long _size; + /** chunks containing the file data. the length of the chunk array is + always a power of 2 and the chunks are always CHUNK_SIZE. */ + private byte[][] _data; + + private MemFileChannel() + { + this(0L, 0L, EMPTY_DATA); + } + + private MemFileChannel(long position, long size, byte[][] data) { + _position = position; + _size = size; + _data = data; + } + + /** + * Creates a new read/write, empty MemFileChannel. + */ + public static MemFileChannel newChannel() { + return new MemFileChannel(); + } + + /** + * Creates a new read/write MemFileChannel containing the contents of the + * given File. Note, modifications to the returned channel will not + * affect the original File source. + */ + public static MemFileChannel newChannel(File file) throws IOException { + return newChannel(file, Database.RW_CHANNEL_MODE); + } + + /** + * Creates a new MemFileChannel containing the contents of the + * given File with the given mode (for mode details see + * {@link RandomAccessFile#RandomAccessFile(File,String)}). Note, + * modifications to the returned channel will not affect the original + * File source. + */ + public static MemFileChannel newChannel(File file, String mode) + throws IOException + { + FileChannel in = null; + try { + return newChannel(in = new RandomAccessFile( + file, Database.RO_CHANNEL_MODE).getChannel(), + mode); + } finally { + if(in != null) { + try { + in.close(); + } catch(IOException e) { + // ignore close failure + } + } + } + } + + /** + * Creates a new read/write MemFileChannel containing the contents of the + * given InputStream. + */ + public static MemFileChannel newChannel(InputStream in) throws IOException { + return newChannel(in, Database.RW_CHANNEL_MODE); + } + + /** + * Creates a new MemFileChannel containing the contents of the + * given InputStream with the given mode (for mode details see + * {@link RandomAccessFile#RandomAccessFile(File,String)}). + */ + public static MemFileChannel newChannel(InputStream in, String mode) + throws IOException + { + return newChannel(Channels.newChannel(in), mode); + } + + /** + * Creates a new read/write MemFileChannel containing the contents of the + * given ReadableByteChannel. + */ + public static MemFileChannel newChannel(ReadableByteChannel in) + throws IOException + { + return newChannel(in, Database.RW_CHANNEL_MODE); + } + + /** + * Creates a new MemFileChannel containing the contents of the + * given ReadableByteChannel with the given mode (for mode details see + * {@link RandomAccessFile#RandomAccessFile(File,String)}). + */ + public static MemFileChannel newChannel(ReadableByteChannel in, String mode) + throws IOException + { + MemFileChannel channel = new MemFileChannel(); + channel.transferFrom(in, 0L, Long.MAX_VALUE); + if(!mode.contains("w")) { + channel = new ReadOnlyChannel(channel); + } + return channel; + } + + @Override + public int read(ByteBuffer dst) throws IOException { + int bytesRead = read(dst, _position); + if(bytesRead > 0) { + _position += bytesRead; + } + return bytesRead; + } + + @Override + public int read(ByteBuffer dst, long position) throws IOException { + if(position >= _size) { + return -1; + } + + // we assume reads will always be within a single chunk (due to how mdb + // files work) + byte[] chunk = _data[getChunkIndex(position)]; + int chunkOffset = getChunkOffset(position); + int numBytes = dst.remaining(); + dst.put(chunk, chunkOffset, numBytes); + + return numBytes; + } + + @Override + public int write(ByteBuffer src) throws IOException { + int bytesWritten = write(src, _position); + _position += bytesWritten; + return bytesWritten; + } + + @Override + public int write(ByteBuffer src, long position) throws IOException { + int numBytes = src.remaining(); + long newSize = position + numBytes; + ensureCapacity(newSize); + + // we assume writes will always be within a single chunk (due to how mdb + // files work) + byte[] chunk = _data[getChunkIndex(position)]; + int chunkOffset = getChunkOffset(position); + src.get(chunk, chunkOffset, numBytes); + if(newSize > _size) { + _size = newSize; + } + + return numBytes; + } + + @Override + public long position() throws IOException { + return _position; + } + + @Override + public FileChannel position(long newPosition) throws IOException { + if(newPosition < 0L) { + throw new IllegalArgumentException("negative position"); + } + _position = newPosition; + return this; + } + + @Override + public long size() throws IOException { + return _size; + } + + @Override + public FileChannel truncate(long newSize) throws IOException { + if(newSize < 0L) { + throw new IllegalArgumentException("negative size"); + } + if(newSize < _size) { + // we'll optimize for memory over speed and aggressively free unused + // chunks + for(int i = getNumChunks(newSize); i < getNumChunks(_size); ++i) { + _data[i] = null; + } + _size = newSize; + } + _position = Math.min(newSize, _position); + return this; + } + + @Override + public void force(boolean metaData) throws IOException { + // nothing to do + } + + /** + * Convenience method for writing the entire contents of this channel to the + * given destination channel. + * @see #transferTo(long,long,WritableByteChannel) + */ + public long transferTo(WritableByteChannel dst) + throws IOException + { + return transferTo(0L, _size, dst); + } + + @Override + public long transferTo(long position, long count, WritableByteChannel dst) + throws IOException + { + if(position >= _size) { + return 0L; + } + + count = Math.min(count, _size - position); + + int chunkIndex = getChunkIndex(position); + int chunkOffset = getChunkOffset(position); + + long numBytes = 0; + while(count > 0L) { + + int chunkBytes = (int)Math.min(count, CHUNK_SIZE - chunkOffset); + ByteBuffer src = ByteBuffer.wrap(_data[chunkIndex], chunkOffset, + chunkBytes); + + do { + int bytesWritten = dst.write(src); + if(bytesWritten == 0L) { + // dst full + return numBytes; + } + numBytes += bytesWritten; + count -= bytesWritten; + } while(src.hasRemaining()); + + ++chunkIndex; + chunkOffset = 0; + } + + return numBytes; + } + + /** + * Convenience method for writing the entire contents of this channel to the + * given destination stream. + * @see #transferTo(long,long,WritableByteChannel) + */ + public long transferTo(OutputStream dst) + throws IOException + { + return transferTo(0L, _size, dst); + } + + /** + * Convenience method for writing the selected portion of this channel to + * the given destination stream. + * @see #transferTo(long,long,WritableByteChannel) + */ + public long transferTo(long position, long count, WritableByteChannel dst) + throws IOException + { + return transferTo(position, count, Channels.newChannel(dst)); + } + + @Override + public long transferFrom(ReadableByteChannel src, + long position, long count) + throws IOException + { + int chunkIndex = getChunkIndex(position); + int chunkOffset = getChunkOffset(position); + + long numBytes = 0L; + while(count > 0L) { + + ensureCapacity(position + numBytes + 1); + + int chunkBytes = (int)Math.min(count, CHUNK_SIZE - chunkOffset); + ByteBuffer dst = ByteBuffer.wrap(_data[chunkIndex], chunkOffset, + chunkBytes); + do { + int bytesRead = src.read(dst); + if(bytesRead <= 0) { + // src empty + return numBytes; + } + numBytes += bytesRead; + count -= bytesRead; + _size = Math.max(_size, position + numBytes); + } while(dst.hasRemaining()); + + ++chunkIndex; + chunkOffset = 0; + } + + return numBytes; + } + + @Override + protected void implCloseChannel() throws IOException { + // release data + _data = EMPTY_DATA; + _size = _position = 0L; + } + + private void ensureCapacity(long newSize) + { + if(newSize <= _size) { + // nothing to do + return; + } + + int newNumChunks = getNumChunks(newSize); + int numChunks = getNumChunks(_size); + + if(newNumChunks > _data.length) { + + // need to extend chunk array (use powers of 2) + int newDataLen = Math.max(_data.length, INIT_CHUNKS); + while(newDataLen < newNumChunks) { + newDataLen <<= 1; + } + + byte[][] newData = new byte[newDataLen][]; + + // copy existing chunks + System.arraycopy(_data, 0, newData, 0, numChunks); + + _data = newData; + } + + // allocate new chunks + for(int i = numChunks; i < newNumChunks; ++i) { + _data[i] = new byte[CHUNK_SIZE]; + } + } + + private static int getChunkIndex(long pos) { + return (int)(pos / CHUNK_SIZE); + } + + private static int getChunkOffset(long pos) { + return (int)(pos % CHUNK_SIZE); + } + + private static int getNumChunks(long size) { + return getChunkIndex(size + CHUNK_SIZE - 1); + } + + @Override + public long write(ByteBuffer[] srcs, int offset, int length) + throws IOException + { + throw new UnsupportedOperationException(); + } + + @Override + public long read(ByteBuffer[] dsts, int offset, int length) + throws IOException + { + throw new UnsupportedOperationException(); + } + + @Override + public MappedByteBuffer map(MapMode mode, long position, long size) + throws IOException + { + throw new UnsupportedOperationException(); + } + + @Override + public FileLock lock(long position, long size, boolean shared) + throws IOException + { + throw new UnsupportedOperationException(); + } + + @Override + public FileLock tryLock(long position, long size, boolean shared) + throws IOException + { + throw new UnsupportedOperationException(); + } + + /** + * Subclass of MemFileChannel which is read-only. + */ + private static final class ReadOnlyChannel extends MemFileChannel + { + private ReadOnlyChannel(MemFileChannel channel) + { + super(channel._position, channel._size, channel._data); + } + + @Override + public int write(ByteBuffer src, long position) throws IOException { + throw new NonWritableChannelException(); + } + + @Override + public FileChannel truncate(long newSize) throws IOException { + throw new NonWritableChannelException(); + } + + @Override + public long transferFrom(ReadableByteChannel src, + long position, long count) + throws IOException + { + throw new NonWritableChannelException(); + } + } +} diff --git a/test/src/java/com/healthmarketscience/jackcess/DatabaseTest.java b/test/src/java/com/healthmarketscience/jackcess/DatabaseTest.java index 51c7efd..1c0c2eb 100644 --- a/test/src/java/com/healthmarketscience/jackcess/DatabaseTest.java +++ b/test/src/java/com/healthmarketscience/jackcess/DatabaseTest.java @@ -37,6 +37,7 @@ import java.io.OutputStream; import java.io.PrintWriter; import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.sql.Types; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -78,8 +79,17 @@ public class DatabaseTest extends TestCase { public static Database open(FileFormat fileFormat, File file) throws Exception { + return open(fileFormat, file, false); + } + + private static Database open(FileFormat fileFormat, File file, + boolean inMem) + throws Exception + { + FileChannel channel = (inMem ? MemFileChannel.newChannel(file, "r") + : null); final Database db = new DatabaseBuilder(file).setReadOnly(true) - .setAutoSync(_autoSync).open(); + .setAutoSync(_autoSync).setChannel(channel).open(); assertEquals("Wrong JetFormat.", fileFormat.getJetFormat(), db.getFormat()); assertEquals("Wrong FileFormat.", fileFormat, db.getFileFormat()); @@ -90,6 +100,10 @@ public class DatabaseTest extends TestCase { return open(testDB.getExpectedFileFormat(), testDB.getFile()); } + public static Database openMem(TestDB testDB) throws Exception { + return open(testDB.getExpectedFileFormat(), testDB.getFile(), true); + } + public static Database create(FileFormat fileFormat) throws Exception { return create(fileFormat, false); } @@ -97,8 +111,20 @@ public class DatabaseTest extends TestCase { public static Database create(FileFormat fileFormat, boolean keep) throws Exception { + return create(fileFormat, keep, false); + } + + public static Database createMem(FileFormat fileFormat) throws Exception { + return create(fileFormat, false, true); + } + + private static Database create(FileFormat fileFormat, boolean keep, + boolean inMem) + throws Exception + { + FileChannel channel = (inMem ? MemFileChannel.newChannel() : null); return new DatabaseBuilder(createTempFile(keep)).setFileFormat(fileFormat) - .setAutoSync(_autoSync).create(); + .setAutoSync(_autoSync).setChannel(channel).create(); } @@ -301,6 +327,20 @@ public class DatabaseTest extends TestCase { public void testWriteAndRead() throws Exception { for (final FileFormat fileFormat : SUPPORTED_FILEFORMATS) { Database db = create(fileFormat); + doTestWriteAndRead(db); + db.close(); + } + } + + public void testWriteAndReadInMem() throws Exception { + for (final FileFormat fileFormat : SUPPORTED_FILEFORMATS) { + Database db = createMem(fileFormat); + doTestWriteAndRead(db); + db.close(); + } + } + + private static void doTestWriteAndRead(Database db) throws Exception { createTestTable(db); Object[] row = createTestRow(); row[3] = null; @@ -320,11 +360,8 @@ public class DatabaseTest extends TestCase { assertEquals(row[6], readRow.get("G")); assertEquals(row[7], readRow.get("H")); } - - db.close(); - } } - + public void testWriteAndReadInBatch() throws Exception { for (final FileFormat fileFormat : SUPPORTED_FILEFORMATS) { Database db = create(fileFormat); diff --git a/test/src/java/com/healthmarketscience/jackcess/IndexCodesTest.java b/test/src/java/com/healthmarketscience/jackcess/IndexCodesTest.java index a479038..ed71ebe 100644 --- a/test/src/java/com/healthmarketscience/jackcess/IndexCodesTest.java +++ b/test/src/java/com/healthmarketscience/jackcess/IndexCodesTest.java @@ -68,7 +68,7 @@ public class IndexCodesTest extends TestCase { public void testIndexCodes() throws Exception { for (final TestDB testDB : TestDB.getSupportedForBasename(Basename.INDEX_CODES)) { - Database db = open(testDB); + Database db = openMem(testDB); for(Table t : db) { for(Index index : t.getIndexes()) { diff --git a/test/src/java/com/healthmarketscience/jackcess/MemFileChannelTest.java b/test/src/java/com/healthmarketscience/jackcess/MemFileChannelTest.java new file mode 100644 index 0000000..6fa0425 --- /dev/null +++ b/test/src/java/com/healthmarketscience/jackcess/MemFileChannelTest.java @@ -0,0 +1,147 @@ +/* +Copyright (c) 2012 James Ahlborn + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 +USA +*/ + +package com.healthmarketscience.jackcess; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.NonWritableChannelException; +import java.util.Arrays; + +import junit.framework.TestCase; + +/** + * + * @author James Ahlborn + */ +public class MemFileChannelTest extends TestCase +{ + + public MemFileChannelTest(String name) { + super(name); + } + + public void testReadOnlyChannel() throws Exception + { + File testFile = new File("test/data/V1997/compIndexTestV1997.mdb"); + MemFileChannel ch = MemFileChannel.newChannel(testFile, "r"); + assertEquals(testFile.length(), ch.size()); + assertEquals(0L, ch.position()); + + try { + ByteBuffer bb = ByteBuffer.allocate(1024); + ch.write(bb); + fail("NonWritableChannelException should have been thrown"); + } catch(NonWritableChannelException ignored) { + // success + } + + try { + ch.truncate(0L); + fail("NonWritableChannelException should have been thrown"); + } catch(NonWritableChannelException ignored) { + // success + } + + try { + ch.transferFrom(null, 0L, 10L); + fail("NonWritableChannelException should have been thrown"); + } catch(NonWritableChannelException ignored) { + // success + } + + assertEquals(testFile.length(), ch.size()); + assertEquals(0L, ch.position()); + + ch.close(); + } + + public void testChannel() throws Exception + { + ByteBuffer bb = ByteBuffer.allocate(1024); + + MemFileChannel ch = MemFileChannel.newChannel(); + assertTrue(ch.isOpen()); + assertEquals(0L, ch.size()); + assertEquals(0L, ch.position()); + assertEquals(-1, ch.read(bb)); + + ch.close(); + + assertFalse(ch.isOpen()); + + File testFile = new File("test/data/V1997/compIndexTestV1997.mdb"); + ch = MemFileChannel.newChannel(testFile, "r"); + assertEquals(testFile.length(), ch.size()); + assertEquals(0L, ch.position()); + + MemFileChannel ch2 = MemFileChannel.newChannel(); + ch.transferTo(ch2); + ch2.force(true); + assertEquals(testFile.length(), ch2.size()); + assertEquals(testFile.length(), ch2.position()); + + long trucSize = ch2.size()/3; + ch2.truncate(trucSize); + assertEquals(trucSize, ch2.size()); + assertEquals(trucSize, ch2.position()); + ch2.position(0L); + copy(ch, ch2, bb); + + File tmpFile = File.createTempFile("chtest_", ".dat"); + tmpFile.deleteOnExit(); + FileChannel fc = new RandomAccessFile(tmpFile, "rw").getChannel(); + + copy(ch2, fc, bb); + + fc.close(); + + assertEquals(testFile.length(), tmpFile.length()); + + assertTrue(Arrays.equals(DatabaseTest.toByteArray(testFile), + DatabaseTest.toByteArray(tmpFile))); + + ch2.truncate(0L); + assertTrue(ch2.isOpen()); + assertEquals(0L, ch2.size()); + assertEquals(0L, ch2.position()); + assertEquals(-1, ch2.read(bb)); + + ch2.close(); + assertFalse(ch2.isOpen()); + } + + private static void copy(FileChannel src, FileChannel dst, ByteBuffer bb) + throws IOException + { + src.position(0L); + while(true) { + bb.clear(); + if(src.read(bb) < 0) { + break; + } + bb.flip(); + dst.write(bb); + } + } + +}