initial support for encoded databases, MSISAM support (issue #3065010)

git-svn-id: https://svn.code.sf.net/p/jackcess/code/jackcess/trunk@488 f203690c-595d-4dc9-a70b-905162fa7fd2

src/java/com/healthmarketscience/jackcess/CodecHandler.java

@@ -0,0 +1,69 @@
+/*
+Copyright (c) 2005 Health Market Science, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
+USA
+
+You can contact Health Market Science at info@healthmarketscience.com
+or at the following address:
+
+Health Market Science
+2700 Horizon Drive
+Suite 200
+King of Prussia, PA 19406
+*/
+
+package com.healthmarketscience.jackcess;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Interface for a handler which can encode/decode a specific access page
+ * encoding.
+ *
+ * @author James Ahlborn
+ */
+public interface CodecHandler 
+{
+
+  /**
+   * Decodes the given page buffer inline.
+   *
+   * @param page the page to be decoded
+   * @param pageNumber the page number of the given page
+   * 
+   * @throws IOException if an exception occurs during decoding
+   */
+  public void decodePage(ByteBuffer page, int pageNumber) throws IOException;
+
+  /**
+   * Encodes the given page buffer into a new page buffer and returns it.  The
+   * returned page buffer will be used immediately and discarded so that it
+   * may be re-used for subsequent page encodings.
+   *
+   * @param page the page to be encoded, should not be modified
+   * @param pageNumber the page number of the given page
+   * @param pageOffset offset within the page at which to start writing the
+   *                   page data
+   * 
+   * @throws IOException  if an exception occurs during decoding
+   *
+   * @return the properly encoded page buffer for the given page buffer 
+   */
+  public ByteBuffer encodePage(ByteBuffer page, int pageNumber, 
+                               int pageOffset) 
+    throws IOException;
+}

src/java/com/healthmarketscience/jackcess/CodecProvider.java

@@ -0,0 +1,56 @@
+/*
+Copyright (c) 2005 Health Market Science, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
+USA
+
+You can contact Health Market Science at info@healthmarketscience.com
+or at the following address:
+
+Health Market Science
+2700 Horizon Drive
+Suite 200
+King of Prussia, PA 19406
+*/
+
+package com.healthmarketscience.jackcess;
+
+import java.io.IOException;
+
+/**
+ * Interface for a provider which can generate CodecHandlers for various types
+ * of database encodings.  The {@link DefaultCodecProvider} is the default
+ * implementation of this inferface, but it does not have any actual
+ * encoding/decoding support (due to possible export issues with calling
+ * encryption APIs).  See the separate
+ * <a href="https://sourceforge.net/projects/jackcessencrypt/">Jackcess
+ * Encrypt</a> project for an implementation of this interface which supports
+ * various access database encryption types.
+ *
+ * @author James Ahlborn
+ */
+public interface CodecProvider 
+{
+  /**
+   * Returns a new CodecHandler for the database associated with the given
+   * PageChannel.
+   * 
+   * @param channel the PageChannel for a Database
+   * 
+   * @return a new CodecHandler, may not be {@code null}
+   */
+  public CodecHandler createHandler(PageChannel channel)
+    throws IOException;
+}

src/java/com/healthmarketscience/jackcess/Database.java

@@ -397,6 +397,33 @@ public class Database
                               Charset charset, TimeZone timeZone)
     throws IOException
   {    
+    return open(mdbFile, readOnly, autoSync, charset, timeZone, null);
+  }
+
+  /**
+   * Open an existing Database.  If the existing file is not writeable or the
+   * readOnly flag is <code>true</code>, the file will be opened read-only.
+   * @param mdbFile File containing the database
+   * @param readOnly iff <code>true</code>, force opening file in read-only
+   *                 mode
+   * @param autoSync whether or not to enable auto-syncing on write.  if
+   *                 {@code true}, writes will be immediately flushed to disk.
+   *                 This leaves the database in a (fairly) consistent state
+   *                 on each write, but can be very inefficient for many
+   *                 updates.  if {@code false}, flushing to disk happens at
+   *                 the jvm's leisure, which can be much faster, but may
+   *                 leave the database in an inconsistent state if failures
+   *                 are encountered during writing.
+   * @param charset Charset to use, if {@code null}, uses default
+   * @param timeZone TimeZone to use, if {@code null}, uses default
+   * @param provider CodecProvider for handling page encoding/decoding, may be
+   *                 {@code null} if no special encoding is necessary
+   */
+  public static Database open(File mdbFile, boolean readOnly, boolean autoSync,
+                              Charset charset, TimeZone timeZone, 
+                              CodecProvider provider)
+    throws IOException
+  {
     if(!mdbFile.exists() || !mdbFile.canRead()) {
       throw new FileNotFoundException("given file does not exist: " + mdbFile);
     }
@@ -423,7 +450,7 @@ public class Database
       }
     }
 
-    return new Database(channel, autoSync, null, charset, timeZone);
+    return new Database(channel, autoSync, null, charset, timeZone, provider);
   }
   
   /**
@@ -533,7 +560,8 @@ public class Database
     channel.transferFrom(Channels.newChannel(
         Thread.currentThread().getContextClassLoader().getResourceAsStream(
             fileFormat._emptyFile)), 0, Integer.MAX_VALUE);
-    return new Database(channel, autoSync, fileFormat, charset, timeZone);
+    return new Database(channel, autoSync, fileFormat, charset, timeZone,
+                        null);
   }
 
   /**
@@ -574,7 +602,8 @@ public class Database
    * @param timeZone TimeZone to use, if {@code null}, uses default
    */
   protected Database(FileChannel channel, boolean autoSync,
-                     FileFormat fileFormat, Charset charset, TimeZone timeZone)
+                     FileFormat fileFormat, Charset charset, TimeZone timeZone,
+                     CodecProvider provider)
     throws IOException
   {
     boolean success = false;
@@ -585,10 +614,13 @@ public class Database
       _fileFormat = fileFormat;
       _pageChannel = new PageChannel(channel, _format, autoSync);
       _timeZone = ((timeZone == null) ? getDefaultTimeZone() : timeZone);
+      if(provider == null) {
+        provider = DefaultCodecProvider.INSTANCE;
+      }
       // note, it's slighly sketchy to pass ourselves along partially
       // constructed, but only our _format and _pageChannel refs should be
       // needed
-      _pageChannel.initialize(this);
+      _pageChannel.initialize(this, provider);
       _buffer = _pageChannel.createPageBuffer();
       readSystemCatalog();
       success = true;

src/java/com/healthmarketscience/jackcess/DefaultCodecProvider.java

@@ -0,0 +1,125 @@
+/*
+Copyright (c) 2005 Health Market Science, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
+USA
+
+You can contact Health Market Science at info@healthmarketscience.com
+or at the following address:
+
+Health Market Science
+2700 Horizon Drive
+Suite 200
+King of Prussia, PA 19406
+*/
+
+package com.healthmarketscience.jackcess;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Default implementation of CodecProvider which does not have any actual
+ * encoding/decoding support.  See {@link CodecProvider} for details on a more
+ * useful implementation.
+ *
+ * @author James Ahlborn
+ */
+public class DefaultCodecProvider implements CodecProvider
+{
+  /** common instance of DefaultCodecProvider */
+  public static final CodecProvider INSTANCE = 
+    new DefaultCodecProvider();
+
+  /** common instance of {@link DummyHandler} */
+  public static final CodecHandler DUMMY_HANDLER = 
+    new DummyHandler();
+
+  /** common instance of {@link UnsupportedHandler} */
+  public static final CodecHandler UNSUPPORTED_HANDLER = 
+    new UnsupportedHandler();
+
+
+  /**
+   * {@inheritDoc}
+   * <p>
+   * This implementation returns DUMMY_HANDLER for databases with no encoding
+   * and UNSUPPORTED_HANDLER for databases with any encoding.
+   */
+  public CodecHandler createHandler(PageChannel channel)
+    throws IOException
+  {
+    JetFormat format = channel.getFormat();
+    switch(format.CODEC_TYPE) {
+    case NONE:
+      // no encoding, all good
+      return DUMMY_HANDLER;
+
+    case JET:
+      // check for an encode key.  if 0, not encoded
+      ByteBuffer bb = channel.createPageBuffer();
+      channel.readPage(bb, 0);
+      int codecKey = bb.getInt(format.OFFSET_ENCODING_KEY);
+      return((codecKey == 0) ? DUMMY_HANDLER : UNSUPPORTED_HANDLER);
+
+    case MSISAM:
+      // always encoded, we don't handle it
+      return UNSUPPORTED_HANDLER;
+    default:
+      throw new RuntimeException("Unknown codec type " + format.CODEC_TYPE);
+    }
+  }
+
+  /**
+   * CodecHandler implementation which does nothing, useful for databases with
+   * no extra encoding.
+   */
+  public static class DummyHandler implements CodecHandler
+  {
+    public void decodePage(ByteBuffer page, int pageNumber) throws IOException
+    {
+      // does nothing
+    }
+
+    public ByteBuffer encodePage(ByteBuffer page, int pageNumber, 
+                                 int pageOffset) 
+      throws IOException
+    {
+      // does nothing
+      return page;
+    }
+  }
+
+  /**
+   * CodecHandler implementation which always throws
+   * UnsupportedOperationException, useful for databases with unsupported
+   * encodings.
+   */
+  public static class UnsupportedHandler implements CodecHandler
+  {
+    public void decodePage(ByteBuffer page, int pageNumber) throws IOException
+    {
+      throw new UnsupportedOperationException("Decoding not supported");
+    }
+
+    public ByteBuffer encodePage(ByteBuffer page, int pageNumber, 
+                                 int pageOffset) 
+      throws IOException
+    {
+      throw new UnsupportedOperationException("Encoding not supported");
+    }
+  }
+
+}

src/java/com/healthmarketscience/jackcess/JetFormat.java

@@ -48,10 +48,14 @@ public abstract class JetFormat {
   public static final short TEXT_FIELD_UNIT_SIZE = 2;
   /** Maximum size of a text field */
   public static final short TEXT_FIELD_MAX_LENGTH = 255 * TEXT_FIELD_UNIT_SIZE;
+
+  public enum CodecType {
+    NONE, JET, MSISAM;
+  }
   
   /** Offset in the file that holds the byte describing the Jet format
       version */
-  private static final long OFFSET_VERSION = 20L;
+  private static final int OFFSET_VERSION = 20;
   /** Version code for Jet version 3 */
   private static final byte CODE_VERSION_3 = 0x0;
   /** Version code for Jet version 4 */
@@ -59,6 +63,15 @@ public abstract class JetFormat {
   /** Version code for Jet version 5 */
   private static final byte CODE_VERSION_5 = 0x2;
 
+  /** location of the engine name in the header */
+  private static final int OFFSET_ENGINE_NAME = 0x4;
+  /** amount of initial data to be read to determine database type */
+  private static final int HEADER_LENGTH = 21;
+  
+  private final static byte[] MSISAM_ENGINE = new byte[] {
+    'M', 'S', 'I', 'S', 'A', 'M', ' ', 'D', 'a', 't', 'a', 'b', 'a', 's', 'e'
+  };
+
   /** mask used to obfuscate the db header */
   private static final byte[] BASE_HEADER_MASK = new byte[]{
     (byte)0xB5, (byte)0x6F, (byte)0x03, (byte)0x62, (byte)0x61, (byte)0x08,
@@ -122,6 +135,9 @@ public abstract class JetFormat {
   
   /** whether or not we can use indexes in this format */
   public final boolean INDEXES_SUPPORTED;
+
+  /** type of page encoding supported */
+  public final CodecType CODEC_TYPE;
   
   /** Database page size in bytes */
   public final int PAGE_SIZE;
@@ -135,6 +151,7 @@ public abstract class JetFormat {
   public final int OFFSET_HEADER_DATE;
   public final int OFFSET_PASSWORD;
   public final int SIZE_PASSWORD;
+  public final int OFFSET_ENCODING_KEY;
   public final int OFFSET_NEXT_TABLE_DEF_PAGE;
   public final int OFFSET_NUM_ROWS;
   public final int OFFSET_NEXT_AUTO_NUMBER;
@@ -212,6 +229,7 @@ public abstract class JetFormat {
   
   public static final JetFormat VERSION_3 = new Jet3Format();
   public static final JetFormat VERSION_4 = new Jet4Format();
+  public static final JetFormat VERSION_MSISAM = new MSISAMFormat();
   public static final JetFormat VERSION_5 = new Jet5Format();
 
   /**
@@ -220,16 +238,19 @@ public abstract class JetFormat {
    * @throws IOException if the database file format is unsupported.
    */
   public static JetFormat getFormat(FileChannel channel) throws IOException {
-    ByteBuffer buffer = ByteBuffer.allocate(1);
-    int bytesRead = channel.read(buffer, OFFSET_VERSION);
-    if(bytesRead < 1) {
+    ByteBuffer buffer = ByteBuffer.allocate(HEADER_LENGTH);
+    int bytesRead = channel.read(buffer, 0L);
+    if(bytesRead < HEADER_LENGTH) {
       throw new IOException("Empty database file");
     }
     buffer.flip();
-    byte version = buffer.get();
+    byte version = buffer.get(OFFSET_VERSION);
     if (version == CODE_VERSION_3) {
       return VERSION_3;
     } else if (version == CODE_VERSION_4) {
+      if(ByteUtil.matchesRange(buffer, OFFSET_ENGINE_NAME, MSISAM_ENGINE)) {
+        return VERSION_MSISAM;
+      }
       return VERSION_4;
     } else if (version == CODE_VERSION_5) {
       return VERSION_5;
@@ -244,6 +265,7 @@ public abstract class JetFormat {
     
     READ_ONLY = defineReadOnly();
     INDEXES_SUPPORTED = defineIndexesSupported();
+    CODEC_TYPE = defineCodecType();
     
     PAGE_SIZE = definePageSize();
     MAX_DATABASE_SIZE = defineMaxDatabaseSize();
@@ -256,6 +278,7 @@ public abstract class JetFormat {
     OFFSET_HEADER_DATE = defineOffsetHeaderDate();
     OFFSET_PASSWORD = defineOffsetPassword();
     SIZE_PASSWORD = defineSizePassword();
+    OFFSET_ENCODING_KEY = defineOffsetEncodingKey();
     OFFSET_NEXT_TABLE_DEF_PAGE = defineOffsetNextTableDefPage();
     OFFSET_NUM_ROWS = defineOffsetNumRows();
     OFFSET_NEXT_AUTO_NUMBER = defineOffsetNextAutoNumber();
@@ -334,6 +357,7 @@ public abstract class JetFormat {
   
   protected abstract boolean defineReadOnly();
   protected abstract boolean defineIndexesSupported();
+  protected abstract CodecType defineCodecType();
   
   protected abstract int definePageSize();
   protected abstract long defineMaxDatabaseSize();
@@ -346,6 +370,7 @@ public abstract class JetFormat {
   protected abstract int defineOffsetHeaderDate();
   protected abstract int defineOffsetPassword();
   protected abstract int defineSizePassword();
+  protected abstract int defineOffsetEncodingKey();
   protected abstract int defineOffsetNextTableDefPage();
   protected abstract int defineOffsetNumRows();
   protected abstract int defineOffsetNextAutoNumber();
@@ -439,6 +464,11 @@ public abstract class JetFormat {
 	    
     @Override
     protected boolean defineIndexesSupported() { return false; }
+
+    @Override
+    protected CodecType defineCodecType() { 
+      return CodecType.JET; 
+    }
 	    
     @Override
     protected int definePageSize() { return 2048; }
@@ -468,6 +498,8 @@ public abstract class JetFormat {
     @Override
     protected int defineSizePassword() { return 20; }
     @Override
+    protected int defineOffsetEncodingKey() { return 62; }
+    @Override
     protected int defineOffsetNextTableDefPage() { return 4; }
     @Override
     protected int defineOffsetNumRows() { return 12; }
@@ -630,6 +662,11 @@ public abstract class JetFormat {
     @Override
     protected boolean defineIndexesSupported() { return true; }
 	    
+    @Override
+    protected CodecType defineCodecType() { 
+      return CodecType.JET; 
+    }
+
     @Override
     protected int definePageSize() { return 4096; }
     
@@ -654,6 +691,8 @@ public abstract class JetFormat {
     @Override
     protected int defineSizePassword() { return 40; }
     @Override
+    protected int defineOffsetEncodingKey() { return 62; }
+    @Override
     protected int defineOffsetNextTableDefPage() { return 4; }
     @Override
     protected int defineOffsetNumRows() { return 16; }
@@ -800,6 +839,22 @@ public abstract class JetFormat {
 
   }
   
+  private static final class MSISAMFormat extends Jet4Format {
+    private MSISAMFormat() {
+      super("MSISAM");
+    }
+
+    @Override
+    protected boolean defineReadOnly() {
+      return true;
+    }
+
+    @Override
+    protected CodecType defineCodecType() { 
+      return CodecType.MSISAM; 
+    }
+  }
+
   private static final class Jet5Format extends Jet4Format {
       private Jet5Format() {
         super("VERSION_5");

src/java/com/healthmarketscience/jackcess/PageChannel.java

@@ -74,6 +74,8 @@ public class PageChannel implements Channel, Flushable {
   private final ByteBuffer _forceBytes = ByteBuffer.allocate(1);
   /** Tracks free pages in the database. */
   private UsageMap _globalUsageMap;
+  /** handler for the current database encoding type */
+  private CodecHandler _codecHandler = DefaultCodecProvider.DUMMY_HANDLER;
   
   /**
    * @param channel Channel containing the database
@@ -90,13 +92,16 @@ public class PageChannel implements Channel, Flushable {
   /**
    * Does second-stage initialization, must be called after construction.
    */
-  public void initialize(Database database)
+  public void initialize(Database database, CodecProvider codecProvider)
     throws IOException
   {
     // note the global usage map is a special map where any page outside of
     // the current range is assumed to be "on"
     _globalUsageMap = UsageMap.read(database, PAGE_GLOBAL_USAGE_MAP,
                                     ROW_GLOBAL_USAGE_MAP, true);
+
+    // initialize page en/decoding support
+    _codecHandler = codecProvider.createHandler(this);
   }
   
   /**
@@ -163,8 +168,10 @@ public class PageChannel implements Channel, Flushable {
     }
 
     if(pageNumber == 0) {
-      // de-mask header
+      // de-mask header (note, page 0 never has additional encoding)
       applyHeaderMask(buffer);
+    } else {
+      _codecHandler.decodePage(buffer, pageNumber);
     }
   }
   
@@ -184,8 +191,7 @@ public class PageChannel implements Channel, Flushable {
    * @param pageOffset offset within the page at which to start writing the
    *                   page data
    */
-  public void writePage(ByteBuffer page, int pageNumber,
-                        int pageOffset)
+  public void writePage(ByteBuffer page, int pageNumber, int pageOffset)
     throws IOException
   {
     validatePageNumber(pageNumber);
@@ -197,13 +203,17 @@ public class PageChannel implements Channel, Flushable {
           "Page buffer is too large, size " + (page.remaining() - pageOffset));
     }
     
+    ByteBuffer encodedPage = page;
     if(pageNumber == 0) {
       // re-mask header
       applyHeaderMask(page);
+    } else {
+      // re-encode page
+      encodedPage = _codecHandler.encodePage(page, pageNumber, pageOffset);
     }
     try {
-      page.position(pageOffset);
-      _channel.write(page, (getPageOffset(pageNumber) + pageOffset));
+      encodedPage.position(pageOffset);
+      _channel.write(encodedPage, (getPageOffset(pageNumber) + pageOffset));
       if(_autoSync) {
         flush();
       }
@@ -243,9 +253,11 @@ public class PageChannel implements Channel, Flushable {
     // push the buffer to the end of the page, so that a full page's worth of
     // data is written regardless of the incoming buffer size (we use a tiny
     // buffer in allocateNewPage)
-    long offset = size + (getFormat().PAGE_SIZE - page.remaining());
-    _channel.write(page, offset);
+    int pageOffset = (getFormat().PAGE_SIZE - page.remaining());
+    long offset = size + pageOffset;
     int pageNumber = getNextPageNumber(size);
+    _channel.write(_codecHandler.encodePage(page, pageNumber, pageOffset),
+                   offset);
     _globalUsageMap.removePageNumber(pageNumber);  //force is done here
     return pageNumber;
   }