merge branch a97_indexes changes through r1313

git-svn-id: https://svn.code.sf.net/p/jackcess/code/jackcess/trunk@1314 f203690c-595d-4dc9-a70b-905162fa7fd2

src/main/java/com/healthmarketscience/jackcess/CursorBuilder.java

@@ -27,6 +27,7 @@ import com.healthmarketscience.jackcess.impl.IndexCursorImpl;
 import com.healthmarketscience.jackcess.impl.IndexData;
 import com.healthmarketscience.jackcess.impl.IndexImpl;
 import com.healthmarketscience.jackcess.impl.TableImpl;
+import com.healthmarketscience.jackcess.util.CaseInsensitiveColumnMatcher;
 import com.healthmarketscience.jackcess.util.ColumnMatcher;
 
 
@@ -82,7 +83,7 @@ public class CursorBuilder {
     _beforeFirst = true;
     return this;
   }
-  
+
   /**
    * Sets the cursor so that it will start at the end (unless a savepoint is
    * given).
@@ -167,7 +168,7 @@ public class CursorBuilder {
     setEndRow(specificRow);
     return this;
   }
-  
+
   /**
    * Sets the starting and ending row for a range based index cursor to the
    * given entry (where the given values correspond to the index's columns).
@@ -181,7 +182,7 @@ public class CursorBuilder {
     return this;
   }
 
-  
+
   /**
    * Sets the starting row for a range based index cursor.
    * <p>
@@ -191,7 +192,7 @@ public class CursorBuilder {
     _startRow = startRow;
     return this;
   }
-  
+
   /**
    * Sets the starting row for a range based index cursor to the given entry
    * (where the given values correspond to the index's columns).
@@ -224,7 +225,7 @@ public class CursorBuilder {
     _endRow = endRow;
     return this;
   }
-  
+
   /**
    * Sets the ending row for a range based index cursor to the given entry
    * (where the given values correspond to the index's columns).
@@ -256,6 +257,13 @@ public class CursorBuilder {
     return this;
   }
 
+  /**
+   * Sets the ColumnMatcher to an instance of CaseInsensitiveColumnMatcher
+   */
+  public CursorBuilder setCaseInsensitive() {
+    return setColumnMatcher(CaseInsensitiveColumnMatcher.INSTANCE);
+  }
+
   /**
    * Returns a new cursor for the table, constructed to the given
    * specifications.
@@ -280,7 +288,7 @@ public class CursorBuilder {
     }
     return cursor;
   }
-  
+
   /**
    * Returns a new index cursor for the table, constructed to the given
    * specifications.
@@ -304,7 +312,7 @@ public class CursorBuilder {
    * Note, index based table traversal may not include all rows, as certain
    * types of indexes do not include all entries (namely, some indexes ignore
    * null entries, see {@link Index#shouldIgnoreNulls}).
-   * 
+   *
    * @param index index for the table which will define traversal order as
    *              well as enhance certain lookups
    */
@@ -323,7 +331,7 @@ public class CursorBuilder {
   {
     return createCursor(table.getPrimaryKeyIndex());
   }
-  
+
   /**
    * Creates an indexed cursor for the given table, narrowed to the given
    * range.
@@ -331,7 +339,7 @@ public class CursorBuilder {
    * Note, index based table traversal may not include all rows, as certain
    * types of indexes do not include all entries (namely, some indexes ignore
    * null entries, see {@link Index#shouldIgnoreNulls}).
-   * 
+   *
    * @param index index for the table which will define traversal order as
    *              well as enhance certain lookups
    * @param startRow the first row of data for the cursor (inclusive), or
@@ -348,7 +356,7 @@ public class CursorBuilder {
       .setEndRow(endRow)
       .toIndexCursor();
   }
-  
+
   /**
    * Creates an indexed cursor for the given table, narrowed to the given
    * range.
@@ -356,7 +364,7 @@ public class CursorBuilder {
    * Note, index based table traversal may not include all rows, as certain
    * types of indexes do not include all entries (namely, some indexes ignore
    * null entries, see {@link Index#shouldIgnoreNulls}).
-   * 
+   *
    * @param index index for the table which will define traversal order as
    *              well as enhance certain lookups
    * @param startRow the first row of data for the cursor, or {@code null} for
@@ -388,7 +396,7 @@ public class CursorBuilder {
    * <p>
    * Warning, this method <i>always</i> starts searching from the beginning of
    * the Table (you cannot use it to find successive matches).
-   * 
+   *
    * @param table the table to search
    * @param rowPattern pattern to be used to find the row
    * @return the matching row or {@code null} if a match could not be found.
@@ -402,12 +410,12 @@ public class CursorBuilder {
     }
     return null;
   }
-  
+
   /**
    * Convenience method for finding a specific row (as defined by the cursor)
    * where the index entries match the given values.  See {@link
    * IndexCursor#findRowByEntry(Object...)} for details on the entryValues.
-   * 
+   *
    * @param index the index to search
    * @param entryValues the column values for the index's columns.
    * @return the matching row or {@code null} if a match could not be found.
@@ -417,12 +425,12 @@ public class CursorBuilder {
   {
     return createCursor(index).findRowByEntry(entryValues);
   }
-  
+
   /**
    * Convenience method for finding a specific row by the primary key of the
    * table.  See {@link IndexCursor#findRowByEntry(Object...)} for details on
    * the entryValues.
-   * 
+   *
    * @param table the table to search
    * @param entryValues the column values for the table's primary key columns.
    * @return the matching row or {@code null} if a match could not be found.
@@ -432,7 +440,7 @@ public class CursorBuilder {
   {
     return findRowByEntry(table.getPrimaryKeyIndex(), entryValues);
   }
-  
+
   /**
    * Convenience method for finding a specific row in a table which matches a
    * given row "pattern".  See {@link Cursor#findFirstRow(Column,Object)} for
@@ -442,7 +450,7 @@ public class CursorBuilder {
    * match or a matching row with {@code null} for the desired value.  If
    * distinguishing this situation is important, you will need to use a Cursor
    * directly instead of this convenience method.
-   * 
+   *
    * @param table the table to search
    * @param column column whose value should be returned
    * @param columnPattern column being matched by the valuePattern
@@ -460,7 +468,7 @@ public class CursorBuilder {
     }
     return null;
   }
-  
+
   /**
    * Convenience method for finding a specific row in an indexed table which
    * matches a given row "pattern".  See {@link Cursor#findFirstRow(Map)} for
@@ -468,7 +476,7 @@ public class CursorBuilder {
    * <p>
    * Warning, this method <i>always</i> starts searching from the beginning of
    * the Table (you cannot use it to find successive matches).
-   * 
+   *
    * @param index index to assist the search
    * @param rowPattern pattern to be used to find the row
    * @return the matching row or {@code null} if a match could not be found.
@@ -482,7 +490,7 @@ public class CursorBuilder {
     }
     return null;
   }
-  
+
   /**
    * Convenience method for finding a specific row in a table which matches a
    * given row "pattern".  See {@link Cursor#findFirstRow(Column,Object)} for
@@ -492,7 +500,7 @@ public class CursorBuilder {
    * match or a matching row with {@code null} for the desired value.  If
    * distinguishing this situation is important, you will need to use a Cursor
    * directly instead of this convenience method.
-   * 
+   *
    * @param index index to assist the search
    * @param column column whose value should be returned
    * @param columnPattern column being matched by the valuePattern

src/main/java/com/healthmarketscience/jackcess/impl/ByteUtil.java

@@ -710,7 +710,10 @@ public final class ByteUtil {
     }
 
     protected void ensureNewCapacity(int numBytes) {
-      int newLength = _length + numBytes;
+      ensureCapacity(_length + numBytes);
+    }
+
+    protected void ensureCapacity(int newLength) {
       if(newLength > _bytes.length) {
         byte[] temp = new byte[newLength * 2];
         System.arraycopy(_bytes, 0, temp, 0, _length);
@@ -744,6 +747,10 @@ public final class ByteUtil {
       _bytes[offset] = b;
     }
 
+    public void setBits(int offset, byte b) {
+      _bytes[offset] |= b;
+    }
+
     public void writeFill(int length, byte b) {
       ensureNewCapacity(length);
       int oldLength = _length;

src/main/java/com/healthmarketscience/jackcess/impl/ColumnImpl.java

@@ -159,19 +159,26 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl>, DateTimeConte
   /** the value for the "general" sort order */
   private static final short GENERAL_SORT_ORDER_VALUE = 1033;
 
+  /**
+   * the "general" text sort order, version (access 1997)
+   * @usage _intermediate_field_
+   */
+  public static final SortOrder GENERAL_97_SORT_ORDER =
+    new SortOrder(GENERAL_SORT_ORDER_VALUE, (short)-1);
+
   /**
    * the "general" text sort order, legacy version (access 2000-2007)
    * @usage _intermediate_field_
    */
   public static final SortOrder GENERAL_LEGACY_SORT_ORDER =
-    new SortOrder(GENERAL_SORT_ORDER_VALUE, (byte)0);
+    new SortOrder(GENERAL_SORT_ORDER_VALUE, (short)0);
 
   /**
    * the "general" text sort order, latest version (access 2010+)
    * @usage _intermediate_field_
    */
   public static final SortOrder GENERAL_SORT_ORDER =
-    new SortOrder(GENERAL_SORT_ORDER_VALUE, (byte)1);
+    new SortOrder(GENERAL_SORT_ORDER_VALUE, (short)1);
 
   /** pattern matching textual guid strings (allows for optional surrounding
       '{' and '}') */
@@ -2077,22 +2084,26 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl>, DateTimeConte
                                  JetFormat format)
   {
     short value = buffer.getShort(position);
-    byte version = 0;
-    if(format.SIZE_SORT_ORDER == 4) {
-      version = buffer.get(position + 3);
-    }
 
     if(value == 0) {
       // probably a file we wrote, before handling sort order
       return format.DEFAULT_SORT_ORDER;
     }
 
+    short version = format.DEFAULT_SORT_ORDER.getVersion();
+    if(format.SIZE_SORT_ORDER == 4) {
+      version = buffer.get(position + 3);
+    }
+
     if(value == GENERAL_SORT_ORDER_VALUE) {
+      if(version == GENERAL_SORT_ORDER.getVersion()) {
+        return GENERAL_SORT_ORDER;
+      }
       if(version == GENERAL_LEGACY_SORT_ORDER.getVersion()) {
         return GENERAL_LEGACY_SORT_ORDER;
       }
-      if(version == GENERAL_SORT_ORDER.getVersion()) {
-        return GENERAL_SORT_ORDER;
+      if(version == GENERAL_97_SORT_ORDER.getVersion()) {
+        return GENERAL_97_SORT_ORDER;
       }
     }
     return new SortOrder(value, version);
@@ -2128,7 +2139,7 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl>, DateTimeConte
     buffer.putShort(sortOrder.getValue());
     if(format.SIZE_SORT_ORDER == 4) {
       buffer.put((byte)0x00); // unknown
-      buffer.put(sortOrder.getVersion());
+      buffer.put((byte)sortOrder.getVersion());
     }
   }
 
@@ -2563,9 +2574,9 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl>, DateTimeConte
   public static final class SortOrder
   {
     private final short _value;
-    private final byte _version;
+    private final short _version;
 
-    public SortOrder(short value, byte version) {
+    public SortOrder(short value, short version) {
       _value = value;
       _version = version;
     }
@@ -2574,7 +2585,7 @@ public class ColumnImpl implements Column, Comparable<ColumnImpl>, DateTimeConte
       return _value;
     }
 
-    public byte getVersion() {
+    public short getVersion() {
       return _version;
     }
 

src/main/java/com/healthmarketscience/jackcess/impl/DatabaseImpl.java

@@ -942,7 +942,7 @@ public class DatabaseImpl implements Database, DateTimeContext
   private void initRootPageInfo() throws IOException {
     ByteBuffer buffer = takeSharedBuffer();
     try {
-      _pageChannel.readPage(buffer, 0);
+      _pageChannel.readRootPage(buffer);
       _defaultSortOrder = ColumnImpl.readSortOrder(
           buffer, _format.OFFSET_SORT_ORDER, _format);
       _defaultCodePage = buffer.getShort(_format.OFFSET_CODE_PAGE);
@@ -1586,7 +1586,7 @@ public class DatabaseImpl implements Database, DateTimeContext
   {
     ByteBuffer buffer = takeSharedBuffer();
     try {
-      _pageChannel.readPage(buffer, 0);
+      _pageChannel.readRootPage(buffer);
 
       byte[] pwdBytes = new byte[_format.SIZE_PASSWORD];
       buffer.position(_format.OFFSET_PASSWORD);

src/main/java/com/healthmarketscience/jackcess/impl/DefaultCodecProvider.java

@@ -30,15 +30,15 @@ import java.nio.charset.Charset;
 public class DefaultCodecProvider implements CodecProvider
 {
   /** common instance of DefaultCodecProvider */
-  public static final CodecProvider INSTANCE = 
+  public static final CodecProvider INSTANCE =
     new DefaultCodecProvider();
 
   /** common instance of {@link DummyHandler} */
-  public static final CodecHandler DUMMY_HANDLER = 
+  public static final CodecHandler DUMMY_HANDLER =
     new DummyHandler();
 
   /** common instance of {@link UnsupportedHandler} */
-  public static final CodecHandler UNSUPPORTED_HANDLER = 
+  public static final CodecHandler UNSUPPORTED_HANDLER =
     new UnsupportedHandler();
 
 
@@ -62,7 +62,7 @@ public class DefaultCodecProvider implements CodecProvider
     case OFFICE:
       // check for an encode key.  if 0, not encoded
       ByteBuffer bb = channel.createPageBuffer();
-      channel.readPage(bb, 0);
+      channel.readRootPage(bb);
       int codecKey = bb.getInt(format.OFFSET_ENCODING_KEY);
       return((codecKey == 0) ? DUMMY_HANDLER : UNSUPPORTED_HANDLER);
 
@@ -93,15 +93,15 @@ public class DefaultCodecProvider implements CodecProvider
 
     @Override
     public void decodePage(ByteBuffer inPage, ByteBuffer outPage,
-                           int pageNumber) 
-      throws IOException 
+                           int pageNumber)
+      throws IOException
     {
       // does nothing
     }
 
     @Override
-    public ByteBuffer encodePage(ByteBuffer page, int pageNumber, 
-                                 int pageOffset) 
+    public ByteBuffer encodePage(ByteBuffer page, int pageNumber,
+                                 int pageOffset)
       throws IOException
     {
       // does nothing
@@ -127,16 +127,16 @@ public class DefaultCodecProvider implements CodecProvider
     }
 
     @Override
-    public void decodePage(ByteBuffer inPage, ByteBuffer outPage, 
-                           int pageNumber) 
-      throws IOException 
+    public void decodePage(ByteBuffer inPage, ByteBuffer outPage,
+                           int pageNumber)
+      throws IOException
     {
       throw new UnsupportedCodecException("Decoding not supported.  Please choose a CodecProvider which supports reading the current database encoding.");
     }
 
     @Override
-    public ByteBuffer encodePage(ByteBuffer page, int pageNumber, 
-                                 int pageOffset) 
+    public ByteBuffer encodePage(ByteBuffer page, int pageNumber,
+                                 int pageOffset)
       throws IOException
     {
       throw new UnsupportedCodecException("Encoding not supported.  Please choose a CodecProvider which supports writing the current database encoding.");

src/main/java/com/healthmarketscience/jackcess/impl/General97IndexCodes.java

@@ -0,0 +1,292 @@
+/*
+Copyright (c) 2019 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.impl;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import static com.healthmarketscience.jackcess.impl.ByteUtil.ByteStream;
+
+/**
+ * Various constants used for creating "general" (access 1997) sort order
+ * text index entries.
+ *
+ * @author James Ahlborn
+ */
+public class General97IndexCodes extends GeneralLegacyIndexCodes
+{
+  // stash the codes in some resource files
+  private static final String CODES_FILE =
+    DatabaseImpl.RESOURCE_PATH + "index_codes_gen_97.txt";
+  private static final String EXT_MAPPINGS_FILE =
+    DatabaseImpl.RESOURCE_PATH + "index_mappings_ext_gen_97.txt";
+
+  // we only have a small range of extended chars which can mapped back into
+  // the valid chars
+  private static final char FIRST_MAP_CHAR = 338;
+  private static final char LAST_MAP_CHAR = 8482;
+
+  private static final byte EXT_CODES_BOUNDS_NIBBLE = (byte)0x00;
+
+  private static final class Codes
+  {
+    /** handlers for the first 256 chars.  use nested class to lazy load the
+        handlers */
+    private static final CharHandler[] _values = loadCodes(
+        CODES_FILE, FIRST_CHAR, LAST_CHAR);
+  }
+
+  private static final class ExtMappings
+  {
+    /** mappings for a small subset of the rest of the chars in BMP 0.  use
+        nested class to lazy load the handlers.  since these codes are for
+        single byte encodings, you would think you wouldn't need any ext
+        codes.  however, some chars in the extended range have corollaries in
+        the single byte range. this array holds the mappings from the ext
+        range to the single byte range.  chars without mappings go to 0
+        (ignored). */
+    private static final short[] _values = loadMappings(
+        EXT_MAPPINGS_FILE, FIRST_MAP_CHAR, LAST_MAP_CHAR);
+  }
+
+  static final General97IndexCodes GEN_97_INSTANCE = new General97IndexCodes();
+
+  General97IndexCodes() {}
+
+  /**
+   * Returns the CharHandler for the given character.
+   */
+  @Override
+  CharHandler getCharHandler(char c)
+  {
+    if(c <= LAST_CHAR) {
+      return Codes._values[c];
+    }
+
+    if((c < FIRST_MAP_CHAR) || (c > LAST_MAP_CHAR)) {
+      // outside the mapped range, ignored
+      return IGNORED_CHAR_HANDLER;
+    }
+
+    // some ext chars are equivalent to single byte chars.  most chars have no
+    // equivalent, and they map to 0 (which is an "ignored" char, so it all
+    // works out)
+    int extOffset = asUnsignedChar(c) - asUnsignedChar(FIRST_MAP_CHAR);
+    return Codes._values[ExtMappings._values[extOffset]];
+  }
+
+  /**
+   * Converts a 97 index value for a text column into the entry value (which
+   * is based on a variety of nifty codes).
+   */
+  @Override
+  void writeNonNullIndexTextValue(
+      Object value, ByteStream bout, boolean isAscending)
+    throws IOException
+  {
+    // first, convert to string
+    String str = ColumnImpl.toCharSequence(value).toString();
+
+    // all text columns (including memos) are only indexed up to the max
+    // number of chars in a VARCHAR column
+    if(str.length() > MAX_TEXT_INDEX_CHAR_LENGTH) {
+      str = str.substring(0, MAX_TEXT_INDEX_CHAR_LENGTH);
+    }
+
+    // record previous entry length so we can do any post-processing
+    // necessary for this entry (handling descending)
+    int prevLength = bout.getLength();
+
+    // now, convert each character to a "code" of one or more bytes
+    NibbleStream extraCodes = null;
+    int sigCharCount = 0;
+    for(int i = 0; i < str.length(); ++i) {
+
+      char c = str.charAt(i);
+      CharHandler ch = getCharHandler(c);
+
+      byte[] bytes = ch.getInlineBytes();
+      if(bytes != null) {
+        // write the "inline" codes immediately
+        bout.write(bytes);
+      }
+
+      if(ch.getType() == Type.SIMPLE) {
+        // common case, skip further code handling
+        continue;
+      }
+
+      if(ch.isSignificantChar()) {
+        ++sigCharCount;
+        // significant chars never have extra bytes
+        continue;
+      }
+
+      bytes = ch.getExtraBytes();
+      if(bytes != null) {
+        if(extraCodes == null) {
+          extraCodes = new NibbleStream(str.length());
+          extraCodes.writeNibble(EXT_CODES_BOUNDS_NIBBLE);
+        }
+
+        // keep track of the extra code for later
+        writeExtraCodes(sigCharCount, bytes, extraCodes);
+        sigCharCount = 0;
+      }
+    }
+
+    if(extraCodes != null) {
+
+      // write the extra codes to the end
+      extraCodes.writeNibble(EXT_CODES_BOUNDS_NIBBLE);
+      extraCodes.writeTo(bout);
+
+    } else {
+
+      // write end extra text
+      bout.write(END_EXTRA_TEXT);
+    }
+
+    // handle descending order by inverting the bytes
+    if(!isAscending) {
+
+      // flip the bytes that we have written thus far for this text value
+      IndexData.flipBytes(bout.getBytes(), prevLength,
+                          (bout.getLength() - prevLength));
+    }
+  }
+
+  private static void writeExtraCodes(int numSigChars, byte[] bytes,
+                                      NibbleStream extraCodes)
+  {
+    // need to fill in placeholder nibbles for any "significant" chars
+    if(numSigChars > 0) {
+      extraCodes.writeFillNibbles(numSigChars, INTERNATIONAL_EXTRA_PLACEHOLDER);
+    }
+
+    // there should only ever be a single "extra" byte
+    extraCodes.writeNibble(bytes[0]);
+  }
+
+  static short[] loadMappings(String mappingsFilePath,
+                              char firstChar, char lastChar)
+  {
+    int firstCharCode = asUnsignedChar(firstChar);
+    int numMappings = (asUnsignedChar(lastChar) - firstCharCode) + 1;
+    short[] values = new short[numMappings];
+
+    BufferedReader reader = null;
+    try {
+
+      reader = new BufferedReader(
+          new InputStreamReader(
+              DatabaseImpl.getResourceAsStream(mappingsFilePath), "US-ASCII"));
+
+      // this is a sparse file with entries like <fromCode>,<toCode>
+      String mappingLine = null;
+      while((mappingLine = reader.readLine()) != null) {
+        mappingLine = mappingLine.trim();
+        if(mappingLine.length() == 0) {
+          continue;
+        }
+
+        String[] mappings = mappingLine.split(",");
+        int fromCode = Integer.parseInt(mappings[0]);
+        int toCode = Integer.parseInt(mappings[1]);
+
+        values[fromCode - firstCharCode] = (short)toCode;
+      }
+
+    } catch(IOException e) {
+      throw new RuntimeException("failed loading index mappings file " +
+                                 mappingsFilePath, e);
+    } finally {
+      ByteUtil.closeQuietly(reader);
+    }
+
+    return values;
+  }
+
+  /**
+   * Extension of ByteStream which enables writing individual nibbles.
+   */
+  protected static final class NibbleStream extends ByteStream
+  {
+    private int _nibbleLen;
+
+    protected NibbleStream(int length) {
+      super(length);
+    }
+
+    private boolean nextIsHi() {
+      return (_nibbleLen % 2) == 0;
+    }
+
+    private static int asLowNibble(int b) {
+      return (b & 0x0F);
+    }
+
+    private static int asHiNibble(int b) {
+      return ((b << 4) & 0xF0);
+    }
+
+    private void writeLowNibble(int b) {
+      int byteOff = _nibbleLen / 2;
+      setBits(byteOff, (byte)asLowNibble(b));
+    }
+
+    public void writeNibble(int b) {
+
+      if(nextIsHi()) {
+        write(asHiNibble(b));
+      } else {
+        writeLowNibble(b);
+      }
+
+      ++_nibbleLen;
+    }
+
+    public void writeFillNibbles(int length, byte b) {
+
+      int newNibbleLen = _nibbleLen + length;
+      ensureCapacity((newNibbleLen + 1) / 2);
+
+      if(!nextIsHi()) {
+        writeLowNibble(b);
+        --length;
+      }
+
+      if(length > 1) {
+        byte doubleB = (byte)(asHiNibble(b) | asLowNibble(b));
+
+        do {
+          write(doubleB);
+          length -= 2;
+        } while(length > 1);
+      }
+
+      if(length == 1) {
+        write(asHiNibble(b));
+      }
+
+      _nibbleLen = newNibbleLen;
+    }
+
+  }
+
+}

src/main/java/com/healthmarketscience/jackcess/impl/GeneralLegacyIndexCodes.java

@@ -52,20 +52,20 @@ public class GeneralLegacyIndexCodes {
   // international char is replaced with ascii char.
   // pattern for international chars in the extra bytes:
   // [ 02 (for each normal char) ] [ <symbol_code> (for each inat char) ]
-  static final byte INTERNATIONAL_EXTRA_PLACEHOLDER = (byte)0x02;  
+  static final byte INTERNATIONAL_EXTRA_PLACEHOLDER = (byte)0x02;
 
   // see Index.writeCrazyCodes for details on writing crazy codes
   static final byte CRAZY_CODE_START = (byte)0x80;
   static final byte CRAZY_CODE_1 = (byte)0x02;
   static final byte CRAZY_CODE_2 = (byte)0x03;
-  static final byte[] CRAZY_CODES_SUFFIX = 
+  static final byte[] CRAZY_CODES_SUFFIX =
     new byte[]{(byte)0xFF, (byte)0x02, (byte)0x80, (byte)0xFF, (byte)0x80};
   static final byte CRAZY_CODES_UNPRINT_SUFFIX = (byte)0xFF;
 
   // stash the codes in some resource files
-  private static final String CODES_FILE = 
+  private static final String CODES_FILE =
     DatabaseImpl.RESOURCE_PATH + "index_codes_genleg.txt";
-  private static final String EXT_CODES_FILE = 
+  private static final String EXT_CODES_FILE =
     DatabaseImpl.RESOURCE_PATH + "index_codes_ext_genleg.txt";
 
   /**
@@ -98,6 +98,11 @@ public class GeneralLegacyIndexCodes {
         return parseInternationalExtCodes(codeStrings);
       }
     },
+    SIGNIFICANT("G") {
+      @Override public CharHandler parseCodes(String[] codeStrings) {
+        return parseSignificantCodes(codeStrings);
+      }
+    },
     IGNORED("X") {
       @Override public CharHandler parseCodes(String[] codeStrings) {
         return IGNORED_CHAR_HANDLER;
@@ -138,13 +143,16 @@ public class GeneralLegacyIndexCodes {
     public byte getCrazyFlag() {
       return 0;
     }
+    public boolean isSignificantChar() {
+      return false;
+    }
   }
 
   /**
    * CharHandler for Type.SIMPLE
    */
   private static final class SimpleCharHandler extends CharHandler {
-    private byte[] _bytes;
+    private final byte[] _bytes;
     private SimpleCharHandler(byte[] bytes) {
       _bytes = bytes;
     }
@@ -160,8 +168,8 @@ public class GeneralLegacyIndexCodes {
    * CharHandler for Type.INTERNATIONAL
    */
   private static final class InternationalCharHandler extends CharHandler {
-    private byte[] _bytes;
-    private byte[] _extraBytes;
+    private final byte[] _bytes;
+    private final byte[] _extraBytes;
     private InternationalCharHandler(byte[] bytes, byte[] extraBytes) {
       _bytes = bytes;
       _extraBytes = extraBytes;
@@ -181,7 +189,7 @@ public class GeneralLegacyIndexCodes {
    * CharHandler for Type.UNPRINTABLE
    */
   private static final class UnprintableCharHandler extends CharHandler {
-    private byte[] _unprintBytes;
+    private final byte[] _unprintBytes;
     private UnprintableCharHandler(byte[] unprintBytes) {
       _unprintBytes = unprintBytes;
     }
@@ -197,7 +205,7 @@ public class GeneralLegacyIndexCodes {
    * CharHandler for Type.UNPRINTABLE_EXT
    */
   private static final class UnprintableExtCharHandler extends CharHandler {
-    private byte _extraByteMod;
+    private final byte _extraByteMod;
     private UnprintableExtCharHandler(Byte extraByteMod) {
       _extraByteMod = extraByteMod;
     }
@@ -213,9 +221,9 @@ public class GeneralLegacyIndexCodes {
    * CharHandler for Type.INTERNATIONAL_EXT
    */
   private static final class InternationalExtCharHandler extends CharHandler {
-    private byte[] _bytes;
-    private byte[] _extraBytes;
-    private byte _crazyFlag;
+    private final byte[] _bytes;
+    private final byte[] _extraBytes;
+    private final byte _crazyFlag;
     private InternationalExtCharHandler(byte[] bytes, byte[] extraBytes,
                                         byte crazyFlag) {
       _bytes = bytes;
@@ -236,6 +244,25 @@ public class GeneralLegacyIndexCodes {
     }
   }
 
+  /**
+   * CharHandler for Type.SIGNIFICANT
+   */
+  private static final class SignificantCharHandler extends CharHandler {
+    private final byte[] _bytes;
+    private SignificantCharHandler(byte[] bytes) {
+      _bytes = bytes;
+    }
+    @Override public Type getType() {
+      return Type.SIGNIFICANT;
+    }
+    @Override public byte[] getInlineBytes() {
+      return _bytes;
+    }
+    @Override public boolean isSignificantChar() {
+      return true;
+    }
+  }
+
   /** shared CharHandler instance for Type.IGNORED */
   static final CharHandler IGNORED_CHAR_HANDLER = new CharHandler() {
     @Override public Type getType() {
@@ -267,7 +294,7 @@ public class GeneralLegacyIndexCodes {
     private static final CharHandler[] _values = loadCodes(
         CODES_FILE, FIRST_CHAR, LAST_CHAR);
   }
-  
+
   private static final class ExtCodes
   {
     /** handlers for the rest of the chars in BMP 0.  use nested class to
@@ -276,9 +303,9 @@ public class GeneralLegacyIndexCodes {
         EXT_CODES_FILE, FIRST_EXT_CHAR, LAST_EXT_CHAR);
   }
 
-  static final GeneralLegacyIndexCodes GEN_LEG_INSTANCE = 
+  static final GeneralLegacyIndexCodes GEN_LEG_INSTANCE =
     new GeneralLegacyIndexCodes();
-  
+
   GeneralLegacyIndexCodes() {
   }
 
@@ -316,7 +343,7 @@ public class GeneralLegacyIndexCodes {
       reader = new BufferedReader(
           new InputStreamReader(
               DatabaseImpl.getResourceAsStream(codesFilePath), "US-ASCII"));
-      
+
       int start = asUnsignedChar(firstChar);
       int end = asUnsignedChar(lastChar);
       for(int i = start; i <= end; ++i) {
@@ -357,7 +384,7 @@ public class GeneralLegacyIndexCodes {
   /**
    * Returns a SimpleCharHandler parsed from the given index code strings.
    */
-  private static CharHandler parseSimpleCodes(String[] codeStrings) 
+  private static CharHandler parseSimpleCodes(String[] codeStrings)
   {
     if(codeStrings.length != 1) {
       throw new IllegalStateException("Unexpected code strings " +
@@ -397,7 +424,7 @@ public class GeneralLegacyIndexCodes {
    * Returns a UnprintableExtCharHandler parsed from the given index code
    * strings.
    */
-  private static CharHandler parseUnprintableExtCodes(String[] codeStrings) 
+  private static CharHandler parseUnprintableExtCodes(String[] codeStrings)
   {
     if(codeStrings.length != 1) {
       throw new IllegalStateException("Unexpected code strings " +
@@ -415,7 +442,7 @@ public class GeneralLegacyIndexCodes {
    * Returns a InternationalExtCharHandler parsed from the given index code
    * strings.
    */
-  private static CharHandler parseInternationalExtCodes(String[] codeStrings) 
+  private static CharHandler parseInternationalExtCodes(String[] codeStrings)
   {
     if(codeStrings.length != 3) {
       throw new IllegalStateException("Unexpected code strings " +
@@ -429,6 +456,18 @@ public class GeneralLegacyIndexCodes {
                                            crazyFlag);
   }
 
+  /**
+   * Returns a SignificantCharHandler parsed from the given index code strings.
+   */
+  private static CharHandler parseSignificantCodes(String[] codeStrings)
+  {
+    if(codeStrings.length != 1) {
+      throw new IllegalStateException("Unexpected code strings " +
+                                      Arrays.asList(codeStrings));
+    }
+    return new SignificantCharHandler(codesToBytes(codeStrings[0], true));
+  }
+
   /**
    * Converts a string of hex encoded bytes to a byte[], optionally throwing
    * an exception if no codes are given.
@@ -481,10 +520,10 @@ public class GeneralLegacyIndexCodes {
       str = str.substring(0, MAX_TEXT_INDEX_CHAR_LENGTH);
     }
 
-    // record pprevious entry length so we can do any post-processing
+    // record previous entry length so we can do any post-processing
     // necessary for this entry (handling descending)
     int prevLength = bout.getLength();
-    
+
     // now, convert each character to a "code" of one or more bytes
     ExtraCodesStream extraCodes = null;
     ByteStream unprintableCodes = null;
@@ -526,12 +565,12 @@ public class GeneralLegacyIndexCodes {
         if(unprintableCodes == null) {
           unprintableCodes = new ByteStream();
         }
-          
+
         // keep track of the unprintable codes for later
         writeUnprintableCodes(curCharOffset, bytes, unprintableCodes,
                               extraCodes);
       }
-      
+
       byte crazyFlag = ch.getCrazyFlag();
       if(crazyFlag != 0) {
         if(crazyCodes == null) {
@@ -580,7 +619,7 @@ public class GeneralLegacyIndexCodes {
 
           // write another end flag
           bout.write(END_TEXT);
-        
+
           unprintableCodes.writeTo(bout);
         }
       }
@@ -592,14 +631,14 @@ public class GeneralLegacyIndexCodes {
       // we actually write the end byte before flipping the bytes, and write
       // another one after flipping
       bout.write(END_EXTRA_TEXT);
-      
+
       // flip the bytes that we have written thus far for this text value
-      IndexData.flipBytes(bout.getBytes(), prevLength, 
+      IndexData.flipBytes(bout.getBytes(), prevLength,
                           (bout.getLength() - prevLength));
     }
 
     // write end extra text
-    bout.write(END_EXTRA_TEXT);    
+    bout.write(END_EXTRA_TEXT);
   }
 
   /**
@@ -619,7 +658,7 @@ public class GeneralLegacyIndexCodes {
     }
 
     if(bytes != null) {
-      
+
       // write the actual extra codes and update the number of chars
       extraCodes.write(bytes);
       extraCodes.incrementNumChars(1);
@@ -696,7 +735,7 @@ public class GeneralLegacyIndexCodes {
     // write offset as big-endian short
     unprintableCodes.write((offset >> 8) & 0xFF);
     unprintableCodes.write(offset & 0xFF);
-          
+
     unprintableCodes.write(UNPRINTABLE_MIDFIX);
     unprintableCodes.write(bytes);
   }
@@ -748,7 +787,7 @@ public class GeneralLegacyIndexCodes {
   private static final class ExtraCodesStream extends ByteStream
   {
     private int _numChars;
-    private int _unprintablePrefixLen; 
+    private int _unprintablePrefixLen;
 
     private ExtraCodesStream(int length) {
       super(length);
@@ -757,7 +796,7 @@ public class GeneralLegacyIndexCodes {
     public int getNumChars() {
       return _numChars;
     }
-    
+
     public void incrementNumChars(int inc) {
       _numChars += inc;
     }

src/main/java/com/healthmarketscience/jackcess/impl/IndexCursorImpl.java

@@ -42,7 +42,7 @@ import org.apache.commons.logging.LogFactory;
  */
 public class IndexCursorImpl extends CursorImpl implements IndexCursor
 {
-  private static final Log LOG = LogFactory.getLog(IndexCursorImpl.class);  
+  private static final Log LOG = LogFactory.getLog(IndexCursorImpl.class);
 
   /** IndexDirHandler for forward traversal */
   private final IndexDirHandler _forwardDirHandler =
@@ -68,7 +68,7 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
     _index.initialize();
     _entryCursor = entryCursor;
   }
-  
+
   /**
    * Creates an indexed cursor for the given table, narrowed to the given
    * range.
@@ -76,7 +76,7 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
    * Note, index based table traversal may not include all rows, as certain
    * types of indexes do not include all entries (namely, some indexes ignore
    * null entries, see {@link Index#shouldIgnoreNulls}).
-   * 
+   *
    * @param table the table over which this cursor will traverse
    * @param index index for the table which will define traversal order as
    *              well as enhance certain lookups
@@ -98,14 +98,9 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
       throw new IllegalArgumentException(
           "Given index is not for given table: " + index + ", " + table);
     }
-    if(!table.getFormat().INDEXES_SUPPORTED) {
-      throw new IllegalArgumentException(
-          "JetFormat " + table.getFormat() + 
-          " does not currently support index lookups");
-    }
     if(index.getIndexData().getUnsupportedReason() != null) {
       throw new IllegalArgumentException(
-          "Given index " + index + 
+          "Given index " + index +
           " is not usable for indexed lookups due to " +
           index.getIndexData().getUnsupportedReason());
     }
@@ -115,7 +110,7 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
     // init the column matcher appropriately for the index type
     cursor.setColumnMatcher(null);
     return cursor;
-  }  
+  }
 
   private Set<String> getIndexEntryPattern()
   {
@@ -135,7 +130,7 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
   }
 
   @Override
-  public Row findRowByEntry(Object... entryValues) 
+  public Row findRowByEntry(Object... entryValues)
     throws IOException
   {
     if(findFirstRowByEntry(entryValues)) {
@@ -143,16 +138,16 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
     }
     return null;
   }
-  
+
   @Override
-  public boolean findFirstRowByEntry(Object... entryValues) 
-    throws IOException 
+  public boolean findFirstRowByEntry(Object... entryValues)
+    throws IOException
   {
     PositionImpl curPos = _curPos;
     PositionImpl prevPos = _prevPos;
     boolean found = false;
     try {
-      found = findFirstRowByEntryImpl(toRowValues(entryValues), true, 
+      found = findFirstRowByEntryImpl(toRowValues(entryValues), true,
                                       _columnMatcher);
       return found;
     } finally {
@@ -167,8 +162,8 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
   }
 
   @Override
-  public void findClosestRowByEntry(Object... entryValues) 
-    throws IOException 
+  public void findClosestRowByEntry(Object... entryValues)
+    throws IOException
   {
     PositionImpl curPos = _curPos;
     PositionImpl prevPos = _prevPos;
@@ -189,8 +184,8 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
   }
 
   @Override
-  public boolean currentRowMatchesEntry(Object... entryValues) 
-    throws IOException 
+  public boolean currentRowMatchesEntry(Object... entryValues)
+    throws IOException
   {
     return currentRowMatchesEntryImpl(toRowValues(entryValues), _columnMatcher);
   }
@@ -205,17 +200,17 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
                              toRowValues(iterBuilder.getEntryValues()),
                              iterBuilder.getColumnMatcher());
   }
-  
+
   @Override
   protected IndexDirHandler getDirHandler(boolean moveForward) {
     return (moveForward ? _forwardDirHandler : _reverseDirHandler);
   }
-    
+
   @Override
   protected boolean isUpToDate() {
     return(super.isUpToDate() && _entryCursor.isUpToDate());
   }
-    
+
   @Override
   protected void reset(boolean moveForward) {
     _entryCursor.reset(moveForward);
@@ -259,7 +254,7 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
       return super.findAnotherRowImpl(columnPattern, valuePattern, moveForward,
                                       columnMatcher, rowValues);
     }
-      
+
     // sweet, we can use our index
     if(!findPotentialRow(rowValues, true)) {
       return false;
@@ -274,14 +269,14 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
    * match the given values.  Caller manages save/restore on failure.
    *
    * @param rowValues the column values built from the index column values
-   * @param requireMatch whether or not an exact match is found
+   * @param requireMatch whether or not an exact match is desired
    * @return {@code true} if a valid row was found with the given values,
    *         {@code false} if no row was found
    */
   protected boolean findFirstRowByEntryImpl(Object[] rowValues,
                                             boolean requireMatch,
-                                            ColumnMatcher columnMatcher) 
-    throws IOException 
+                                            ColumnMatcher columnMatcher)
+    throws IOException
   {
     if(!findPotentialRow(rowValues, requireMatch)) {
       return false;
@@ -317,7 +312,7 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
     // determine if the pattern columns exactly match the index columns
     boolean exactColumnMatch = rowPattern.keySet().equals(
         getIndexEntryPattern());
-    
+
     // there may be multiple rows which fit the pattern subset used by
     // the index, so we need to keep checking until our index values no
     // longer match
@@ -337,12 +332,12 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
       }
 
     } while(moveToAnotherRow(moveForward));
-        
+
     // none of the potential rows matched
     return false;
   }
 
-  private boolean currentRowMatchesEntryImpl(Object[] rowValues, 
+  private boolean currentRowMatchesEntryImpl(Object[] rowValues,
                                              ColumnMatcher columnMatcher)
     throws IOException
   {
@@ -353,7 +348,7 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
 
       Object patValue = rowValues[col.getColumnIndex()];
 
-      if((patValue == IndexData.MIN_VALUE) || 
+      if((patValue == IndexData.MIN_VALUE) ||
          (patValue == IndexData.MAX_VALUE)) {
         // all remaining entry values are "special" (used for partial lookups)
         return true;
@@ -366,9 +361,9 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
       }
     }
 
-    return true;    
+    return true;
   }
-  
+
   private boolean findPotentialRow(Object[] rowValues, boolean requireMatch)
     throws IOException
   {
@@ -400,8 +395,8 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
   }
 
   @Override
-  protected boolean keepSearching(ColumnMatcher columnMatcher, 
-                                  Object searchInfo) 
+  protected boolean keepSearching(ColumnMatcher columnMatcher,
+                                  Object searchInfo)
     throws IOException
   {
     if(searchInfo instanceof Object[]) {
@@ -420,7 +415,7 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
     return _entryCursor.getIndexData().constructPartialIndexRowFromEntry(
         IndexData.MIN_VALUE, entryValues);
   }
-  
+
   @Override
   protected PositionImpl findAnotherPosition(
       RowState rowState, PositionImpl curPos, boolean moveForward)
@@ -451,7 +446,7 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
     public abstract IndexData.Entry getAnotherEntry()
       throws IOException;
   }
-    
+
   /**
    * Handles moving the table index cursor forward.
    */
@@ -469,7 +464,7 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
       return _entryCursor.getNextEntry();
     }
   }
-    
+
   /**
    * Handles moving the table index cursor backward.
    */
@@ -486,15 +481,15 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
     public IndexData.Entry getAnotherEntry() throws IOException {
       return _entryCursor.getPreviousEntry();
     }
-  }    
-    
+  }
+
   /**
    * Value object which maintains the current position of an IndexCursor.
    */
   private static final class IndexPosition extends PositionImpl
   {
     private final IndexData.Entry _entry;
-    
+
     private IndexPosition(IndexData.Entry entry) {
       _entry = entry;
     }
@@ -503,11 +498,11 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
     public RowIdImpl getRowId() {
       return getEntry().getRowId();
     }
-    
+
     public IndexData.Entry getEntry() {
       return _entry;
     }
-    
+
     @Override
     protected boolean equalsImpl(Object o) {
       return getEntry().equals(((IndexPosition)o).getEntry());
@@ -525,7 +520,7 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
   private final class EntryIterator extends BaseIterator
   {
     private final Object[] _rowValues;
-    
+
     private EntryIterator(Collection<String> columnNames, Object[] rowValues,
                           ColumnMatcher columnMatcher)
     {
@@ -541,9 +536,9 @@ public class IndexCursorImpl extends CursorImpl implements IndexCursor
 
     @Override
     protected boolean findNext() throws IOException {
-      return (moveToNextRow() && 
+      return (moveToNextRow() &&
               currentRowMatchesEntryImpl(_rowValues, _colMatcher));
-    }    
+    }
   }
 
 }

src/main/java/com/healthmarketscience/jackcess/impl/IndexData.java

@@ -1515,11 +1515,14 @@ public class IndexData {
     case TEXT:
     case MEMO:
       ColumnImpl.SortOrder sortOrder = col.getTextSortOrder();
+      if(ColumnImpl.GENERAL_SORT_ORDER.equals(sortOrder)) {
+        return new GenTextColumnDescriptor(col, flags);
+      }
       if(ColumnImpl.GENERAL_LEGACY_SORT_ORDER.equals(sortOrder)) {
         return new GenLegTextColumnDescriptor(col, flags);
       }
-      if(ColumnImpl.GENERAL_SORT_ORDER.equals(sortOrder)) {
-        return new GenTextColumnDescriptor(col, flags);
+      if(ColumnImpl.GENERAL_97_SORT_ORDER.equals(sortOrder)) {
+        return new Gen97TextColumnDescriptor(col, flags);
       }
       // unsupported sort order
       setUnsupportedReason("unsupported collating sort order " + sortOrder +
@@ -1914,6 +1917,27 @@ public class IndexData {
     }
   }
 
+  /**
+   * ColumnDescriptor for "general 97" sort order text based columns.
+   */
+  private static final class Gen97TextColumnDescriptor
+    extends ColumnDescriptor
+  {
+    private Gen97TextColumnDescriptor(ColumnImpl column, byte flags)
+      throws IOException
+    {
+      super(column, flags);
+    }
+
+    @Override
+    protected void writeNonNullValue(Object value, ByteStream bout)
+      throws IOException
+    {
+      General97IndexCodes.GEN_97_INSTANCE.writeNonNullIndexTextValue(
+          value, bout, isAscending());
+    }
+  }
+
   /**
    * ColumnDescriptor for guid columns.
    */

src/main/java/com/healthmarketscience/jackcess/impl/JetFormat.java

@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -544,7 +545,7 @@ public abstract class JetFormat {
     protected boolean defineReadOnly() { return true; }
 
     @Override
-    protected boolean defineIndexesSupported() { return false; }
+    protected boolean defineIndexesSupported() { return true; }
 
     @Override
     protected CodecType defineCodecType() {
@@ -740,7 +741,7 @@ public abstract class JetFormat {
 
     @Override
     protected ColumnImpl.SortOrder defineDefaultSortOrder() {
-      return ColumnImpl.GENERAL_LEGACY_SORT_ORDER;
+      return ColumnImpl.GENERAL_97_SORT_ORDER;
     }
 
     @Override
@@ -973,7 +974,7 @@ public abstract class JetFormat {
     protected boolean defineLegacyNumericIndexes() { return true; }
 
     @Override
-    protected Charset defineCharset() { return Charset.forName("UTF-16LE"); }
+    protected Charset defineCharset() { return StandardCharsets.UTF_16LE; }
 
     @Override
     protected ColumnImpl.SortOrder defineDefaultSortOrder() {

src/main/java/com/healthmarketscience/jackcess/impl/OleUtil.java

@@ -24,6 +24,7 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.sql.Blob;
 import java.sql.SQLException;
 import java.sql.SQLFeatureNotSupportedException;
@@ -43,7 +44,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder;
  * @author James Ahlborn
  * @usage _advanced_class_
  */
-public class OleUtil 
+public class OleUtil
 {
   /**
    * Interface used to allow optional inclusion of the poi library for working
@@ -57,9 +58,9 @@ public class OleUtil
   }
 
   private static final int PACKAGE_SIGNATURE = 0x1C15;
-  private static final Charset OLE_CHARSET = Charset.forName("US-ASCII");
-  private static final Charset OLE_UTF_CHARSET = Charset.forName("UTF-16LE");
-  private static final byte[] COMPOUND_STORAGE_SIGNATURE = 
+  private static final Charset OLE_CHARSET = StandardCharsets.US_ASCII;
+  private static final Charset OLE_UTF_CHARSET = StandardCharsets.UTF_16LE;
+  private static final byte[] COMPOUND_STORAGE_SIGNATURE =
     {(byte)0xd0,(byte)0xcf,(byte)0x11,(byte)0xe0,
      (byte)0xa1,(byte)0xb1,(byte)0x1a,(byte)0xe1};
   private static final String SIMPLE_PACKAGE_TYPE = "Package";
@@ -79,7 +80,7 @@ public class OleUtil
   };
 
   // regex pattern which matches all the crazy extra stuff in unicode
-  private static final Pattern UNICODE_ACCENT_PATTERN = 
+  private static final Pattern UNICODE_ACCENT_PATTERN =
     Pattern.compile("[\\p{InCombiningDiacriticalMarks}\\p{IsLm}\\p{IsSk}]+");
 
   private static final CompoundPackageFactory COMPOUND_FACTORY;
@@ -111,13 +112,13 @@ public class OleUtil
     throws IOException
   {
     try {
-      
+
       if(!WRITEABLE_TYPES.contains(oleBuilder.getType())) {
         throw new IllegalArgumentException(
             "Cannot currently create ole values of type " +
             oleBuilder.getType());
       }
-      
+
       long contentLen = oleBuilder.getContentLength();
       byte[] contentBytes = oleBuilder.getBytes();
       InputStream contentStream = oleBuilder.getStream();
@@ -132,12 +133,12 @@ public class OleUtil
         contentBytes = getZeroTermStrBytes(oleBuilder.getFilePath());
         contentLen = contentBytes.length;
         break;
-        
+
       case SIMPLE_PACKAGE:
         packageStreamHeader = writePackageStreamHeader(oleBuilder);
         packageStreamFooter = writePackageStreamFooter(oleBuilder);
         break;
-        
+
       case OTHER:
         // nothing more to do
         break;
@@ -148,19 +149,19 @@ public class OleUtil
       long payloadLen = packageStreamHeader.length + packageStreamFooter.length +
         contentLen;
       byte[] packageHeader = writePackageHeader(oleBuilder, payloadLen);
-            
+
       long totalOleLen = packageHeader.length + PACKAGE_FOOTER.length +
         payloadLen;
       if(totalOleLen > DataType.OLE.getMaxSize()) {
         throw new IllegalArgumentException("Content size of " + totalOleLen +
                                            " is too large for ole column");
       }
-      
+
       byte[] oleBytes = new byte[(int)totalOleLen];
       ByteBuffer bb = PageChannel.wrap(oleBytes);
       bb.put(packageHeader);
       bb.put(packageStreamHeader);
-      
+
       if(contentLen > 0L) {
         if(contentBytes != null) {
           bb.put(contentBytes);
@@ -175,9 +176,9 @@ public class OleUtil
 
       bb.put(packageStreamFooter);
       bb.put(PACKAGE_FOOTER);
-    
+
       return parseBlob(oleBytes);
-      
+
     } finally {
       ByteUtil.closeQuietly(oleBuilder.getStream());
     }
@@ -196,13 +197,13 @@ public class OleUtil
     }
     byte[] classNameBytes = getZeroTermStrBytes(className);
     byte[] typeNameBytes = getZeroTermStrBytes(typeName);
-    
+
     int packageHeaderLen = 20 + prettyNameBytes.length + classNameBytes.length;
 
     int oleHeaderLen = 24 + typeNameBytes.length;
 
     byte[] headerBytes = new byte[packageHeaderLen + oleHeaderLen];
-    
+
     ByteBuffer bb = PageChannel.wrap(headerBytes);
 
     // write outer package header
@@ -225,7 +226,7 @@ public class OleUtil
     bb.put(typeNameBytes);
     bb.putLong(0L);
     bb.putInt((int)contentLen);
-    
+
     return headerBytes;
   }
 
@@ -239,7 +240,7 @@ public class OleUtil
     if(oleBuilder.getType() == ContentType.SIMPLE_PACKAGE) {
 
       headerLen += 8 + filePathBytes.length;
-      
+
     } else {
 
       headerLen += 2;
@@ -260,7 +261,7 @@ public class OleUtil
       bb.putInt(PS_LINKED_FILE);
       bb.putShort((short)LINK_HEADER);
     }
-    
+
     return headerBytes;
   }
 
@@ -280,20 +281,20 @@ public class OleUtil
     bb.putInt(fileNameBytes.length/2);
     bb.put(fileNameBytes);
     bb.putInt(filePathBytes.length/2);
-    bb.put(filePathBytes);    
+    bb.put(filePathBytes);
 
     return footerBytes;
   }
-  
+
   /**
    * creates the appropriate ContentImpl for the given blob.
    */
-  private static ContentImpl parseContent(OleBlobImpl blob) 
-    throws IOException 
+  private static ContentImpl parseContent(OleBlobImpl blob)
+    throws IOException
   {
     ByteBuffer bb = PageChannel.wrap(blob.getBytes());
 
-    if((bb.remaining() < 2) || (bb.getShort() != PACKAGE_SIGNATURE)) {  
+    if((bb.remaining() < 2) || (bb.getShort() != PACKAGE_SIGNATURE)) {
       return new UnknownContentImpl(blob);
     }
 
@@ -303,7 +304,7 @@ public class OleUtil
     int prettyNameLen = bb.getShort();
     int classNameLen = bb.getShort();
     int prettyNameOff = bb.getShort();
-    int classNameOff = bb.getShort();       
+    int classNameOff = bb.getShort();
     /* int objSize = */ bb.getInt();
     String prettyName = readStr(bb, prettyNameOff, prettyNameLen);
     String className = readStr(bb, classNameOff, classNameLen);
@@ -337,7 +338,7 @@ public class OleUtil
       return COMPOUND_FACTORY.createCompoundPackageContent(
           blob, prettyName, className, typeName, bb, dataBlockLen);
     }
-    
+
     // this is either some other "special" (as yet unhandled) format, or it is
     // simply an embedded file (or it is compound data and poi isn't available)
     return new OtherContentImpl(blob, prettyName, className,
@@ -349,9 +350,9 @@ public class OleUtil
       ByteBuffer blobBb, int dataBlockLen) {
 
     int dataBlockPos = blobBb.position();
-    ByteBuffer bb = PageChannel.narrowBuffer(blobBb, dataBlockPos, 
+    ByteBuffer bb = PageChannel.narrowBuffer(blobBb, dataBlockPos,
                                              dataBlockPos + dataBlockLen);
-    
+
     int packageSig = bb.getShort();
     if(packageSig != PACKAGE_STREAM_SIGNATURE) {
       return new OtherContentImpl(blob, prettyName, className,
@@ -405,19 +406,19 @@ public class OleUtil
       return new SimplePackageContentImpl(
           blob, prettyName, className, typeName, dataPos, dataLen,
           fileName, filePath, localFilePath);
-    } 
+    }
 
     if(packageType == PS_LINKED_FILE) {
-      
+
       bb.getShort(); //unknown
       String linkStr = readZeroTermStr(bb);
 
-      return new LinkContentImpl(blob, prettyName, className, typeName, 
+      return new LinkContentImpl(blob, prettyName, className, typeName,
                                  fileName, linkStr, filePath);
     }
 
     return new OtherContentImpl(blob, prettyName, className,
-                                typeName, dataBlockPos, dataBlockLen);      
+                                typeName, dataBlockPos, dataBlockLen);
   }
 
   private static String readStr(ByteBuffer bb, int off, int len) {
@@ -436,7 +437,7 @@ public class OleUtil
     return readStr(bb, off, len);
   }
 
-  private static String readStr(ByteBuffer bb, int off, int len, 
+  private static String readStr(ByteBuffer bb, int off, int len,
                                 Charset charset) {
     String str = new String(bb.array(), off, len, charset);
     bb.position(off + len);
@@ -490,8 +491,8 @@ public class OleUtil
     }
 
     @Override
-    public InputStream getBinaryStream(long pos, long len) 
-      throws SQLException 
+    public InputStream getBinaryStream(long pos, long len)
+      throws SQLException
     {
       return new ByteArrayInputStream(_bytes, fromJdbcOffset(pos), (int)len);
     }
@@ -515,11 +516,11 @@ public class OleUtil
 
     @Override
     public long position(byte[] pattern, long start) throws SQLException {
-      int pos = ByteUtil.findRange(PageChannel.wrap(_bytes), 
+      int pos = ByteUtil.findRange(PageChannel.wrap(_bytes),
                                    fromJdbcOffset(start), pattern);
       return((pos >= 0) ? toJdbcOffset(pos) : pos);
     }
-    
+
     @Override
     public long position(Blob pattern, long start) throws SQLException {
       return position(pattern.getBytes(1L, (int)pattern.length()), start);
@@ -529,23 +530,23 @@ public class OleUtil
     public OutputStream setBinaryStream(long position) throws SQLException {
       throw new SQLFeatureNotSupportedException();
     }
-    
+
     @Override
     public void truncate(long len) throws SQLException {
       throw new SQLFeatureNotSupportedException();
     }
-    
+
     @Override
     public int setBytes(long pos, byte[] bytes) throws SQLException {
       throw new SQLFeatureNotSupportedException();
     }
-    
+
     @Override
     public int setBytes(long pos, byte[] bytes, int offset, int lesn)
       throws SQLException {
       throw new SQLFeatureNotSupportedException();
     }
-    
+
     @Override
     public void free() {
       close();
@@ -560,11 +561,11 @@ public class OleUtil
 
     private static int toJdbcOffset(int off) {
       return off + 1;
-    } 
+    }
 
     private static int fromJdbcOffset(long off) {
       return (int)off - 1;
-    } 
+    }
 
     @Override
     public String toString() {
@@ -595,7 +596,7 @@ public class OleUtil
     protected byte[] getBytes() throws IOException {
       return getBlob().getBytes();
     }
-    
+
     @Override
     public void close() {
       // base does nothing
@@ -604,7 +605,7 @@ public class OleUtil
     protected ToStringBuilder toString(ToStringBuilder sb) {
       sb.append("type", getType());
       return sb;
-    } 
+    }
   }
 
   static abstract class EmbeddedContentImpl extends ContentImpl
@@ -613,7 +614,7 @@ public class OleUtil
     private final int _position;
     private final int _length;
 
-    protected EmbeddedContentImpl(OleBlobImpl blob, int position, int length) 
+    protected EmbeddedContentImpl(OleBlobImpl blob, int position, int length)
     {
       super(blob);
       _position = position;
@@ -642,10 +643,10 @@ public class OleUtil
         sb.append("content", ByteBuffer.wrap(_blob._bytes, _position, _length));
       }
       return sb;
-    } 
+    }
   }
 
-  static abstract class EmbeddedPackageContentImpl 
+  static abstract class EmbeddedPackageContentImpl
     extends EmbeddedContentImpl
     implements PackageContent
   {
@@ -685,10 +686,10 @@ public class OleUtil
         .append("typeName", _typeName);
       super.toString(sb);
       return sb;
-    } 
+    }
   }
 
-  private static final class LinkContentImpl 
+  private static final class LinkContentImpl
     extends EmbeddedPackageContentImpl
     implements LinkContent
   {
@@ -698,13 +699,13 @@ public class OleUtil
 
     private LinkContentImpl(OleBlobImpl blob, String prettyName,
                             String className, String typeName,
-                            String fileName, String linkPath, 
-                            String filePath) 
+                            String fileName, String linkPath,
+                            String filePath)
     {
       super(blob, prettyName, className, typeName, -1, -1);
       _fileName = fileName;
       _linkPath = linkPath;
-      _filePath = filePath;      
+      _filePath = filePath;
     }
 
     @Override
@@ -742,7 +743,7 @@ public class OleUtil
     }
   }
 
-  private static final class SimplePackageContentImpl 
+  private static final class SimplePackageContentImpl
     extends EmbeddedPackageContentImpl
     implements SimplePackageContent
   {
@@ -754,11 +755,11 @@ public class OleUtil
                                      String className, String typeName,
                                      int position, int length,
                                      String fileName, String filePath,
-                                     String localFilePath) 
+                                     String localFilePath)
     {
       super(blob, prettyName, className, typeName, position, length);
       _fileName = fileName;
-      _filePath = filePath;      
+      _filePath = filePath;
       _localFilePath = localFilePath;
     }
 
@@ -792,16 +793,16 @@ public class OleUtil
     }
   }
 
-  private static final class OtherContentImpl 
+  private static final class OtherContentImpl
     extends EmbeddedPackageContentImpl
     implements OtherContent
   {
     private OtherContentImpl(
         OleBlobImpl blob, String prettyName, String className,
-        String typeName, int position, int length) 
+        String typeName, int position, int length)
     {
       super(blob, prettyName, className, typeName, position, length);
-    }        
+    }
 
     @Override
     public ContentType getType() {
@@ -833,5 +834,5 @@ public class OleUtil
         .toString();
     }
   }
-  
+
 }

src/main/java/com/healthmarketscience/jackcess/impl/PageChannel.java

@@ -29,22 +29,22 @@ import java.nio.channels.FileChannel;
  * @author Tim McCune
  */
 public class PageChannel implements Channel, Flushable {
-  
+
   static final int INVALID_PAGE_NUMBER = -1;
 
   static final ByteOrder DEFAULT_BYTE_ORDER = ByteOrder.LITTLE_ENDIAN;
-  
+
   /** invalid page header, used when deallocating old pages.  data pages
       generally have 4 interesting bytes at the beginning which we want to
       reset. */
   private static final byte[] INVALID_PAGE_BYTE_HEADER =
     new byte[]{PageTypes.INVALID, (byte)0, (byte)0, (byte)0};
-  
+
   /** Global usage map always lives on page 1 */
   static final int PAGE_GLOBAL_USAGE_MAP = 1;
   /** Global usage map always lives at row 0 */
   static final int ROW_GLOBAL_USAGE_MAP = 0;
-  
+
   /** Channel containing the database */
   private final FileChannel _channel;
   /** whether or not the _channel should be closed by this class */
@@ -67,7 +67,7 @@ public class PageChannel implements Channel, Flushable {
   private TempPageHolder _fullPageEncodeBufferH;
   private TempBufferHolder _tempDecodeBufferH;
   private int _writeCount;
-  
+
   /**
    * Only used by unit tests
    */
@@ -117,7 +117,7 @@ public class PageChannel implements Channel, Flushable {
     _globalUsageMap = UsageMap.read(database, PAGE_GLOBAL_USAGE_MAP,
                                     ROW_GLOBAL_USAGE_MAP, true);
   }
-  
+
   public JetFormat getFormat() {
     return _format;
   }
@@ -177,7 +177,7 @@ public class PageChannel implements Channel, Flushable {
       throw new IllegalStateException("No write operation in progress");
     }
   }
-  
+
   /**
    * Returns the next page number based on the given file size.
    */
@@ -191,7 +191,7 @@ public class PageChannel implements Channel, Flushable {
   private long getPageOffset(int pageNumber) {
     return((long) pageNumber * (long) getFormat().PAGE_SIZE);
   }
-  
+
   /**
    * Validates that the given pageNumber is valid for this database.
    */
@@ -203,7 +203,7 @@ public class PageChannel implements Channel, Flushable {
       throw new IllegalStateException("invalid page number " + pageNumber);
     }
   }
-  
+
   /**
    * @param buffer Buffer to read the page into
    * @param pageNumber Number of the page to read in (starting at 0)
@@ -211,11 +211,16 @@ public class PageChannel implements Channel, Flushable {
   public void readPage(ByteBuffer buffer, int pageNumber)
     throws IOException
   {
+    if(pageNumber == 0) {
+      readRootPage(buffer);
+      return;
+    }
+
     validatePageNumber(pageNumber);
 
     ByteBuffer inPage = buffer;
     ByteBuffer outPage = buffer;
-    if((pageNumber != 0) && !_codecHandler.canDecodeInline()) {
+    if(!_codecHandler.canDecodeInline()) {
       inPage = _tempDecodeBufferH.getPageBuffer(this);
       outPage.clear();
     }
@@ -230,14 +235,30 @@ public class PageChannel implements Channel, Flushable {
                             pageNumber + ", only read " + bytesRead);
     }
 
-    if(pageNumber == 0) {
-      // de-mask header (note, page 0 never has additional encoding)
-      applyHeaderMask(buffer);
-    } else {
-      _codecHandler.decodePage(inPage, outPage, pageNumber);
+    _codecHandler.decodePage(inPage, outPage, pageNumber);
+  }
+
+  /**
+   * @param buffer Buffer to read the root page into
+   */
+  public void readRootPage(ByteBuffer buffer)
+    throws IOException
+  {
+    // special method for reading root page, can be done before PageChannel is
+    // fully initialized
+    buffer.clear();
+    int bytesRead = _channel.read(buffer, 0L);
+    buffer.flip();
+    if(bytesRead != getFormat().PAGE_SIZE) {
+      throw new IOException("Failed attempting to read " +
+                            getFormat().PAGE_SIZE + " bytes from page " +
+                            0 + ", only read " + bytesRead);
     }
+
+    // de-mask header (note, page 0 never has additional encoding)
+    applyHeaderMask(buffer);
   }
-  
+
   /**
    * Write a page to disk
    * @param page Page to write
@@ -246,7 +267,7 @@ public class PageChannel implements Channel, Flushable {
   public void writePage(ByteBuffer page, int pageNumber) throws IOException {
     writePage(page, pageNumber, 0);
   }
-  
+
   /**
    * Write a page (or part of a page) to disk
    * @param page Page to write
@@ -259,7 +280,7 @@ public class PageChannel implements Channel, Flushable {
   {
     assertWriting();
     validatePageNumber(pageNumber);
-    
+
     page.rewind().position(pageOffset);
 
     int writeLen = page.remaining();
@@ -267,7 +288,7 @@ public class PageChannel implements Channel, Flushable {
       throw new IllegalArgumentException(
           "Page buffer is too large, size " + (writeLen + pageOffset));
     }
-    
+
     ByteBuffer encodedPage = page;
     if(pageNumber == 0) {
       // re-mask header
@@ -315,7 +336,7 @@ public class PageChannel implements Channel, Flushable {
       }
     }
   }
-  
+
   /**
    * Allocates a new page in the database.  Data in the page is undefined
    * until it is written in a call to {@link #writePage(ByteBuffer,int)}.
@@ -334,9 +355,9 @@ public class PageChannel implements Channel, Flushable {
                             " is not multiple of page size " +
                             getFormat().PAGE_SIZE);
     }
-    
+
     _forceBytes.rewind();
-    
+
     // push the buffer to the end of the page, so that a full page's worth of
     // data is written
     int pageOffset = (getFormat().PAGE_SIZE - _forceBytes.remaining());
@@ -358,15 +379,15 @@ public class PageChannel implements Channel, Flushable {
     assertWriting();
 
     validatePageNumber(pageNumber);
-    
+
     // don't write the whole page, just wipe out the header (which should be
     // enough to let us know if we accidentally try to use an invalid page)
     _invalidPageBytes.rewind();
     _channel.write(_invalidPageBytes, getPageOffset(pageNumber));
-    
+
     _globalUsageMap.addPageNumber(pageNumber);  //force is done here
   }
-  
+
   /**
    * @return A newly-allocated buffer that can be passed to readPage
    */
@@ -381,19 +402,19 @@ public class PageChannel implements Channel, Flushable {
   public static ByteBuffer createBuffer(int size) {
     return createBuffer(size, DEFAULT_BYTE_ORDER);
   }
-  
+
   /**
    * @return A newly-allocated buffer of the given size and byte order
    */
   public static ByteBuffer createBuffer(int size, ByteOrder order) {
     return ByteBuffer.allocate(size).order(order);
   }
-  
+
   @Override
   public void flush() throws IOException {
     _channel.force(true);
   }
-  
+
   @Override
   public void close() throws IOException {
     flush();
@@ -401,7 +422,7 @@ public class PageChannel implements Channel, Flushable {
       _channel.close();
     }
   }
-  
+
   @Override
   public boolean isOpen() {
     return _channel.isOpen();
@@ -419,7 +440,7 @@ public class PageChannel implements Channel, Flushable {
         buffer.put(pos, b);
       }
   }
-  
+
   /**
    * @return a duplicate of the current buffer narrowed to the given position
    *         and limit.  mark will be set at the current position.

src/main/resources/com/healthmarketscience/jackcess/index_codes_gen_97.txt

@@ -0,0 +1,256 @@
+X
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S11
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S10
+S11
+S12
+S13
+S14
+S15
+S16
+S17
+S18
+S19
+S1A
+S1B
+S1C
+S1D
+S1E
+S1F
+S20
+S56
+S57
+S58
+S59
+S5A
+S5B
+S5C
+S5D
+S5E
+S5F
+S21
+S22
+S23
+S24
+S25
+S26
+S27
+G60
+S61
+G62
+S64
+G66
+S67
+S68
+S69
+G6A
+S6B
+S6C
+S6D
+S6F
+G70
+G72
+S73
+S74
+S75
+G76
+S77
+G78
+S7A
+S7B
+S7C
+G7D
+S7E
+S28
+S29
+S2A
+S2B
+S2C
+S2D
+G60
+S61
+G62
+S64
+G66
+S67
+S68
+S69
+G6A
+S6B
+S6C
+S6D
+S6F
+G70
+G72
+S73
+S74
+S75
+G76
+S77
+G78
+S7A
+S7B
+S7C
+G7D
+S7E
+S2E
+S2F
+S30
+S31
+S10
+S10
+S10
+S18
+S32
+S13
+S33
+S34
+S35
+S36
+S37
+I76,0A
+S18
+S7266
+S10
+S10
+S10
+S10
+S18
+S18
+S13
+S13
+S38
+S1E
+S1E
+S39
+S3A
+I76,0A
+S18
+S7266
+S10
+S10
+I7D,06
+S11
+S3B
+S3C
+S3D
+S3E
+S3F
+S40
+S41
+S42
+S43
+S44
+S13
+S45
+S1E
+S46
+S47
+S48
+S49
+S58
+S59
+S4A
+S4B
+S4C
+S4D
+S4E
+S57
+S4F
+S13
+S50
+S51
+S52
+S53
+I60,03
+I60,04
+I60,05
+I60,07
+I60,06
+I60,08
+S6066
+I62,09
+I66,03
+I66,04
+I66,05
+I66,06
+I6A,03
+I6A,04
+I6A,05
+I6A,06
+S65
+I70,07
+I72,03
+I72,04
+I72,05
+I72,07
+I72,06
+S54
+S81
+I78,03
+I78,04
+I78,05
+I78,06
+I7D,04
+S7F
+S7676
+I60,03
+I60,04
+I60,05
+I60,07
+I60,06
+I60,08
+S6066
+I62,09
+I66,03
+I66,04
+I66,05
+I66,06
+I6A,03
+I6A,04
+I6A,05
+I6A,06
+S65
+I70,07
+I72,03
+I72,04
+I72,05
+I72,07
+I72,06
+S55
+S81
+I78,03
+I78,04
+I78,05
+I78,06
+I7D,04
+S7F
+I7D,06

src/main/resources/com/healthmarketscience/jackcess/index_mappings_ext_gen_97.txt

@@ -0,0 +1,27 @@
+338,140
+339,156
+352,138
+353,154
+376,159
+381,142
+382,158
+402,131
+710,136
+732,152
+8211,150
+8212,151
+8216,145
+8217,146
+8218,130
+8220,147
+8221,148
+8222,132
+8224,134
+8225,135
+8226,149
+8230,133
+8240,137
+8249,139
+8250,155
+8364,128
+8482,153

src/test/java/com/healthmarketscience/jackcess/CursorTest.java

@@ -43,7 +43,7 @@ import static com.healthmarketscience.jackcess.TestUtil.*;
  */
 public class CursorTest extends TestCase {
 
-  static final List<TestDB> INDEX_CURSOR_DBS = 
+  static final List<TestDB> INDEX_CURSOR_DBS =
     TestDB.getSupportedForBasename(Basename.INDEX_CURSOR);
 
 
@@ -82,9 +82,9 @@ public class CursorTest extends TestCase {
     expectedRows.subList(0, startIdx).clear();
     return expectedRows;
   }
-  
-  private static Database createTestTable(final FileFormat fileFormat) 
-    throws Exception 
+
+  private static Database createTestTable(final FileFormat fileFormat)
+    throws Exception
   {
     Database db = createMem(fileFormat);
 
@@ -110,10 +110,10 @@ public class CursorTest extends TestCase {
       expectedRows.add(createExpectedRow("id", i, "value", "data" + i));
     }
     return expectedRows;
-  }  
+  }
 
-  static Database createTestIndexTable(final TestDB indexCursorDB) 
-    throws Exception 
+  static Database createTestIndexTable(final TestDB indexCursorDB)
+    throws Exception
   {
     Database db = openMem(indexCursorDB);
 
@@ -141,8 +141,8 @@ public class CursorTest extends TestCase {
     return expectedRows;
   }
 
-  private static Database createDupeTestTable(final FileFormat fileFormat) 
-    throws Exception 
+  private static Database createDupeTestTable(final FileFormat fileFormat)
+    throws Exception
   {
     Database db = createMem(fileFormat);
 
@@ -158,8 +158,8 @@ public class CursorTest extends TestCase {
     return db;
   }
 
-  static Database createDupeTestTable(final TestDB indexCursorDB) 
-    throws Exception 
+  static Database createDupeTestTable(final TestDB indexCursorDB)
+    throws Exception
   {
     Database db = openMem(indexCursorDB);
 
@@ -185,7 +185,7 @@ public class CursorTest extends TestCase {
       .setEndRowInclusive(type == 0)
       .toCursor();
   }
-  
+
   public void testRowId() throws Exception {
     // test special cases
     RowIdImpl rowId1 = new RowIdImpl(1, 2);
@@ -201,7 +201,7 @@ public class CursorTest extends TestCase {
                                RowIdImpl.LAST_ROW_ID),
                  sortedRowIds);
   }
-  
+
   public void testSimple() throws Exception {
     for (final FileFormat fileFormat : JetFormatTest.SUPPORTED_FILEFORMATS) {
       Database db = createTestTable(fileFormat);
@@ -266,7 +266,7 @@ public class CursorTest extends TestCase {
     assertTrue(cursor.moveToNextRow());
     assertTrue(cursor.moveToPreviousRow());
     assertEquals(expectedRow, cursor.getCurrentRow());
-    
+
     while(cursor.moveToNextRow()) {
       foundRows.add(cursor.getCurrentRow());
     }
@@ -285,7 +285,7 @@ public class CursorTest extends TestCase {
     assertTrue(cursor.isAfterLast());
 
     cursor.restoreSavepoint(savepoint);
-    assertEquals(expectedRow, cursor.getCurrentRow());    
+    assertEquals(expectedRow, cursor.getCurrentRow());
   }
 
   public void testMoveNoReset() throws Exception {
@@ -305,18 +305,18 @@ public class CursorTest extends TestCase {
   {
     List<Map<String, Object>> expectedRows = createTestTableData();
     List<Map<String, Object>> foundRows = new ArrayList<Map<String, Object>>();
-    
+
     Iterator<Row> iter = cursor.newIterable().iterator();
 
     for(int i = 0; i < 6; ++i) {
       foundRows.add(iter.next());
-    }    
+    }
 
     iter = cursor.newIterable().reset(false).reverse().iterator();
     iter.next();
     Map<String, Object> row = iter.next();
     assertEquals(expectedRows.get(4), row);
-    
+
     iter = cursor.newIterable().reset(false).iterator();
     iter.next();
     row = iter.next();
@@ -326,11 +326,11 @@ public class CursorTest extends TestCase {
     iter = cursor.newIterable().reset(false).iterator();
     for(int i = 6; i < 10; ++i) {
       foundRows.add(iter.next());
-    }    
+    }
 
     assertEquals(expectedRows, foundRows);
   }
-  
+
   public void testSearch() throws Exception {
     for (final FileFormat fileFormat : JetFormatTest.SUPPORTED_FILEFORMATS) {
       Database db = createTestTable(fileFormat);
@@ -372,7 +372,7 @@ public class CursorTest extends TestCase {
     assertEquals(createExpectedRow("id", 7,
                                    "value", "data" + 7),
                  cursor.getCurrentRow());
-    
+
     assertTrue(cursor.findFirstRow(table.getColumn("value"), "data" + 4));
     assertEquals(createExpectedRow("id", 4,
                                    "value", "data" + 4),
@@ -387,7 +387,7 @@ public class CursorTest extends TestCase {
                                      "id", outOfRangeValue,
                                      "value", "data" + outOfRangeValue)));
     }
-    
+
     assertEquals("data" + 5,
                  CursorBuilder.findValue(table,
                                   table.getColumn("value"),
@@ -441,9 +441,9 @@ public class CursorTest extends TestCase {
     for(Map<String, Object> row : cursor.newIterable().reverse()) {
       foundRows.add(row);
     }
-    assertEquals(expectedRows, foundRows);    
+    assertEquals(expectedRows, foundRows);
   }
-  
+
   public void testLiveAddition() throws Exception {
     for (final FileFormat fileFormat : JetFormatTest.SUPPORTED_FILEFORMATS) {
       Database db = createTestTable(fileFormat);
@@ -470,7 +470,7 @@ public class CursorTest extends TestCase {
     assertTrue(cursor2.isAfterLast());
 
     table.addRow(newRowNum, "data" + newRowNum);
-    Map<String,Object> expectedRow = 
+    Map<String,Object> expectedRow =
       createExpectedRow("id", newRowNum, "value", "data" + newRowNum);
 
     assertFalse(cursor1.isAfterLast());
@@ -484,7 +484,7 @@ public class CursorTest extends TestCase {
     assertTrue(cursor2.isAfterLast());
   }
 
-  
+
   public void testLiveDeletion() throws Exception {
     for (final FileFormat fileFormat : JetFormatTest.SUPPORTED_FILEFORMATS) {
       Database db = createTestTable(fileFormat);
@@ -524,7 +524,7 @@ public class CursorTest extends TestCase {
 
     assertEquals(expectedDeletedRow, cursor2.getCurrentRow());
     assertEquals(expectedDeletedRow, cursor3.getCurrentRow());
-    
+
     assertFalse(cursor2.isCurrentRowDeleted());
     assertFalse(cursor3.isCurrentRowDeleted());
 
@@ -536,7 +536,7 @@ public class CursorTest extends TestCase {
     assertEquals(expectedNextRow, cursor1.getNextRow());
     assertEquals(expectedNextRow, cursor2.getNextRow());
     assertEquals(expectedNextRow, cursor3.getNextRow());
-    
+
     assertEquals(expectedPrevRow, cursor3.getPreviousRow());
 
     assertTrue(cursor3.moveToNextRow());
@@ -606,7 +606,7 @@ public class CursorTest extends TestCase {
       db.close();
     }
   }
-  
+
   public void testReverseIndex() throws Exception {
     for (final TestDB indexCursorDB : INDEX_CURSOR_DBS) {
       Database db = createTestIndexTable(indexCursorDB);
@@ -684,7 +684,7 @@ public class CursorTest extends TestCase {
       }
     }
   }
-  
+
   public void testMoveIndexSubRange() throws Exception {
     for (final TestDB indexCursorDB : INDEX_CURSOR_DBS) {
       for(int i = 0; i < 2; ++i) {
@@ -704,7 +704,7 @@ public class CursorTest extends TestCase {
       }
     }
   }
-  
+
   public void testSearchIndexSubRange() throws Exception {
     for (final TestDB indexCursorDB : INDEX_CURSOR_DBS) {
       for(int i = 0; i < 2; ++i) {
@@ -759,7 +759,7 @@ public class CursorTest extends TestCase {
       }
     }
   }
-  
+
   public void testLiveDeletionIndexSubRange() throws Exception {
     for (final TestDB indexCursorDB : INDEX_CURSOR_DBS) {
       for(int i = 0; i < 2; ++i) {
@@ -873,7 +873,7 @@ public class CursorTest extends TestCase {
     rows = RowFilterTest.toList(
         cursor.newIterable().setMatchPattern(
             Collections.singletonMap("id", 8)));
-    
+
     expectedRows =
       createExpectedTable(
           createExpectedRow(
@@ -883,7 +883,7 @@ public class CursorTest extends TestCase {
     assertEquals(expectedRows, rows);
 
     for(Map<String,Object> row : table) {
-      
+
       List<Map<String,Object>> tmpRows = new ArrayList<Map<String,Object>>();
       for(Map<String,Object> tmpRow : cursor) {
         if(row.equals(tmpRow)) {
@@ -892,7 +892,7 @@ public class CursorTest extends TestCase {
       }
       expectedRows = tmpRows;
       assertFalse(expectedRows.isEmpty());
-      
+
       rows = RowFilterTest.toList(cursor.newIterable().setMatchPattern(row));
 
       assertEquals(expectedRows, rows);
@@ -944,9 +944,9 @@ public class CursorTest extends TestCase {
       db.close();
     }
   }
-  
+
   public void testColumnMatcher() throws Exception {
-    
+
 
     for (final FileFormat fileFormat : JetFormatTest.SUPPORTED_FILEFORMATS) {
       Database db = createTestTable(fileFormat);
@@ -958,7 +958,7 @@ public class CursorTest extends TestCase {
 
       Cursor cursor = CursorBuilder.createCursor(table);
       doTestMatcher(table, cursor, SimpleColumnMatcher.INSTANCE, false);
-      doTestMatcher(table, cursor, CaseInsensitiveColumnMatcher.INSTANCE, 
+      doTestMatcher(table, cursor, CaseInsensitiveColumnMatcher.INSTANCE,
                     true);
       db.close();
     }
@@ -979,8 +979,8 @@ public class CursorTest extends TestCase {
       assertFalse(columnMatcher.matches(table, "value", null, 13));
       assertTrue(columnMatcher.matches(table, "value", 13, 13));
   }
-  
-  private static void doTestMatcher(Table table, Cursor cursor, 
+
+  private static void doTestMatcher(Table table, Cursor cursor,
                                     ColumnMatcher columnMatcher,
                                     boolean caseInsensitive)
     throws Exception
@@ -1021,7 +1021,7 @@ public class CursorTest extends TestCase {
     assertEquals(createExpectedRow("id", 7,
                                    "value", "data" + 7),
                  cursor.getCurrentRow());
-    
+
     assertTrue(cursor.findFirstRow(createExpectedRow(
                                     "value", "Data" + 7)) == caseInsensitive);
     if(caseInsensitive) {
@@ -1029,13 +1029,13 @@ public class CursorTest extends TestCase {
                                      "value", "data" + 7),
                    cursor.getCurrentRow());
     }
-    
+
     assertTrue(cursor.findFirstRow(table.getColumn("value"), "data" + 4));
     assertEquals(createExpectedRow("id", 4,
                                    "value", "data" + 4),
                  cursor.getCurrentRow());
 
-    assertTrue(cursor.findFirstRow(table.getColumn("value"), "Data" + 4) 
+    assertTrue(cursor.findFirstRow(table.getColumn("value"), "Data" + 4)
                == caseInsensitive);
     if(caseInsensitive) {
       assertEquals(createExpectedRow("id", 4,
@@ -1067,8 +1067,8 @@ public class CursorTest extends TestCase {
   }
 
   public void testIndexCursor() throws Exception
-  { 
-    for (final TestDB testDB : TestDB.getSupportedForBasename(Basename.INDEX)) {
+  {
+    for (final TestDB testDB : TestDB.getSupportedForBasename(Basename.INDEX, true)) {
 
       Database db = openMem(testDB);
       Table t1 = db.getTable("Table1");
@@ -1081,7 +1081,7 @@ public class CursorTest extends TestCase {
 
       assertTrue(cursor.findFirstRowByEntry(1));
       assertEquals(1, cursor.getCurrentRow().get("id"));
-      
+
       cursor.findClosestRowByEntry(2);
       assertEquals(2, cursor.getCurrentRow().get("id"));
 
@@ -1090,11 +1090,11 @@ public class CursorTest extends TestCase {
       assertTrue(cursor.isAfterLast());
 
       db.close();
-    }    
+    }
   }
-  
+
   public void testIndexCursorDelete() throws Exception
-  { 
+  {
     for (final TestDB testDB : TestDB.getSupportedForBasename(Basename.INDEX)) {
 
       Database db = openMem(testDB);
@@ -1111,7 +1111,7 @@ public class CursorTest extends TestCase {
       assertEquals(Arrays.asList("baz11", "baz11-2"), expectedData);
 
       expectedData = new ArrayList<String>();
-      for(Iterator<? extends Row> iter = 
+      for(Iterator<? extends Row> iter =
             cursor.newEntryIterable(1).iterator();
           iter.hasNext(); ) {
         expectedData.add(iter.next().getString("data"));
@@ -1134,7 +1134,7 @@ public class CursorTest extends TestCase {
       }
 
       assertEquals(Arrays.asList("baz11", "baz11-2"), expectedData);
-      
+
       expectedData = new ArrayList<String>();
       for(Row row : cursor.newEntryIterable(1)
             .addColumnNames("data")) {
@@ -1142,13 +1142,13 @@ public class CursorTest extends TestCase {
       }
 
       assertTrue(expectedData.isEmpty());
-      
+
       db.close();
-    }    
+    }
   }
-  
+
   public void testCursorDelete() throws Exception
-  { 
+  {
     for (final TestDB testDB : TestDB.getSupportedForBasename(Basename.INDEX)) {
 
       Database db = openMem(testDB);
@@ -1191,7 +1191,7 @@ public class CursorTest extends TestCase {
       }
 
       assertEquals(Arrays.asList("baz11", "baz11-2"), expectedData);
-      
+
       expectedData = new ArrayList<String>();
       for(Row row : cursor.newIterable().setColumnNames(
               Arrays.asList("otherfk1", "data"))) {
@@ -1201,11 +1201,11 @@ public class CursorTest extends TestCase {
       }
 
       assertTrue(expectedData.isEmpty());
-      
+
       db.close();
-    }    
+    }
   }
-  
+
   public void testFindByRowId() throws Exception {
     for (final FileFormat fileFormat : JetFormatTest.SUPPORTED_FILEFORMATS) {
       Database db = createTestTable(fileFormat);
@@ -1233,25 +1233,25 @@ public class CursorTest extends TestCase {
     }
   }
 
-  private static void doTestFindByRowId(Cursor cursor) 
+  private static void doTestFindByRowId(Cursor cursor)
     throws Exception
   {
     for(int i = 0; i < 3; ++i) {
       cursor.moveToNextRow();
-    } 
+    }
 
     Row r1 = cursor.getCurrentRow();
 
     for(int i = 0; i < 3; ++i) {
       cursor.moveToNextRow();
-    } 
+    }
 
     Row r2 = cursor.getCurrentRow();
 
     doTestFindByRowId(cursor, r1, 2);
 
     doTestFindByRowId(cursor, r2, 5);
-  } 
+  }
 
   private static void doTestFindByRowId(Cursor cursor, Row row, int id)
     throws Exception
@@ -1273,7 +1273,7 @@ public class CursorTest extends TestCase {
 
     assertFalse(cursor.findRow(RowIdImpl.FIRST_ROW_ID));
 
-    assertEquals(id - 1, cursor.getCurrentRow().get("id"));    
+    assertEquals(id - 1, cursor.getCurrentRow().get("id"));
   }
 
   public void testIterationEarlyExit() throws Exception {
@@ -1301,7 +1301,7 @@ public class CursorTest extends TestCase {
 
       IndexCursor cursor = CursorBuilder.createCursor(
           table.getIndex("value_idx"));
-      
+
       try {
         cursor.newIterable()
           .addMatchPattern("value", "val-9")
@@ -1336,7 +1336,7 @@ public class CursorTest extends TestCase {
   public void testPartialIndexFind() throws Exception
   {
     for (final FileFormat fileFormat : JetFormatTest.SUPPORTED_FILEFORMATS) {
-      
+
       Database db = createMem(fileFormat);
 
       TableImpl t = (TableImpl)new TableBuilder("Test")
@@ -1348,23 +1348,23 @@ public class CursorTest extends TestCase {
         .addIndex(new IndexBuilder("idx3").addColumns("data1", "num2", "key3"))
         .toTable(db);
 
-      Index idx = t.findIndexForColumns(Arrays.asList("data1"), 
+      Index idx = t.findIndexForColumns(Arrays.asList("data1"),
                                         TableImpl.IndexFeature.ANY_MATCH);
       assertEquals("idx3", idx.getName());
 
-      idx = t.findIndexForColumns(Arrays.asList("data1", "num2"), 
+      idx = t.findIndexForColumns(Arrays.asList("data1", "num2"),
                                   TableImpl.IndexFeature.ANY_MATCH);
       assertEquals("idx3", idx.getName());
 
-      idx = t.findIndexForColumns(Arrays.asList("data1", "num2", "key3"), 
+      idx = t.findIndexForColumns(Arrays.asList("data1", "num2", "key3"),
                                   TableImpl.IndexFeature.ANY_MATCH);
       assertEquals("idx3", idx.getName());
 
-      assertNull(t.findIndexForColumns(Arrays.asList("num2"), 
+      assertNull(t.findIndexForColumns(Arrays.asList("num2"),
                                        TableImpl.IndexFeature.ANY_MATCH));
-      assertNull(t.findIndexForColumns(Arrays.asList("data1", "key3"), 
+      assertNull(t.findIndexForColumns(Arrays.asList("data1", "key3"),
                                        TableImpl.IndexFeature.ANY_MATCH));
-      assertNull(t.findIndexForColumns(Arrays.asList("data1"), 
+      assertNull(t.findIndexForColumns(Arrays.asList("data1"),
                                        TableImpl.IndexFeature.EXACT_MATCH));
 
 
@@ -1372,23 +1372,23 @@ public class CursorTest extends TestCase {
         .addColumns("data1", "num2")
         .addToTable(t);
 
-      idx = t.findIndexForColumns(Arrays.asList("data1"), 
+      idx = t.findIndexForColumns(Arrays.asList("data1"),
                                   TableImpl.IndexFeature.ANY_MATCH);
       assertEquals("idx2", idx.getName());
 
-      idx = t.findIndexForColumns(Arrays.asList("data1", "num2"), 
+      idx = t.findIndexForColumns(Arrays.asList("data1", "num2"),
                                   TableImpl.IndexFeature.ANY_MATCH);
       assertEquals("idx2", idx.getName());
 
-      idx = t.findIndexForColumns(Arrays.asList("data1", "num2", "key3"), 
+      idx = t.findIndexForColumns(Arrays.asList("data1", "num2", "key3"),
                                   TableImpl.IndexFeature.ANY_MATCH);
       assertEquals("idx3", idx.getName());
 
-      assertNull(t.findIndexForColumns(Arrays.asList("num2"), 
+      assertNull(t.findIndexForColumns(Arrays.asList("num2"),
                                        TableImpl.IndexFeature.ANY_MATCH));
-      assertNull(t.findIndexForColumns(Arrays.asList("data1", "key3"), 
+      assertNull(t.findIndexForColumns(Arrays.asList("data1", "key3"),
                                        TableImpl.IndexFeature.ANY_MATCH));
-      assertNull(t.findIndexForColumns(Arrays.asList("data1"), 
+      assertNull(t.findIndexForColumns(Arrays.asList("data1"),
                                        TableImpl.IndexFeature.EXACT_MATCH));
 
 
@@ -1396,21 +1396,21 @@ public class CursorTest extends TestCase {
         .addColumns("data1")
         .addToTable(t);
 
-      idx = t.findIndexForColumns(Arrays.asList("data1"), 
+      idx = t.findIndexForColumns(Arrays.asList("data1"),
                                   TableImpl.IndexFeature.ANY_MATCH);
       assertEquals("idx1", idx.getName());
 
-      idx = t.findIndexForColumns(Arrays.asList("data1", "num2"), 
+      idx = t.findIndexForColumns(Arrays.asList("data1", "num2"),
                                   TableImpl.IndexFeature.ANY_MATCH);
       assertEquals("idx2", idx.getName());
 
-      idx = t.findIndexForColumns(Arrays.asList("data1", "num2", "key3"), 
+      idx = t.findIndexForColumns(Arrays.asList("data1", "num2", "key3"),
                                   TableImpl.IndexFeature.ANY_MATCH);
       assertEquals("idx3", idx.getName());
 
-      assertNull(t.findIndexForColumns(Arrays.asList("num2"), 
+      assertNull(t.findIndexForColumns(Arrays.asList("num2"),
                                        TableImpl.IndexFeature.ANY_MATCH));
-      assertNull(t.findIndexForColumns(Arrays.asList("data1", "key3"), 
+      assertNull(t.findIndexForColumns(Arrays.asList("data1", "key3"),
                                        TableImpl.IndexFeature.ANY_MATCH));
 
       db.close();
@@ -1420,7 +1420,7 @@ public class CursorTest extends TestCase {
   public void testPartialIndexLookup() throws Exception
   {
     for (final FileFormat fileFormat : JetFormatTest.SUPPORTED_FILEFORMATS) {
-      
+
       Database db = createMem(fileFormat);
 
       TableImpl t = (TableImpl)new TableBuilder("Test")
@@ -1482,7 +1482,7 @@ public class CursorTest extends TestCase {
       doFindFirstByEntry(c, 27, "C", 1, "K1");
       doFindFirstByEntry(c, null, "C", 4, "K3");
     }
-      
+
     try {
       if(colCount > 2) {
         c.findFirstRowByEntry("C", 4, "K1", 14);
@@ -1509,27 +1509,27 @@ public class CursorTest extends TestCase {
       doFindByEntry(c, new int[]{}, "B", 3, "K3");
     }
 
-    doFindByRow(idx, 13, 
+    doFindByRow(idx, 13,
                 "data1", "B", "value", "value13");
-    doFindByRow(idx, 13, 
+    doFindByRow(idx, 13,
                 "data1", "B", "key3", "K1", "value", "value13");
-    doFindByRow(idx, 13, 
+    doFindByRow(idx, 13,
         "data1", "B", "num2", 3, "key3", "K1", "value", "value13");
-    doFindByRow(idx, 13, 
+    doFindByRow(idx, 13,
         "num2", 3, "value", "value13");
-    doFindByRow(idx, 13, 
+    doFindByRow(idx, 13,
         "value", "value13");
-    doFindByRow(idx, null, 
+    doFindByRow(idx, null,
         "data1", "B", "num2", 5, "key3", "K1", "value", "value13");
-    doFindByRow(idx, null, 
+    doFindByRow(idx, null,
         "data1", "B", "value", "value4");
 
     Column col = idx.getTable().getColumn("data1");
     doFindValue(idx, 21, col, "C");
-    doFindValue(idx, null, col, "Z");    
+    doFindValue(idx, null, col, "Z");
     col = idx.getTable().getColumn("value");
     doFindValue(idx, 21, col, "value21");
-    doFindValue(idx, null, col, "valueZ");    
+    doFindValue(idx, null, col, "valueZ");
   }
 
   private static void doFindFirstByEntry(IndexCursor c, Integer expectedId,
@@ -1543,7 +1543,7 @@ public class CursorTest extends TestCase {
       assertFalse(c.findFirstRowByEntry(entry));
     }
   }
-    
+
   private static void doFindByEntryRange(IndexCursor c, int start, int end,
                                          Object... entry)
   {
@@ -1553,7 +1553,7 @@ public class CursorTest extends TestCase {
     }
     doFindByEntry(c, expectedIds, entry);
   }
-    
+
   private static void doFindByEntry(IndexCursor c, int[] ids,
                                     Object... entry)
   {
@@ -1563,7 +1563,7 @@ public class CursorTest extends TestCase {
     }
     doFindByEntry(c, expectedIds, entry);
   }
-    
+
   private static void doFindByEntry(IndexCursor c, List<Integer> expectedIds,
                                     Object... entry)
   {
@@ -1585,9 +1585,9 @@ public class CursorTest extends TestCase {
     } else {
       assertNull(r);
     }
-  } 
+  }
 
-  private static void doFindValue(Index idx, Integer id, 
+  private static void doFindValue(Index idx, Integer id,
                                   Column columnPattern, Object valuePattern)
     throws Exception
   {
@@ -1598,6 +1598,5 @@ public class CursorTest extends TestCase {
     } else {
       assertNull(value);
     }
-  } 
+  }
 }
-  

src/test/java/com/healthmarketscience/jackcess/TestUtil.java

@@ -26,11 +26,11 @@ import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 import java.nio.channels.FileChannel;
 import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
-import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
@@ -89,12 +89,20 @@ public class TestUtil
 
   public static Database open(FileFormat fileFormat, File file, boolean inMem)
     throws Exception
+  {
+    return open(fileFormat, file, inMem, null);
+  }
+
+  public static Database open(FileFormat fileFormat, File file, boolean inMem,
+                              Charset charset)
+    throws Exception
   {
     FileChannel channel = (inMem ? MemFileChannel.newChannel(
                                file, MemFileChannel.RW_CHANNEL_MODE)
                            : null);
     final Database db = new DatabaseBuilder(file).setReadOnly(true)
-      .setAutoSync(getTestAutoSync()).setChannel(channel).open();
+      .setAutoSync(getTestAutoSync()).setChannel(channel)
+      .setCharset(charset).open();
     Assert.assertEquals("Wrong JetFormat.",
                  DatabaseImpl.getFileFormatDetails(fileFormat).getFormat(),
                  ((DatabaseImpl)db).getFormat());
@@ -103,11 +111,13 @@ public class TestUtil
   }
 
   public static Database open(TestDB testDB) throws Exception {
-    return open(testDB.getExpectedFileFormat(), testDB.getFile());
+    return open(testDB.getExpectedFileFormat(), testDB.getFile(), false,
+                testDB.getExpectedCharset());
   }
 
   public static Database openMem(TestDB testDB) throws Exception {
-    return open(testDB.getExpectedFileFormat(), testDB.getFile(), true);
+    return open(testDB.getExpectedFileFormat(), testDB.getFile(), true,
+                testDB.getExpectedCharset());
   }
 
   public static Database create(FileFormat fileFormat) throws Exception {

src/test/java/com/healthmarketscience/jackcess/impl/IndexCodesTest.java

@@ -56,14 +56,14 @@ public class IndexCodesTest extends TestCase {
     SPECIAL_CHARS.put('\'', "\\'");
     SPECIAL_CHARS.put('\\', "\\\\");
   }
-  
+
   public IndexCodesTest(String name) throws Exception {
     super(name);
   }
 
   public void testIndexCodes() throws Exception
   {
-    for (final TestDB testDB : TestDB.getSupportedForBasename(Basename.INDEX_CODES)) {
+    for (final TestDB testDB : TestDB.getSupportedForBasename(Basename.INDEX_CODES, true)) {
       Database db = openMem(testDB);
 
       for(Table t : db) {
@@ -77,7 +77,7 @@ public class IndexCodesTest extends TestCase {
     }
   }
 
-  private static void checkIndexEntries(final TestDB testDB, Table t, Index index) throws Exception
+  public static void checkIndexEntries(final TestDB testDB, Table t, Index index) throws Exception
   {
 //         index.initialize();
 //         System.out.println("Ind " + index);
@@ -86,6 +86,14 @@ public class IndexCodesTest extends TestCase {
     while(cursor.moveToNextRow()) {
 
       Row row = cursor.getCurrentRow();
+
+      Object data = row.get("data");
+      if((testDB.getExpectedFileFormat() == Database.FileFormat.V1997) &&
+         (data instanceof String) && ((String)data).contains("\uFFFD")) {
+        // this row has a character not supported in the v1997 charset
+        continue;
+      }
+
       Cursor.Position curPos = cursor.getSavepoint().getCurrentPosition();
       boolean success = false;
       try {
@@ -94,14 +102,14 @@ public class IndexCodesTest extends TestCase {
       } finally {
         if(!success) {
           System.out.println("CurPos: " + curPos);
-          System.out.println("Value: " + row + ": " + 
+          System.out.println("Value: " + row + ": " +
                              toUnicodeStr(row.get("data")));
-        }          
+        }
       }
     }
-    
+
   }
-  
+
   private static void findRow(final TestDB testDB, Table t, Index index,
                               Row expectedRow,
                               Cursor.Position expectedPos)
@@ -111,7 +119,7 @@ public class IndexCodesTest extends TestCase {
     Cursor cursor = CursorBuilder.createCursor(index, idxRow, idxRow);
 
     Cursor.Position startPos = cursor.getSavepoint().getCurrentPosition();
-    
+
     cursor.beforeFirst();
     while(cursor.moveToNextRow()) {
       Row row = cursor.getCurrentRow();
@@ -125,11 +133,12 @@ public class IndexCodesTest extends TestCase {
 
     // TODO long rows not handled completely yet in V2010
     // seems to truncate entry at 508 bytes with some trailing 2 byte seq
-    if(testDB.getExpectedFileFormat() == Database.FileFormat.V2010) {
+    if((testDB != null) &&
+       (testDB.getExpectedFileFormat() == Database.FileFormat.V2010)) {
       String rowId = expectedRow.getString("name");
       String tName = t.getName();
       if(("Table11".equals(tName) || "Table11_desc".equals(tName)) &&
-         ("row10".equals(rowId) || "row11".equals(rowId) || 
+         ("row10".equals(rowId) || "row11".equals(rowId) ||
           "row12".equals(rowId))) {
         System.out.println(
             "TODO long rows not handled completely yet in V2010: " + tName +
@@ -142,13 +151,13 @@ public class IndexCodesTest extends TestCase {
          entryToString(startPos));
   }
 
-  
+
   //////
   //
   // The code below is for use in reverse engineering index entries.
   //
   //////
-  
+
   public void testNothing() throws Exception {
     // keep this so build doesn't fail if other tests are disabled
   }
@@ -161,7 +170,7 @@ public class IndexCodesTest extends TestCase {
       .addColumn(new ColumnBuilder("row", DataType.TEXT))
       .addColumn(new ColumnBuilder("data", DataType.TEXT))
       .toTable(db);
-    
+
     for(int i = 0; i < 256; ++i) {
       String str = "AA" + ((char)i) + "AA";
       t.addRow("row" + i, str);
@@ -182,7 +191,7 @@ public class IndexCodesTest extends TestCase {
                (byte)42 + i, (short)53 + i, 13 * i,
                (6.7d / i), null, null, true);
     }
-    
+
     db.close();
   }
 
@@ -212,10 +221,10 @@ public class IndexCodesTest extends TestCase {
       .toTable(db);
 
     char c = (char)0x3041;   // crazy 7F 02 ... A0
-    char c2 = (char)0x30A2;  // crazy 7F 02 ... 
+    char c2 = (char)0x30A2;  // crazy 7F 02 ...
     char c3 = (char)0x2045;  // inat 27 ... 1C
     char c4 = (char)0x3043;  // crazy 7F 03 ... A0
-    char c5 = (char)0x3046;  // crazy 7F 04 ... 
+    char c5 = (char)0x3046;  // crazy 7F 04 ...
     char c6 = (char)0x30F6;  // crazy 7F 0D ... A0
     char c7 = (char)0x3099;  // unprint 03
     char c8 = (char)0x0041;  // A
@@ -232,13 +241,13 @@ public class IndexCodesTest extends TestCase {
 //     t = new TableBuilder("Table2")
 //       .addColumn(new ColumnBuilder("data", DataType.TEXT))
 //       .toTable(db);
-    
+
 //     writeChars(0x0000, t);
 
 //     t = new TableBuilder("Table3")
 //       .addColumn(new ColumnBuilder("data", DataType.TEXT))
 //       .toTable(db);
-    
+
 //     writeChars(0x0400, t);
 
 
@@ -255,13 +264,13 @@ public class IndexCodesTest extends TestCase {
 
     Index ind = t.getIndexes().iterator().next();
     ((IndexImpl)ind).initialize();
-    
+
     System.out.println("Ind " + ind);
 
     Cursor cursor = CursorBuilder.createCursor(ind);
     while(cursor.moveToNextRow()) {
       System.out.println("=======");
-      String entryStr = 
+      String entryStr =
         entryToString(cursor.getSavepoint().getCurrentPosition());
       System.out.println("Entry Bytes: " + entryStr);
       System.out.println("Value: " + cursor.getCurrentRow() + "; " +
@@ -316,10 +325,10 @@ public class IndexCodesTest extends TestCase {
       System.out.println("Savepoint: " + cursor.getSavepoint());
       System.out.println("Value: " + cursor.getCurrentRow());
     }
-    
+
     db.close();
   }
-  
+
   public void x_testReverseIsoMdb2010() throws Exception
   {
     Database db = open(Database.FileFormat.V2010, new File("/data2/jackcess_test/testAllIndexCodes3_2010.accdb"));
@@ -343,8 +352,8 @@ public class IndexCodesTest extends TestCase {
     Map<Character,String[]> inat2Codes = new TreeMap<Character,String[]>();
     Map<Character,String[]> inat2ExtraCodes = new TreeMap<Character,String[]>();
     Map<Character,String[]> inat2CrazyCodes = new TreeMap<Character,String[]>();
-    
-    
+
+
     Cursor cursor = CursorBuilder.createCursor(index);
     while(cursor.moveToNextRow()) {
 //       System.out.println("=======");
@@ -377,14 +386,14 @@ public class IndexCodesTest extends TestCase {
         handleInlineEntry(m.group(1), c, inlineCodes);
 
       } else if(entryStr.contains("01 01 01 80")) {
-        
+
         // handle most unprintable codes
         type = "UNPRINTABLE";
         Matcher m = unprintPat.matcher(entryStr);
         m.find();
         handleUnprintableEntry(m.group(2), c, unprintCodes);
 
-      } else if(entryStr.contains("01 02 02") && 
+      } else if(entryStr.contains("01 02 02") &&
                 !entryStr.contains("FF 02 80 FF 80")) {
 
         // handle chars w/ symbols
@@ -393,7 +402,7 @@ public class IndexCodesTest extends TestCase {
         m.find();
         handleInternationalEntry(m.group(1), m.group(2), c,
                                  inatInlineCodes, inatExtraCodes);
-        
+
       } else if(entryStr.contains("0E 02 0E 02 0E 02 0E 02 01 02")) {
 
         // handle chars w/ symbols
@@ -401,7 +410,7 @@ public class IndexCodesTest extends TestCase {
         Matcher m = unprint2Pat.matcher(entryStr);
         m.find();
         handleUnprintable2Entry(m.group(1), c, unprint2Codes);
-        
+
       } else if(entryStr.contains("FF 02 80 FF 80")) {
 
         type = "CRAZY_INAT";
@@ -415,8 +424,8 @@ public class IndexCodesTest extends TestCase {
 
         // throw new RuntimeException("unhandled " + entryStr);
         System.out.println("unhandled " + entryStr);
-      }      
-        
+      }
+
       System.out.println("Type: " + type);
     }
 
@@ -456,7 +465,7 @@ public class IndexCodesTest extends TestCase {
                            toByteString(extra));
         continue;
       }
-        
+
       chars = unprintCodes.get(cc);
       if(chars != null) {
         System.out.println("U" + toByteString(chars));
@@ -495,10 +504,10 @@ public class IndexCodesTest extends TestCase {
       throw new RuntimeException("Unhandled char " + toUnicodeStr(c));
     }
     System.out.println("\n***END CODES");
-    
+
     db.close();
   }
- 
+
   public void x_testReverseIsoMdb() throws Exception
   {
     Database db = open(Database.FileFormat.V2000, new File("/data2/jackcess_test/testAllIndexCodes3.mdb"));
@@ -522,8 +531,8 @@ public class IndexCodesTest extends TestCase {
     Map<Character,String[]> inat2Codes = new TreeMap<Character,String[]>();
     Map<Character,String[]> inat2ExtraCodes = new TreeMap<Character,String[]>();
     Map<Character,String[]> inat2CrazyCodes = new TreeMap<Character,String[]>();
-    
-    
+
+
     Cursor cursor = CursorBuilder.createCursor(index);
     while(cursor.moveToNextRow()) {
 //       System.out.println("=======");
@@ -555,14 +564,14 @@ public class IndexCodesTest extends TestCase {
         handleInlineEntry(m.group(1), c, inlineCodes);
 
       } else if(entryStr.contains("01 01 01 80")) {
-        
+
         // handle most unprintable codes
         type = "UNPRINTABLE";
         Matcher m = unprintPat.matcher(entryStr);
         m.find();
         handleUnprintableEntry(m.group(2), c, unprintCodes);
 
-      } else if(entryStr.contains("01 02 02") && 
+      } else if(entryStr.contains("01 02 02") &&
                 !entryStr.contains("FF 02 80 FF 80")) {
 
         // handle chars w/ symbols
@@ -571,7 +580,7 @@ public class IndexCodesTest extends TestCase {
         m.find();
         handleInternationalEntry(m.group(1), m.group(2), c,
                                  inatInlineCodes, inatExtraCodes);
-        
+
       } else if(entryStr.contains("4A 4A 4A 4A 01 02")) {
 
         // handle chars w/ symbols
@@ -579,7 +588,7 @@ public class IndexCodesTest extends TestCase {
         Matcher m = unprint2Pat.matcher(entryStr);
         m.find();
         handleUnprintable2Entry(m.group(1), c, unprint2Codes);
-        
+
       } else if(entryStr.contains("FF 02 80 FF 80")) {
 
         type = "CRAZY_INAT";
@@ -592,8 +601,8 @@ public class IndexCodesTest extends TestCase {
       } else {
 
         throw new RuntimeException("unhandled " + entryStr);
-      }      
-        
+      }
+
       System.out.println("Type: " + type);
     }
 
@@ -633,7 +642,7 @@ public class IndexCodesTest extends TestCase {
                            toByteString(extra));
         continue;
       }
-        
+
       chars = unprintCodes.get(cc);
       if(chars != null) {
         System.out.println("U" + toByteString(chars));
@@ -672,7 +681,7 @@ public class IndexCodesTest extends TestCase {
       throw new RuntimeException("Unhandled char " + toUnicodeStr(c));
     }
     System.out.println("\n***END CODES");
-    
+
     db.close();
   }
 
@@ -691,21 +700,21 @@ public class IndexCodesTest extends TestCase {
   {
     inlineCodes.put(c, entryCodes.trim().split(" "));
   }
-  
+
   private static void handleUnprintableEntry(
       String entryCodes, char c, Map<Character,String[]> unprintCodes)
     throws Exception
   {
     unprintCodes.put(c, entryCodes.trim().split(" "));
   }
-  
+
   private static void handleUnprintable2Entry(
       String entryCodes, char c, Map<Character,String[]> unprintCodes)
     throws Exception
   {
     unprintCodes.put(c, entryCodes.trim().split(" "));
   }
-  
+
   private static void handleInternationalEntry(
       String inlineCodes, String entryCodes, char c,
       Map<Character,String[]> inatInlineCodes,
@@ -732,20 +741,20 @@ public class IndexCodesTest extends TestCase {
     }
   }
 
-  private static String toUnicodeStr(Object obj) throws Exception {
+  public static String toUnicodeStr(Object obj) throws Exception {
     StringBuilder sb = new StringBuilder();
     for(char c : obj.toString().toCharArray()) {
       sb.append(toUnicodeStr(c)).append(" ");
     }
     return sb.toString();
   }
-  
+
   private static String toUnicodeStr(char c) throws Exception {
     String specialStr = SPECIAL_CHARS.get(c);
     if(specialStr != null) {
       return specialStr;
     }
-    
+
     String digits = Integer.toHexString(c).toUpperCase();
     while(digits.length() < 4) {
       digits = "0" + digits;
@@ -769,7 +778,7 @@ public class IndexCodesTest extends TestCase {
     }
     return builder.toString();
   }
-  
+
   public static String entryToString(Cursor.Position curPos)
     throws Exception
   {
@@ -783,5 +792,5 @@ public class IndexCodesTest extends TestCase {
     return ByteUtil.toHexString(ByteBuffer.wrap(entryBytes),
                                 0, entryBytes.length, false);
   }
-  
+
 }

src/test/java/com/healthmarketscience/jackcess/impl/JetFormatTest.java

@@ -3,6 +3,7 @@ package com.healthmarketscience.jackcess.impl;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.nio.channels.FileChannel;
 import java.nio.channels.NonWritableChannelException;
 import java.sql.SQLException;
@@ -67,6 +68,9 @@ public class JetFormatTest extends TestCase {
     public String toString() { return _basename; }
   }
 
+  /** charset for access 97 dbs */
+  public static final Charset A97_CHARSET = Charset.forName("windows-1252");
+
   /** Defines currently supported db file formats.  (can be modified at
       runtime via the system property
       "com.healthmarketscience.jackcess.testFormats") */
@@ -109,12 +113,15 @@ public class JetFormatTest extends TestCase {
 
     private final File dbFile;
     private final FileFormat expectedFileFormat;
+    private final Charset _charset;
 
     private TestDB(File databaseFile,
-                   FileFormat expectedDBFileFormat) {
+                   FileFormat expectedDBFileFormat,
+                   Charset charset) {
 
       dbFile = databaseFile;
       expectedFileFormat = expectedDBFileFormat;
+      _charset = charset;
     }
 
     public final File getFile() { return dbFile; }
@@ -127,6 +134,10 @@ public class JetFormatTest extends TestCase {
       return DatabaseImpl.getFileFormatDetails(expectedFileFormat).getFormat();
     }
 
+    public final Charset getExpectedCharset() {
+      return _charset;
+    }
+
     @Override
     public final String toString() {
       return "dbFile: " + dbFile.getAbsolutePath()
@@ -162,7 +173,12 @@ public class JetFormatTest extends TestCase {
           throw new RuntimeException(e);
         }
 
-        supportedTestDBs.add(new TestDB(testFile, fileFormat));
+        Charset charset = null;
+        if(fileFormat == FileFormat.V1997) {
+          charset = A97_CHARSET;
+        }
+
+        supportedTestDBs.add(new TestDB(testFile, fileFormat, charset));
       }
       return supportedTestDBs;
     }