merge branch newformats changes through r453

git-svn-id: https://svn.code.sf.net/p/jackcess/code/jackcess/trunk@454 f203690c-595d-4dc9-a70b-905162fa7fd2

6 files changed, 295 insertions, 29 deletions

diff --git a/src/java/com/healthmarketscience/jackcess/ByteUtil.java b/src/java/com/healthmarketscience/jackcess/ByteUtil.java
index 7013695..fbb1521 100644
--- a/src/java/com/healthmarketscience/jackcess/ByteUtil.java
+++ b/src/java/com/healthmarketscience/jackcess/ByteUtil.java
@@ -308,6 +308,23 @@ public final class ByteUtil {
     }
     return true;
   }
+
+  /**
+   * Searches for a pattern of bytes in the given buffer starting at the
+   * given offset.
+   * @return the offset of the pattern if a match is found, -1 otherwise
+   */
+  public static int findRange(ByteBuffer buffer, int start, byte[] pattern)
+  {
+    byte firstByte = pattern[0];
+    int limit = buffer.limit() - pattern.length;
+    for(int i = start; i < limit; ++i) {
+      if((firstByte == buffer.get(i)) && matchesRange(buffer, i, pattern)) {
+        return i;
+      }
+    }
+    return -1;
+  }
   
   /**
    * Convert a byte buffer to a hexadecimal string for display
diff --git a/src/java/com/healthmarketscience/jackcess/Database.java b/src/java/com/healthmarketscience/jackcess/Database.java
index 8609b65..5044de7 100644
--- a/src/java/com/healthmarketscience/jackcess/Database.java
+++ b/src/java/com/healthmarketscience/jackcess/Database.java
@@ -168,9 +168,38 @@ public class Database
   private static final String CAT_COL_DATE_UPDATE = "DateUpdate";
   /** System catalog column name of the flags column */
   private static final String CAT_COL_FLAGS = "Flags";
+  /** System catalog column name of the properties column */
+  private static final String CAT_COL_PROPS = "LvProp";
   
-  /** Empty database template for creating new databases */
-  private static final String EMPTY_MDB = "com/healthmarketscience/jackcess/empty.mdb";
+  public static enum FileFormat {
+
+    V1997(null, JetFormat.VERSION_3), // v97 is not supported, so no empty template is provided
+    V2000("com/healthmarketscience/jackcess/empty.mdb", JetFormat.VERSION_4),
+    V2003("com/healthmarketscience/jackcess/empty2003.mdb", JetFormat.VERSION_4),
+    V2007("com/healthmarketscience/jackcess/empty2007.accdb", JetFormat.VERSION_5, ".accdb");
+
+    private final String _emptyFile;
+    private final JetFormat _format;
+    private final String _ext;
+
+    private FileFormat(String emptyDBFile, JetFormat jetFormat) {
+      this(emptyDBFile, jetFormat, ".mdb");
+    }
+
+    private FileFormat(String emptyDBFile, JetFormat jetFormat, String ext) {
+      _emptyFile = emptyDBFile;
+      _format = jetFormat;
+      _ext = ext;
+    }
+
+    public JetFormat getJetFormat() { return _format; }
+
+    public String getFileExtension() { return _ext; }
+
+    @Override
+    public String toString() { return name() + ", jetFormat: " + getJetFormat(); }
+  }
+
   /** Prefix for column or table names that are reserved words */
   private static final String ESCAPE_PREFIX = "x";
   /** Prefix that flags system tables */
@@ -183,6 +212,8 @@ public class Database
   private static final String TABLE_SYSTEM_RELATIONSHIPS = "MSysRelationships";
   /** Name of the table that contains queries */
   private static final String TABLE_SYSTEM_QUERIES = "MSysQueries";
+  /** Name of the table that contains queries */
+  private static final String OBJECT_NAME_DBPROPS = "MSysDb";
   /** System object type for table definitions */
   private static final Short TYPE_TABLE = (short) 1;
   /** System object type for query definitions */
@@ -269,6 +300,8 @@ public class Database
   private boolean _useBigIndex;
   /** optional error handler to use when row errors are encountered */
   private ErrorHandler _dbErrorHandler;
+  /** the file format of the database */
+  private FileFormat _fileFormat;
   
   /**
    * Open an existing Database.  If the existing file is not writeable, the
@@ -329,14 +362,14 @@ public class Database
     }
     return new Database(openChannel(mdbFile,
                                     (!mdbFile.canWrite() || readOnly)),
-                        autoSync);
+                        autoSync, null);
   }
   
   /**
-   * Create a new Database
+   * Create a new Access 2000 Database 
    * <p>
    * Equivalent to:
-   * {@code  create(mdbFile, DEFAULT_AUTO_SYNC);}
+   * {@code  create(FileFormat.V2000, mdbFile, DEFAULT_AUTO_SYNC);}
    * 
    * @param mdbFile Location to write the new database to.  <b>If this file
    *    already exists, it will be overwritten.</b>
@@ -348,7 +381,29 @@ public class Database
   }
   
   /**
-   * Create a new Database
+   * Create a new Database for the given fileFormat
+   * <p>
+   * Equivalent to:
+   * {@code  create(fileFormat, mdbFile, DEFAULT_AUTO_SYNC);}
+   * 
+   * @param fileFormat version of new database.
+   * @param mdbFile Location to write the new database to.  <b>If this file
+   *    already exists, it will be overwritten.</b>
+   *
+   * @see #create(File,boolean)
+   */
+  public static Database create(FileFormat fileFormat, File mdbFile) 
+    throws IOException 
+  {
+    return create(fileFormat, mdbFile, DEFAULT_AUTO_SYNC);
+  }
+  
+  /**
+   * Create a new Access 2000 Database
+   * <p>
+   * Equivalent to:
+   * {@code  create(FileFormat.V2000, mdbFile, DEFAULT_AUTO_SYNC);}
+   * 
    * @param mdbFile Location to write the new database to.  <b>If this file
    *    already exists, it will be overwritten.</b>
    * @param autoSync whether or not to enable auto-syncing on write.  if
@@ -362,19 +417,53 @@ public class Database
    */
   public static Database create(File mdbFile, boolean autoSync)
     throws IOException
-  {    
+  {
+    return create(FileFormat.V2000, mdbFile, autoSync);
+  }
+
+  /**
+   * Create a new Database for the given fileFormat
+   * @param fileFormat version of new database.
+   * @param mdbFile Location to write the new database to.  <b>If this file
+   *    already exists, it will be overwritten.</b>
+   * @param autoSync whether or not to enable auto-syncing on write.  if
+   *                 {@code true}, writes will be immediately flushed to disk.
+   *                 This leaves the database in a (fairly) consistent state
+   *                 on each write, but can be very inefficient for many
+   *                 updates.  if {@code false}, flushing to disk happens at
+   *                 the jvm's leisure, which can be much faster, but may
+   *                 leave the database in an inconsistent state if failures
+   *                 are encountered during writing.
+   */
+  public static Database create(FileFormat fileFormat, File mdbFile, 
+                                boolean autoSync)
+    throws IOException
+  {
     FileChannel channel = openChannel(mdbFile, false);
     channel.truncate(0);
     channel.transferFrom(Channels.newChannel(
         Thread.currentThread().getContextClassLoader().getResourceAsStream(
-            EMPTY_MDB)), 0, Integer.MAX_VALUE);
-    return new Database(channel, autoSync);
+            fileFormat._emptyFile)), 0, Integer.MAX_VALUE);
+    return new Database(channel, autoSync, fileFormat);
   }
-  
-  private static FileChannel openChannel(File mdbFile, boolean readOnly)
+
+  /**
+   * Package visible only to support unit tests via DatabaseTest.openChannel().
+   * @param mdbFile file to open
+   * @param readOnly true if read-only
+   * @return a FileChannel on the given file.
+   * @exception FileNotFoundException
+   *            if the mode is <tt>"r"</tt> but the given file object does
+   *            not denote an existing regular file, or if the mode begins
+   *            with <tt>"rw"</tt> but the given file object does not denote
+   *            an existing, writable regular file and a new regular file of
+   *            that name cannot be created, or if some other error occurs
+   *            while opening or creating the file
+   */
+  static FileChannel openChannel(final File mdbFile, final boolean readOnly)
     throws FileNotFoundException
   {
-    String mode = (readOnly ? "r" : "rw");
+    final String mode = (readOnly ? "r" : "rw");
     return new RandomAccessFile(mdbFile, mode).getChannel();
   }
   
@@ -384,9 +473,12 @@ public class Database
    *    FileChannel instead of a ReadableByteChannel because we need to
    *    randomly jump around to various points in the file.
    */
-  protected Database(FileChannel channel, boolean autoSync) throws IOException
+  protected Database(FileChannel channel, boolean autoSync, 
+                     FileFormat fileFormat)
+    throws IOException
   {
     _format = JetFormat.getFormat(channel);
+    _fileFormat = fileFormat;
     _pageChannel = new PageChannel(channel, _format, autoSync);
     // note, it's slighly sketchy to pass ourselves along partially
     // constructed, but only our _format and _pageChannel refs should be
@@ -449,6 +541,59 @@ public class Database
   public void setErrorHandler(ErrorHandler newErrorHandler) {
     _dbErrorHandler = newErrorHandler;
   }    
+
+  /**
+   * Returns the FileFormat of this database (which may involve inspecting the
+   * database itself).
+   * @throws IllegalStateException if the file format cannot be determined
+   */
+  public FileFormat getFileFormat()
+  {
+    if(_fileFormat == null) {
+
+      Map<Database.FileFormat,byte[]> possibleFileFormats =
+        getFormat().getPossibleFileFormats();
+
+      if(possibleFileFormats.size() == 1) {
+
+        // single possible format, easy enough
+        _fileFormat = possibleFileFormats.keySet().iterator().next();
+
+      } else {
+
+        // need to check the "AccessVersion" property
+        byte[] dbProps = null;
+        for(Map<String,Object> row :
+              Cursor.createCursor(_systemCatalog).iterable(
+                  Arrays.asList(CAT_COL_NAME, CAT_COL_PROPS))) {
+          if(OBJECT_NAME_DBPROPS.equals(row.get(CAT_COL_NAME))) {
+            dbProps = (byte[])row.get(CAT_COL_PROPS);
+            break;
+          }
+        }
+        
+        if(dbProps != null) {
+
+          // search for certain "version strings" in the properties (we
+          // can't fully parse the properties objects, but we can still
+          // find the byte pattern)
+          ByteBuffer dbPropBuf = ByteBuffer.wrap(dbProps);
+          for(Map.Entry<Database.FileFormat,byte[]> possible : 
+                possibleFileFormats.entrySet()) {
+            if(ByteUtil.findRange(dbPropBuf, 0, possible.getValue()) >= 0) {
+              _fileFormat = possible.getKey();
+              break;
+            }
+          }
+        }
+        
+        if(_fileFormat == null) {
+          throw new IllegalStateException("Could not determine FileFormat");
+        }
+      }
+    }
+    return _fileFormat;
+  }
   
   /**
    * Read the system catalog
diff --git a/src/java/com/healthmarketscience/jackcess/Index.java b/src/java/com/healthmarketscience/jackcess/Index.java
index b629049..a2921e4 100644
--- a/src/java/com/healthmarketscience/jackcess/Index.java
+++ b/src/java/com/healthmarketscience/jackcess/Index.java
@@ -1540,17 +1540,31 @@ public abstract class Index implements Comparable<Index> {
       boolean isNegative = ((valueBytes[0] & 0x80) != 0);
 
       // bit twiddling rules:
-      // isAsc && !isNeg => setReverseSignByte
-      // isAsc && isNeg => flipBytes, setReverseSignByte
-      // !isAsc && !isNeg => flipBytes, setReverseSignByte
-      // !isAsc && isNeg => setReverseSignByte
+      // isAsc && !isNeg => setReverseSignByte             => FF 00 00 ...
+      // isAsc && isNeg => flipBytes, setReverseSignByte   => 00 FF FF ...
+      // !isAsc && !isNeg => flipBytes, setReverseSignByte => FF FF FF ...
+      // !isAsc && isNeg => setReverseSignByte             => 00 00 00 ...
       
+      // v2007 bit twiddling rules (old ordering was a bug, MS kb 837148):
+      // isAsc && !isNeg => setSignByte 0xFF            => FF 00 00 ...
+      // isAsc && isNeg => setSignByte 0xFF, flipBytes  => 00 FF FF ...
+      // !isAsc && !isNeg => setSignByte 0xFF           => FF 00 00 ...
+      // !isAsc && isNeg => setSignByte 0xFF, flipBytes => 00 FF FF ...
+
+      boolean alwaysRevFirstByte = getColumn().getFormat().REVERSE_FIRST_BYTE_IN_DESC_NUMERIC_INDEXES;
+      if(alwaysRevFirstByte) {
+        // reverse the sign byte (before any byte flipping)
+        valueBytes[0] = (byte)0xFF;
+      }
+
       if(isNegative == isAscending()) {
         flipBytes(valueBytes);
       }
 
-      // reverse the sign byte (after any previous byte flipping)
-      valueBytes[0] = (isNegative ? (byte)0x00 : (byte)0xFF);
+      if(!alwaysRevFirstByte) {
+        // reverse the sign byte (after any previous byte flipping)
+        valueBytes[0] = (isNegative ? (byte)0x00 : (byte)0xFF);
+      }
       
       bout.write(valueBytes);
     }    
diff --git a/src/java/com/healthmarketscience/jackcess/JetFormat.java b/src/java/com/healthmarketscience/jackcess/JetFormat.java
index 38b4bc5..f5cc175 100644
--- a/src/java/com/healthmarketscience/jackcess/JetFormat.java
+++ b/src/java/com/healthmarketscience/jackcess/JetFormat.java
@@ -31,6 +31,9 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.charset.Charset;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.Map;
 
 /**
  * Encapsulates constants describing a specific version of the Access Jet format
@@ -46,12 +49,41 @@ public abstract class JetFormat {
   /** Maximum size of a text field */
   public static final short TEXT_FIELD_MAX_LENGTH = 255 * TEXT_FIELD_UNIT_SIZE;
   
-  /** Offset in the file that holds the byte describing the Jet format version */
+  /** Offset in the file that holds the byte describing the Jet format
+      version */
   private static final long OFFSET_VERSION = 20L;
   /** Version code for Jet version 3 */
   private static final byte CODE_VERSION_3 = 0x0;
   /** Version code for Jet version 4 */
   private static final byte CODE_VERSION_4 = 0x1;
+  /** Version code for Jet version 5 */
+  private static final byte CODE_VERSION_5 = 0x2;
+
+  /** value of the "AccessVersion" property for access 2000 dbs:
+      {@code "08.50"} */
+  private static final byte[] ACCESS_VERSION_2000 = new byte[] {
+    '0', 0, '8', 0, '.', 0, '5', 0, '0', 0};
+  /** value of the "AccessVersion" property for access 2002/2003 dbs
+      {@code "09.50"}  */
+  private static final byte[] ACCESS_VERSION_2003 = new byte[] {
+    '0', 0, '9', 0, '.', 0, '5', 0, '0', 0};
+
+  // use nested inner class to avoid problematic static init loops
+  private static final class PossibleFileFormats {
+    private static final Map<Database.FileFormat,byte[]> POSSIBLE_VERSION_3 = 
+      Collections.singletonMap(Database.FileFormat.V1997, (byte[])null);
+
+    private static final Map<Database.FileFormat,byte[]> POSSIBLE_VERSION_4 = 
+      new EnumMap<Database.FileFormat,byte[]>(Database.FileFormat.class);
+
+    private static final Map<Database.FileFormat,byte[]> POSSIBLE_VERSION_5 = 
+      Collections.singletonMap(Database.FileFormat.V2007, (byte[])null);
+
+    static {
+      POSSIBLE_VERSION_4.put(Database.FileFormat.V2000, ACCESS_VERSION_2000);
+      POSSIBLE_VERSION_4.put(Database.FileFormat.V2003, ACCESS_VERSION_2003);
+    }
+  }
 
   //These constants are populated by this class's constructor.  They can't be
   //populated by the subclass's constructor because they are final, and Java
@@ -128,13 +160,19 @@ public abstract class JetFormat {
   public final int MAX_TABLE_NAME_LENGTH;
   public final int MAX_COLUMN_NAME_LENGTH;
   public final int MAX_INDEX_NAME_LENGTH;
+
+  public final boolean REVERSE_FIRST_BYTE_IN_DESC_NUMERIC_INDEXES;
   
   public final Charset CHARSET;
   
+  public static final JetFormat VERSION_3 = new Jet3Format();
   public static final JetFormat VERSION_4 = new Jet4Format();
+  public static final JetFormat VERSION_5 = new Jet5Format();
 
   /**
+   * @param channel the database file.
    * @return The Jet Format represented in the passed-in file
+   * @throws IOException if the database file format is unsupported.
    */
   public static JetFormat getFormat(FileChannel channel) throws IOException {
     ByteBuffer buffer = ByteBuffer.allocate(1);
@@ -144,8 +182,12 @@ public abstract class JetFormat {
     }
     buffer.flip();
     byte version = buffer.get();
-    if (version == CODE_VERSION_4) {
+    if (version == CODE_VERSION_3) {
+      return VERSION_3;
+    } else if (version == CODE_VERSION_4) {
       return VERSION_4;
+    } else if (version == CODE_VERSION_5) {
+      return VERSION_5;
     }
     throw new IOException("Unsupported version: " + version);
   }
@@ -220,7 +262,8 @@ public abstract class JetFormat {
     MAX_TABLE_NAME_LENGTH = defineMaxTableNameLength();
     MAX_COLUMN_NAME_LENGTH = defineMaxColumnNameLength();
     MAX_INDEX_NAME_LENGTH = defineMaxIndexNameLength();
-
+    
+    REVERSE_FIRST_BYTE_IN_DESC_NUMERIC_INDEXES = defineReverseFirstByteInDescNumericIndexes();
     
     CHARSET = defineCharset();
   }
@@ -295,15 +338,23 @@ public abstract class JetFormat {
   
   protected abstract Charset defineCharset();
 
+  protected abstract boolean defineReverseFirstByteInDescNumericIndexes();
+
+  protected abstract Map<Database.FileFormat,byte[]> getPossibleFileFormats();
+
   @Override
   public String toString() {
     return _name;
   }
   
-  private static final class Jet4Format extends JetFormat {
+  private static class Jet4Format extends JetFormat {
 
     private Jet4Format() {
-      super("VERSION_4");
+      this("VERSION_4");
+    }
+
+    private Jet4Format(final String name) {
+      super(name);
     }
 
     @Override
@@ -435,7 +486,43 @@ public abstract class JetFormat {
     protected int defineMaxIndexNameLength() { return 64; }
       
     @Override
+    protected boolean defineReverseFirstByteInDescNumericIndexes() { return false; }
+
+    @Override
     protected Charset defineCharset() { return Charset.forName("UTF-16LE"); }
+
+    @Override
+    protected Map<Database.FileFormat,byte[]> getPossibleFileFormats()
+    {
+      return PossibleFileFormats.POSSIBLE_VERSION_4;
+    }
+
   }
   
+  private static final class Jet3Format extends Jet4Format {
+      private Jet3Format() {
+        super("VERSION_3");
+      }
+
+    @Override
+    protected Map<Database.FileFormat,byte[]> getPossibleFileFormats() {
+      return PossibleFileFormats.POSSIBLE_VERSION_3;
+    }
+
+  }
+
+  private static final class Jet5Format extends Jet4Format {
+      private Jet5Format() {
+        super("VERSION_5");
+      }
+
+    @Override
+    protected boolean defineReverseFirstByteInDescNumericIndexes() { return true; }
+
+    @Override
+    protected Map<Database.FileFormat,byte[]> getPossibleFileFormats() {
+      return PossibleFileFormats.POSSIBLE_VERSION_5;
+    }
+  }
+
 }
diff --git a/src/java/com/healthmarketscience/jackcess/PageChannel.java b/src/java/com/healthmarketscience/jackcess/PageChannel.java
index 8adf74b..e72bcd7 100644
--- a/src/java/com/healthmarketscience/jackcess/PageChannel.java
+++ b/src/java/com/healthmarketscience/jackcess/PageChannel.java
@@ -56,9 +56,9 @@ public class PageChannel implements Channel, Flushable {
     new byte[]{PageTypes.INVALID, (byte)0, (byte)0, (byte)0};
   
   /** Global usage map always lives on page 1 */
-  private static final int PAGE_GLOBAL_USAGE_MAP = 1;
+  static final int PAGE_GLOBAL_USAGE_MAP = 1;
   /** Global usage map always lives at row 0 */
-  private static final int ROW_GLOBAL_USAGE_MAP = 0;
+  static final int ROW_GLOBAL_USAGE_MAP = 0;
   
   /** Channel containing the database */
   private final FileChannel _channel;
diff --git a/src/java/com/healthmarketscience/jackcess/UsageMap.java b/src/java/com/healthmarketscience/jackcess/UsageMap.java
index 6495f0c..5e6ec1e 100644
--- a/src/java/com/healthmarketscience/jackcess/UsageMap.java
+++ b/src/java/com/healthmarketscience/jackcess/UsageMap.java
@@ -72,8 +72,11 @@ public class UsageMap
   /** the current handler implementation for reading/writing the specific
       usage map type.  note, this may change over time. */
   private Handler _handler;
-  
-  /**
+
+  /** Error message prefix used when map type is unrecognized. */
+  static final String MSG_PREFIX_UNRECOGNIZED_MAP = "Unrecognized map type: ";
+
+    /**
    * @param database database that contains this usage map
    * @param tableBuffer Buffer that contains this map's declaration
    * @param pageNum Page number that this usage map is contained in
@@ -139,7 +142,7 @@ public class UsageMap
     } else if (mapType == MAP_TYPE_REFERENCE) {
       _handler = new ReferenceHandler();
     } else {
-      throw new IOException("Unrecognized map type: " + mapType);
+      throw new IOException(MSG_PREFIX_UNRECOGNIZED_MAP + mapType);
     }
   }