Add CustomLinkResolver which facilitates loading linked tables from files which are not access databases. fixes feature request #36

git-svn-id: https://svn.code.sf.net/p/jackcess/code/jackcess/trunk@1095 f203690c-595d-4dc9-a70b-905162fa7fd2

src/changes/changes.xml

@@ -18,6 +18,11 @@
               issue="3">
         Allow inserting negative auto number fields, thanks to Gord Thompson.
       </action>
+      <action dev="jahlborn" type="update" system="SourceForge2Features"
+              issue="36">
+        Add CustomLinkResolver which facilitates loading linked tables from
+        files which are not access databases.
+      </action>
     </release>
     <release version="2.1.6" date="2016-11-29">
       <action dev="jahlborn" type="update" system="SourceForge2Features"

src/main/java/com/healthmarketscience/jackcess/impl/DatabaseImpl.java

@@ -454,10 +454,10 @@ public class DatabaseImpl implements Database
     boolean success = false;
     try {
       channel.truncate(0);
-      transferFrom(channel, getResourceAsStream(details.getEmptyFilePath()));
+      transferDbFrom(channel, getResourceAsStream(details.getEmptyFilePath()));
       channel.force(true);
       DatabaseImpl db = new DatabaseImpl(mdbFile, channel, closeChannel, autoSync, 
-                                 fileFormat, charset, timeZone, null);
+                                         fileFormat, charset, timeZone, null);
       success = true;
       return db;
     } finally {
@@ -508,8 +508,8 @@ public class DatabaseImpl implements Database
    * @param timeZone TimeZone to use, if {@code null}, uses default
    */
   protected DatabaseImpl(File file, FileChannel channel, boolean closeChannel,
-                     boolean autoSync, FileFormat fileFormat, Charset charset,
-                     TimeZone timeZone, CodecProvider provider)
+                         boolean autoSync, FileFormat fileFormat, Charset charset,
+                         TimeZone timeZone, CodecProvider provider)
     throws IOException
   {
     _file = file;
@@ -971,7 +971,7 @@ public class DatabaseImpl implements Database
    * @param includeSystemTables whether to consider returning a system table
    * @return The table, or null if it doesn't exist
    */
-  private TableImpl getTable(String name, boolean includeSystemTables) 
+  protected TableImpl getTable(String name, boolean includeSystemTables) 
     throws IOException 
   {
     TableInfo tableInfo = getTableInfo(name, includeSystemTables);
@@ -1948,10 +1948,10 @@ public class DatabaseImpl implements Database
   }
   
   /**
-   * Copies the given InputStream to the given channel using the most
+   * Copies the given db InputStream to the given channel using the most
    * efficient means possible.
    */
-  static void transferFrom(FileChannel channel, InputStream in)
+  protected static void transferDbFrom(FileChannel channel, InputStream in)
     throws IOException
   {
     ReadableByteChannel readChannel = Channels.newChannel(in);
@@ -1991,7 +1991,7 @@ public class DatabaseImpl implements Database
     return pwdMask;
   }
 
-  static InputStream getResourceAsStream(String resourceName)
+  protected static InputStream getResourceAsStream(String resourceName)
     throws IOException
   {
     InputStream stream = DatabaseImpl.class.getClassLoader()

src/main/java/com/healthmarketscience/jackcess/util/CustomLinkResolver.java

@@ -0,0 +1,283 @@
+/*
+Copyright (c) 2017 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.util;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.util.Random;
+
+import com.healthmarketscience.jackcess.Database;
+import com.healthmarketscience.jackcess.Database.FileFormat;
+import com.healthmarketscience.jackcess.Table;
+import com.healthmarketscience.jackcess.impl.ByteUtil;
+import com.healthmarketscience.jackcess.impl.DatabaseImpl;
+import com.healthmarketscience.jackcess.impl.TableImpl;
+
+/**
+ * Utility base implementaton of LinkResolver which facilitates loading linked
+ * tables from files which are not access databases.  The LinkResolver API
+ * ultimately presents linked table information to the primary database using
+ * the jackcess {@link Database} and {@link Table} classes.  In order to
+ * consume linked tables in non-mdb files, they need to somehow be coerced
+ * into the appropriate form.  The approach taken by this utility is to make
+ * it easy to copy the external tables into a temporary mdb file for
+ * consumption by the primary database.
+ * <p>
+ * The primary features of this utility:
+ * <ul>
+ * <li>Supports custom behavior for non-mdb files and default behavior for mdb
+ *     files, see {@link #loadCustomFile}</li>
+ * <li>Temp db can be an actual file or entirely in memory</li>
+ * <li>Linked tables are loaded on-demand, see {@link #loadCustomTable}</li>
+ * <li>Temp db files will be automatically deleted on close</li>
+ * </ul>
+ *
+ * @author James Ahlborn
+ * @usage _intermediate_class_
+ */
+public abstract class CustomLinkResolver implements LinkResolver
+{
+  private static final Random DB_ID = new Random();
+
+  private static final String MEM_DB_PREFIX = "memdb_";
+  private static final String FILE_DB_PREFIX = "linkeddb_";
+
+  /** the default file format used for temp dbs */
+  public static final FileFormat DEFAULT_FORMAT = FileFormat.V2000;
+  /** temp dbs default to the filesystem, not in memory */
+  public static final boolean DEFAULT_IN_MEMORY = false;
+  /** temp dbs end up in the system temp dir by default */
+  public static final File DEFAULT_TEMP_DIR = null;
+
+  private final FileFormat _defaultFormat;
+  private final boolean _defaultInMemory;
+  private final File _defaultTempDir;
+
+  /**
+   * Creates a CustomLinkResolver using the default behavior for creating temp
+   * dbs, see {@link #DEFAULT_FORMAT}, {@link #DEFAULT_IN_MEMORY} and
+   * {@link #DEFAULT_TEMP_DIR}.
+   */
+  protected CustomLinkResolver() {
+    this(DEFAULT_FORMAT, DEFAULT_IN_MEMORY, DEFAULT_TEMP_DIR);
+  }
+
+  /**
+   * Creates a CustomLinkResolver with the given default behavior for creating
+   * temp dbs.
+   *
+   * @param defaultFormat the default format for the temp db
+   * @param defaultInMemory whether or not the temp db should be entirely in
+   *                        memory by default (while this will be faster, it
+   *                        should only be used if table data is expected to
+   *                        fit entirely in memory)
+   * @param defaultTempDir the default temp dir for a file based temp db
+   *                       ({@code null} for the system defaqult temp
+   *                       directory)
+   */
+  protected CustomLinkResolver(FileFormat defaultFormat, boolean defaultInMemory,
+                               File defaultTempDir)
+  {
+    _defaultFormat = defaultFormat;
+    _defaultInMemory = defaultInMemory;
+    _defaultTempDir = defaultTempDir;
+  }
+
+  /**
+   * Custom implementation is:
+   * <pre>
+   *   // attempt to load the linkeeFileName as a custom file
+   *   Object customFile = loadCustomFile(linkerDb, linkeeFileName);
+   *   
+   *   if(customFile != null) {
+   *     // this is a custom file, create and return relevant temp db
+   *     return createTempDb(customFile, _defaultFormat, _defaultInMemory, 
+   *                         _defaultTempDir);
+   *   }
+   *   
+   *   // not a custmom file, load using the default behavior
+   *   return LinkResolver.DEFAULT.resolveLinkedDatabase(linkerDb, linkeeFileName);
+   * </pre>
+   * 
+   * @see #loadCustomFile
+   * @see #createTempDb
+   * @see LinkResolver#DEFAULT
+   */
+  public Database resolveLinkedDatabase(Database linkerDb, String linkeeFileName)
+    throws IOException 
+  {
+    Object customFile = loadCustomFile(linkerDb, linkeeFileName);
+    if(customFile != null) {
+      return createTempDb(customFile, _defaultFormat, _defaultInMemory, 
+                          _defaultTempDir);
+    }
+    return LinkResolver.DEFAULT.resolveLinkedDatabase(linkerDb, linkeeFileName);
+  }
+
+  /**
+   * Creates a temporary database for holding the table data from
+   * linkeeFileName.
+   *
+   * @param customFile custom file state returned from {@link #loadCustomFile}
+   * @param format the access format for the temp db
+   * @param inMemory whether or not the temp db should be entirely in memory
+   *                 (while this will be faster, it should only be used if
+   *                 table data is expected to fit entirely in memory)
+   * @param tempDir the temp dir for a file based temp db ({@code null} for
+   *                the system default temp directory)
+   *
+   * @return the temp db for holding the linked table info
+   */
+  protected Database createTempDb(Object customFile, FileFormat format, 
+                                  boolean inMemory, File tempDir)
+    throws IOException
+  {
+    File dbFile = null;
+    FileChannel channel = null;
+    boolean success = false;
+    try {
+
+      if(inMemory) {
+        dbFile = new File(MEM_DB_PREFIX + DB_ID.nextLong() +
+                          format.getFileExtension());
+        channel = MemFileChannel.newChannel();
+      } else {
+        dbFile = File.createTempFile(FILE_DB_PREFIX, format.getFileExtension(),
+                                     tempDir);
+        channel = new RandomAccessFile(dbFile, DatabaseImpl.RW_CHANNEL_MODE)
+          .getChannel();
+      }
+
+      TempDatabaseImpl.initDbChannel(channel, format);
+      TempDatabaseImpl db = new TempDatabaseImpl(this, customFile, dbFile, 
+                                                 channel, format);
+      success = true;
+      return db;
+
+    } finally {
+      if(!success) {
+        ByteUtil.closeQuietly(channel);
+        deleteDbFile(dbFile);
+        closeCustomFile(customFile);
+      }
+    }
+  }
+
+  private static void deleteDbFile(File dbFile) {
+    if((dbFile != null) && (dbFile.getName().startsWith(FILE_DB_PREFIX))) {
+      dbFile.delete();
+    }
+  }
+
+  private static void closeCustomFile(Object customFile) {
+    if(customFile instanceof Closeable) {
+      ByteUtil.closeQuietly((Closeable)customFile);
+    }
+  }
+  
+  /**
+   * Called by {@link #resolveLinkedDatabase} to determine whether the
+   * linkeeFileName should be treated as a custom file (thus utiliziing a temp
+   * db) or a normal access db (loaded via the default behavior).  Loads any
+   * state necessary for subsequently loading data from linkeeFileName.
+   * <p>
+   * The returned custom file state object will be maintained with the temp db
+   * and passed to {@link #loadCustomTable} whenever a new table needs to be
+   * loaded.  Also, if this object is {@link Closeable}, it will be closed
+   * with the temp db.
+   *
+   * @param linkerDb the primary database in which the link is defined
+   * @param linkeeFileName the name of the linked file
+   *
+   * @return non-{@code null} if linkeeFileName should be treated as a custom
+   *         file (using a temp db) or {@code null} if it should be treated as
+   *         a normal access db.
+   */
+  protected abstract Object loadCustomFile(
+      Database linkerDb, String linkeeFileName) throws IOException;
+
+  /**
+   * Called by an instance of a temp db when a missing table is first requested.
+   *
+   * @param tempDb the temp db instance which should be populated with the
+   *               relevant table info for the given tableName
+   * @param customFile custom file state returned from {@link #loadCustomFile}
+   * @param tableName the name of the table which is requested from the linked
+   *                  file
+   *
+   * @return {@code true} if the table was available in the linked file,
+   *         {@code false} otherwise
+   */
+  protected abstract boolean loadCustomTable(
+      Database tempDb, Object customFile, String tableName)
+    throws IOException;
+
+
+  /**
+   * Subclass of DatabaseImpl which allows us to load tables "on demand" as
+   * well as delete the temporary db on close.
+   */
+  private static class TempDatabaseImpl extends DatabaseImpl
+  {
+    private final CustomLinkResolver _resolver;
+    private final Object _customFile;
+
+    protected TempDatabaseImpl(CustomLinkResolver resolver, Object customFile,
+                               File file, FileChannel channel, 
+                               FileFormat fileFormat)
+      throws IOException
+    {
+      super(file, channel, true, false, fileFormat, null, null, null);
+      _resolver = resolver;
+      _customFile = customFile;
+    }
+
+    @Override
+    protected TableImpl getTable(String name, boolean includeSystemTables) 
+      throws IOException 
+    {
+      TableImpl table = super.getTable(name, includeSystemTables);
+      if((table == null) && 
+         _resolver.loadCustomTable(this, _customFile, name)) {
+        table = super.getTable(name, includeSystemTables);
+      }
+      return table;
+    }
+
+    @Override
+    public void close() throws IOException {
+      try {
+        super.close();
+      } finally {
+        deleteDbFile(getFile());
+        closeCustomFile(_customFile);
+      }
+    }
+
+    static FileChannel initDbChannel(FileChannel channel, FileFormat format)
+      throws IOException
+    {
+      FileFormatDetails details = getFileFormatDetails(format);
+      transferDbFrom(channel, getResourceAsStream(details.getEmptyFilePath()));
+      return channel;
+    }
+  }
+
+}

src/test/java/com/healthmarketscience/jackcess/TestUtil.java

@@ -85,7 +85,8 @@ public class TestUtil
   public static Database open(FileFormat fileFormat, File file, boolean inMem) 
     throws Exception 
   {
-    FileChannel channel = (inMem ? MemFileChannel.newChannel(file, "rw") 
+    FileChannel channel = (inMem ? MemFileChannel.newChannel(
+                               file, DatabaseImpl.RW_CHANNEL_MODE) 
                            : null);
     final Database db = new DatabaseBuilder(file).setReadOnly(true)
       .setAutoSync(getTestAutoSync()).setChannel(channel).open();
@@ -134,7 +135,7 @@ public class TestUtil
           .getResourceAsStream("emptyJet4.mdb");
         File f = createTempFile(keep);
         if (channel != null) {
-          JetFormatTest.transferFrom(channel, inStream);
+          JetFormatTest.transferDbFrom(channel, inStream);
         } else {
           ByteUtil.copy(inStream, outStream = new FileOutputStream(f));
           outStream.close();

src/test/java/com/healthmarketscience/jackcess/impl/JetFormatTest.java

@@ -265,9 +265,9 @@ public class JetFormatTest extends TestCase {
     }
   }
 
-  public static void transferFrom(FileChannel channel, InputStream in)
+  public static void transferDbFrom(FileChannel channel, InputStream in)
     throws IOException
   {
-    DatabaseImpl.transferFrom(channel, in);
+    DatabaseImpl.transferDbFrom(channel, in);
   }
 }

src/test/java/com/healthmarketscience/jackcess/util/CustomLinkResolverTest.java

@@ -0,0 +1,159 @@
+/*
+Copyright (c) 2017 James Ahlborn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package com.healthmarketscience.jackcess.util;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import com.healthmarketscience.jackcess.ColumnBuilder;
+import com.healthmarketscience.jackcess.DataType;
+import com.healthmarketscience.jackcess.Database;
+import com.healthmarketscience.jackcess.Database.FileFormat;
+import com.healthmarketscience.jackcess.Table;
+import com.healthmarketscience.jackcess.TableBuilder;
+import junit.framework.TestCase;
+import static com.healthmarketscience.jackcess.TestUtil.*;
+import static com.healthmarketscience.jackcess.impl.JetFormatTest.*;
+
+/**
+ *
+ * @author James Ahlborn
+ */
+public class CustomLinkResolverTest extends TestCase
+{
+
+  public CustomLinkResolverTest(String name) {
+    super(name);
+  }
+
+  public void testCustomLinkResolver() throws Exception {
+    for(final FileFormat fileFormat : SUPPORTED_FILEFORMATS) {
+      Database db = create(fileFormat);
+
+      db.setLinkResolver(new TestLinkResolver());
+
+      db.createLinkedTable("Table1", "testFile1.txt", "Table1");
+      db.createLinkedTable("Table2", "testFile2.txt", "OtherTable2");
+      db.createLinkedTable("Table3", "missingFile3.txt", "MissingTable3");
+      db.createLinkedTable("Table4", "testFile2.txt", "MissingTable4");
+
+      Table t1 = db.getTable("Table1");
+      assertNotNull(t1);
+      assertNotSame(db, t1.getDatabase());
+      
+      assertTable(createExpectedTable(createExpectedRow("id", 0,
+                                                        "data1", "row0"),
+                                      createExpectedRow("id", 1,
+                                                        "data1", "row1"),
+                                      createExpectedRow("id", 2,
+                                                        "data1", "row2")),
+                  t1);
+
+      Table t2 = db.getTable("Table2");
+      assertNotNull(t2);
+      assertNotSame(db, t2.getDatabase());
+
+      assertTable(createExpectedTable(createExpectedRow("id", 3,
+                                                        "data2", "row3"),
+                                      createExpectedRow("id", 4,
+                                                        "data2", "row4"),
+                                      createExpectedRow("id", 5,
+                                                        "data2", "row5")),
+                  t2);
+
+      assertNull(db.getTable("Table4"));
+
+      try {
+        db.getTable("Table3");
+        fail("FileNotFoundException should have been thrown");
+      } catch(FileNotFoundException e) {
+        // success
+      }
+
+      db.close();
+    }
+  }
+
+  private static class TestLinkResolver extends CustomLinkResolver
+  {
+    private TestLinkResolver()
+    {
+      super(DEFAULT_FORMAT, true, DEFAULT_TEMP_DIR);
+    }
+
+    @Override
+    protected Object loadCustomFile(
+        Database linkerDb, String linkeeFileName) throws IOException
+    {
+      return (("testFile1.txt".equals(linkeeFileName) ||
+               "testFile2.txt".equals(linkeeFileName)) ? 
+              linkeeFileName : null);
+    }
+
+    @Override
+    protected boolean loadCustomTable(
+        Database tempDb, Object customFile, String tableName)
+      throws IOException
+    {
+      if("Table1".equals(tableName)) {
+
+        assertEquals("testFile1.txt", customFile);
+        Table t = new TableBuilder(tableName)
+          .addColumn(new ColumnBuilder("id", DataType.LONG))
+          .addColumn(new ColumnBuilder("data1", DataType.TEXT))
+          .toTable(tempDb);
+
+        for(int i = 0; i < 3; ++i) {
+          t.addRow(i, "row" + i);
+        }
+        
+        return true;
+
+      } else if("OtherTable2".equals(tableName)) {
+
+        assertEquals("testFile2.txt", customFile);
+        Table t = new TableBuilder(tableName)
+          .addColumn(new ColumnBuilder("id", DataType.LONG))
+          .addColumn(new ColumnBuilder("data2", DataType.TEXT))
+          .toTable(tempDb);
+
+        for(int i = 3; i < 6; ++i) {
+          t.addRow(i, "row" + i);
+        }
+        
+        return true;
+
+      } else if("Table4".equals(tableName)) {
+
+        assertEquals("testFile2.txt", customFile);
+        return false;
+      }
+
+      return false;
+    }
+
+    @Override
+    protected Database createTempDb(Object customFile, FileFormat format,
+                                    boolean inMemory, File tempDir)
+      throws IOException
+    {
+      inMemory = "testFile1.txt".equals(customFile);
+      return super.createTempDb(customFile, format, inMemory, tempDir);
+    }
+  }
+}