Browse Source

Add option to DatabaseBuilder for ignoring broken system catalog indexes. Fixes #46

git-svn-id: https://svn.code.sf.net/p/jackcess/code/jackcess/trunk@1395 f203690c-595d-4dc9-a70b-905162fa7fd2
tags/jackcess-4.0.5
James Ahlborn 1 year ago
parent
commit
b02d1df66c

+ 8
- 0
src/changes/changes.xml View File

<author email="javajedi@users.sf.net">Tim McCune</author> <author email="javajedi@users.sf.net">Tim McCune</author>
</properties> </properties>
<body> <body>
<release version="4.0.5" date="TBD">
<action dev="jahlborn" type="add" system="SourceForge2Features"
issue="46">
Add option to DatabaseBuilder for ignoring broken system catalog
indexes. This is a workaround to allow jackcess to read tables from
the database even if the index is non-functional..
</action>
</release>
<release version="4.0.4" date="2022-10-29"> <release version="4.0.4" date="2022-10-29">
<action dev="jahlborn" type="update"> <action dev="jahlborn" type="update">
Update parent pom to fix release process. Update parent pom to fix release process.

+ 14
- 2
src/main/java/com/healthmarketscience/jackcess/DatabaseBuilder.java View File

private Map<String,PropertyMap.Property> _summaryProps; private Map<String,PropertyMap.Property> _summaryProps;
/** database user-defined (if any) */ /** database user-defined (if any) */
private Map<String,PropertyMap.Property> _userProps; private Map<String,PropertyMap.Property> _userProps;

/** flag indicating that the system catalog index is borked */
private boolean _ignoreBrokenSystemCatalogIndex;


public DatabaseBuilder() { public DatabaseBuilder() {
this((Path)null); this((Path)null);
return props; return props;
} }


/**
* Sets flag which, if {@code true}, will make the database ignore the index
* on the system catalog when looking up tables. This will make table
* retrieval slower, but can be used to workaround broken indexes.
*/
public DatabaseBuilder setIgnoreBrokenSystemCatalogIndex(boolean ignore) {
_ignoreBrokenSystemCatalogIndex = ignore;
return this;
}

/** /**
* Opens an existingnew Database using the configured information. * Opens an existingnew Database using the configured information.
*/ */
public Database open() throws IOException { public Database open() throws IOException {
return DatabaseImpl.open(_mdbFile, _readOnly, _channel, _autoSync, _charset, return DatabaseImpl.open(_mdbFile, _readOnly, _channel, _autoSync, _charset,
_timeZone, _codecProvider);
_timeZone, _codecProvider,
_ignoreBrokenSystemCatalogIndex);
} }


/** /**

+ 27
- 14
src/main/java/com/healthmarketscience/jackcess/impl/DatabaseImpl.java View File

public static DatabaseImpl open( public static DatabaseImpl open(
Path mdbFile, boolean readOnly, FileChannel channel, Path mdbFile, boolean readOnly, FileChannel channel,
boolean autoSync, Charset charset, TimeZone timeZone, boolean autoSync, Charset charset, TimeZone timeZone,
CodecProvider provider)
CodecProvider provider, boolean ignoreSystemCatalogIndex)
throws IOException throws IOException
{ {
boolean closeChannel = false; boolean closeChannel = false;


DatabaseImpl db = new DatabaseImpl(mdbFile, channel, closeChannel, autoSync, DatabaseImpl db = new DatabaseImpl(mdbFile, channel, closeChannel, autoSync,
null, charset, timeZone, provider, null, charset, timeZone, provider,
readOnly);
readOnly, ignoreSystemCatalogIndex);
success = true; success = true;
return db; return db;


channel.force(true); channel.force(true);
DatabaseImpl db = new DatabaseImpl(mdbFile, channel, closeChannel, autoSync, DatabaseImpl db = new DatabaseImpl(mdbFile, channel, closeChannel, autoSync,
fileFormat, charset, timeZone, null, fileFormat, charset, timeZone, null,
false);
false, false);
success = true; success = true;
return db; return db;
} finally { } finally {
protected DatabaseImpl(Path file, FileChannel channel, boolean closeChannel, protected DatabaseImpl(Path file, FileChannel channel, boolean closeChannel,
boolean autoSync, FileFormat fileFormat, Charset charset, boolean autoSync, FileFormat fileFormat, Charset charset,
TimeZone timeZone, CodecProvider provider, TimeZone timeZone, CodecProvider provider,
boolean readOnly)
boolean readOnly, boolean ignoreSystemCatalogIndex)
throws IOException throws IOException
{ {
_file = file; _file = file;
// needed // needed
_pageChannel.initialize(this, provider); _pageChannel.initialize(this, provider);
_buffer = _pageChannel.createPageBuffer(); _buffer = _pageChannel.createPageBuffer();
readSystemCatalog();
readSystemCatalog(ignoreSystemCatalogIndex);
} }


@Override @Override
/** /**
* Read the system catalog * Read the system catalog
*/ */
private void readSystemCatalog() throws IOException {
private void readSystemCatalog(boolean ignoreSystemCatalogIndex)
throws IOException {
_systemCatalog = loadTable(TABLE_SYSTEM_CATALOG, PAGE_SYSTEM_CATALOG, _systemCatalog = loadTable(TABLE_SYSTEM_CATALOG, PAGE_SYSTEM_CATALOG,
SYSTEM_OBJECT_FLAGS, TYPE_TABLE); SYSTEM_OBJECT_FLAGS, TYPE_TABLE);


try {
_tableFinder = new DefaultTableFinder(
_systemCatalog.newCursor()
if(!ignoreSystemCatalogIndex) {
try {
_tableFinder = new DefaultTableFinder(
_systemCatalog.newCursor()
.setIndexByColumnNames(CAT_COL_PARENT_ID, CAT_COL_NAME) .setIndexByColumnNames(CAT_COL_PARENT_ID, CAT_COL_NAME)
.setColumnMatcher(CaseInsensitiveColumnMatcher.INSTANCE) .setColumnMatcher(CaseInsensitiveColumnMatcher.INSTANCE)
.toIndexCursor()); .toIndexCursor());
} catch(IllegalArgumentException e) {
} catch(IllegalArgumentException e) {
if(LOG.isDebugEnabled()) {
LOG.debug(withErrorContext(
"Could not find expected index on table " +
_systemCatalog.getName()));
}
// use table scan instead
_tableFinder = new FallbackTableFinder(
_systemCatalog.newCursor()
.setColumnMatcher(CaseInsensitiveColumnMatcher.INSTANCE)
.toCursor());
}
} else {
if(LOG.isDebugEnabled()) { if(LOG.isDebugEnabled()) {
LOG.debug(withErrorContext( LOG.debug(withErrorContext(
"Could not find expected index on table " +
_systemCatalog.getName()));
"Ignoring index on table " + _systemCatalog.getName()));
} }
// use table scan instead // use table scan instead
_tableFinder = new FallbackTableFinder( _tableFinder = new FallbackTableFinder(
_systemCatalog.newCursor() _systemCatalog.newCursor()
.setColumnMatcher(CaseInsensitiveColumnMatcher.INSTANCE)
.toCursor());
.setColumnMatcher(CaseInsensitiveColumnMatcher.INSTANCE)
.toCursor());
} }


_tableParentId = _tableFinder.findObjectId(DB_PARENT_ID, _tableParentId = _tableFinder.findObjectId(DB_PARENT_ID,

+ 4
- 2
src/main/java/com/healthmarketscience/jackcess/impl/IndexData.java View File



/** /**
* Used by unit tests to validate the internal status of the index. * Used by unit tests to validate the internal status of the index.
* @param forceLoad if {@code false} only validate currently loaded index
* data pages, otherwise, load and validate all index pages
* @usage _advanced_method_ * @usage _advanced_method_
*/ */
public void validate() throws IOException {
_pageCache.validate();
public void validate(boolean forceLoad) throws IOException {
_pageCache.validate(forceLoad);
} }


/** /**

+ 269
- 206
src/main/java/com/healthmarketscience/jackcess/impl/IndexPageCache.java View File

import java.util.AbstractList; import java.util.AbstractList;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Queue;
import java.util.RandomAccess; import java.util.RandomAccess;
import java.util.Set;


import static com.healthmarketscience.jackcess.impl.IndexData.*; import static com.healthmarketscience.jackcess.impl.IndexData.*;
import com.healthmarketscience.jackcess.impl.IndexData.DataPage;
import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringBuilder;


/** /**
/** max number of pages to cache (unless a write operation is in /** max number of pages to cache (unless a write operation is in
progress) */ progress) */
private static final int MAX_CACHE_SIZE = 25; private static final int MAX_CACHE_SIZE = 25;
/** the index whose pages this cache is managing */ /** the index whose pages this cache is managing */
private final IndexData _indexData; private final IndexData _indexData;
/** the root page for the index */ /** the root page for the index */
/** the currently modified index pages */ /** the currently modified index pages */
private final List<CacheDataPage> _modifiedPages = private final List<CacheDataPage> _modifiedPages =
new ArrayList<CacheDataPage>(); new ArrayList<CacheDataPage>();
public IndexPageCache(IndexData indexData) { public IndexPageCache(IndexData indexData) {
_indexData = indexData; _indexData = indexData;
} }
public IndexData getIndexData() { public IndexData getIndexData() {
return _indexData; return _indexData;
} }
public PageChannel getPageChannel() { public PageChannel getPageChannel() {
return getIndexData().getPageChannel(); return getIndexData().getPageChannel();
} }
/** /**
* Sets the root page for this index, must be called before normal usage. * Sets the root page for this index, must be called before normal usage.
* *
// root page has no parent // root page has no parent
_rootPage.initParentPage(INVALID_INDEX_PAGE_NUMBER, false); _rootPage.initParentPage(INVALID_INDEX_PAGE_NUMBER, false);
} }
/** /**
* Writes any outstanding changes for this index to the file. * Writes any outstanding changes for this index to the file.
*/ */
} }
} }
} }
/** /**
* Prepares any non-empty modified pages for writing as the second pass * Prepares any non-empty modified pages for writing as the second pass
* during a {@link #write} call. Updates entry prefixes, promotes/demotes * during a {@link #write} call. Updates entry prefixes, promotes/demotes
if(dpMain.hasChildTail()) { if(dpMain.hasChildTail()) {
if(size == 1) { if(size == 1) {
demoteTail(cacheDataPage); demoteTail(cacheDataPage);
}
}
} else { } else {
if(size > 1) { if(size > 1) {
promoteTail(cacheDataPage); promoteTail(cacheDataPage);
} }
} }
} }
// look for pages with more entries than can fit on a page // look for pages with more entries than can fit on a page
if(cacheDataPage.getTotalEntrySize() > maxPageEntrySize) { if(cacheDataPage.getTotalEntrySize() > maxPageEntrySize) {


} }
} }
} }
} while(splitPages); } while(splitPages);
} }


DataPageMain main = getDataPage(pageNumber); DataPageMain main = getDataPage(pageNumber);
return((main != null) ? new CacheDataPage(main) : null); return((main != null) ? new CacheDataPage(main) : null);
} }
/** /**
* Returns a DataPageMain for the given page number, may be {@code null} if * Returns a DataPageMain for the given page number, may be {@code null} if
* the given page number is invalid. Loads the given page if necessary. * the given page number is invalid. Loads the given page if necessary.
getIndexData().writeDataPage(cacheDataPage); getIndexData().writeDataPage(cacheDataPage);


// lastly, mark the page as no longer modified // lastly, mark the page as no longer modified
cacheDataPage._extra._modified = false;
cacheDataPage._extra._modified = false;
} }
/** /**
* Deletes the given index page from the file (clears the page). * Deletes the given index page from the file (clears the page).
*/ */


// discard from our cache // discard from our cache
_dataPages.remove(cacheDataPage._main._pageNumber); _dataPages.remove(cacheDataPage._main._pageNumber);
// lastly, mark the page as no longer modified // lastly, mark the page as no longer modified
cacheDataPage._extra._modified = false;
cacheDataPage._extra._modified = false;
} }
/** /**
* Reads the given index page from the file. * Reads the given index page from the file.
*/ */


// associate the extra info with the main data page // associate the extra info with the main data page
dataPage.setExtra(extra); dataPage.setExtra(extra);
return cacheDataPage; return cacheDataPage;
}
}


/** /**
* Removes the entry with the given index from the given page. * Removes the entry with the given index from the given page.
{ {
updateEntry(cacheDataPage, entryIdx, newEntry, UpdateType.ADD); updateEntry(cacheDataPage, entryIdx, newEntry, UpdateType.ADD);
} }
/** /**
* Updates the entries on the given page according to the given updateType. * Updates the entries on the given page according to the given updateType.
* *
CacheDataPage parentDataPage = (!dpMain.isRoot() ? CacheDataPage parentDataPage = (!dpMain.isRoot() ?
new CacheDataPage(dpMain.getParentPage()) : new CacheDataPage(dpMain.getParentPage()) :
null); null);
Entry oldLastEntry = dpExtra._entryView.getLast(); Entry oldLastEntry = dpExtra._entryView.getLast();
Entry oldEntry = null; Entry oldEntry = null;
int entrySizeDiff = 0; int entrySizeDiff = 0;
} }


boolean updateLast = (oldLastEntry != dpExtra._entryView.getLast()); boolean updateLast = (oldLastEntry != dpExtra._entryView.getLast());
// child tail entry updates do not modify the page // child tail entry updates do not modify the page
if(!updateLast || !dpMain.hasChildTail()) { if(!updateLast || !dpMain.hasChildTail()) {
dpExtra._totalEntrySize += entrySizeDiff; dpExtra._totalEntrySize += entrySizeDiff;
return oldEntry; return oldEntry;
} }


// determine if we need to update our parent page
// determine if we need to update our parent page
if(!updateLast || dpMain.isRoot()) { if(!updateLast || dpMain.isRoot()) {
// no parent // no parent
return oldEntry; return oldEntry;
"Empty page but size is not 0? " + dpExtra._totalEntrySize + ", " + "Empty page but size is not 0? " + dpExtra._totalEntrySize + ", " +
cacheDataPage)); cacheDataPage));
} }
if(dpMain.isRoot()) { if(dpMain.isRoot()) {
// clear out this page (we don't actually remove it) // clear out this page (we don't actually remove it)
dpExtra._entryPrefix = EMPTY_PREFIX; dpExtra._entryPrefix = EMPTY_PREFIX;


Integer prevPageNumber = dpMain._prevPageNumber; Integer prevPageNumber = dpMain._prevPageNumber;
Integer nextPageNumber = dpMain._nextPageNumber; Integer nextPageNumber = dpMain._nextPageNumber;
DataPageMain prevMain = dpMain.getPrevPage(); DataPageMain prevMain = dpMain.getPrevPage();
if(prevMain != null) { if(prevMain != null) {
setModified(new CacheDataPage(prevMain)); setModified(new CacheDataPage(prevMain));
updateParentEntry(parentDataPage, childDataPage, null, updateParentEntry(parentDataPage, childDataPage, null,
childExtra._entryView.getLast(), UpdateType.ADD); childExtra._entryView.getLast(), UpdateType.ADD);
} }
/** /**
* Replaces the entry for the given child page in the given parent page. * Replaces the entry for the given child page in the given parent page.
* *
updateParentEntry(parentDataPage, childDataPage, oldEntry, updateParentEntry(parentDataPage, childDataPage, oldEntry,
childExtra._entryView.getLast(), UpdateType.REPLACE); childExtra._entryView.getLast(), UpdateType.REPLACE);
} }
/** /**
* Updates the entry for the given child page in the given parent page * Updates the entry for the given child page in the given parent page
* according to the given updateType. * according to the given updateType.


boolean expectFound = true; boolean expectFound = true;
int idx = 0; int idx = 0;
switch(upType) { switch(upType) {
case ADD: case ADD:
expectFound = false; expectFound = false;
case REMOVE: case REMOVE:
idx = parentExtra._entryView.find(oldEntry); idx = parentExtra._entryView.find(oldEntry);
break; break;
default: default:
throw new RuntimeException(withErrorContext( throw new RuntimeException(withErrorContext(
"unknown update type " + upType)); "unknown update type " + upType));
} }
if(idx < 0) { if(idx < 0) {
if(expectFound) { if(expectFound) {
throw new IllegalStateException(withErrorContext( throw new IllegalStateException(withErrorContext(
private void updateParentTail(CacheDataPage parentDataPage, private void updateParentTail(CacheDataPage parentDataPage,
CacheDataPage childDataPage, CacheDataPage childDataPage,
UpdateType upType) UpdateType upType)
throws IOException
{ {
DataPageMain parentMain = parentDataPage._main; DataPageMain parentMain = parentDataPage._main;


parentMain._childTailPageNumber = newChildTailPageNumber; parentMain._childTailPageNumber = newChildTailPageNumber;
} }
} }
/** /**
* Verifies that the given entry type (node/leaf) is valid for the given * Verifies that the given entry type (node/leaf) is valid for the given
* page (node/leaf). * page (node/leaf).
DataPageExtra origExtra = origDataPage._extra; DataPageExtra origExtra = origDataPage._extra;


setModified(origDataPage); setModified(origDataPage);
int numEntries = origExtra._entries.size(); int numEntries = origExtra._entries.size();
if(numEntries < 2) { if(numEntries < 2) {
throw new IllegalStateException(withErrorContext( throw new IllegalStateException(withErrorContext(
"Cannot split page with less than 2 entries " + origDataPage)); "Cannot split page with less than 2 entries " + origDataPage));
} }
if(origMain.isRoot()) { if(origMain.isRoot()) {
// we can't split the root page directly, so we need to put another page // we can't split the root page directly, so we need to put another page
// between the root page and its sub-pages, and then split that page. // between the root page and its sub-pages, and then split that page.
// start mucking with our entries because our parent may use our entries. // start mucking with our entries because our parent may use our entries.
DataPageMain parentMain = origMain.getParentPage(); DataPageMain parentMain = origMain.getParentPage();
CacheDataPage parentDataPage = new CacheDataPage(parentMain); CacheDataPage parentDataPage = new CacheDataPage(parentMain);
// note, there are many, many ways this could be improved/tweaked. for // note, there are many, many ways this could be improved/tweaked. for
// now, we just want it to be functional... // now, we just want it to be functional...
// so, we will naively move half the entries from one page to a new page. // so, we will naively move half the entries from one page to a new page.
parentMain._pageNumber, origMain._leaf); parentMain._pageNumber, origMain._leaf);
DataPageMain newMain = newDataPage._main; DataPageMain newMain = newDataPage._main;
DataPageExtra newExtra = newDataPage._extra; DataPageExtra newExtra = newDataPage._extra;
List<Entry> headEntries = List<Entry> headEntries =
origExtra._entries.subList(0, ((numEntries + 1) / 2)); origExtra._entries.subList(0, ((numEntries + 1) / 2));




// insert this new page between the old page and any previous page // insert this new page between the old page and any previous page
addToPeersBefore(newDataPage, origDataPage); addToPeersBefore(newDataPage, origDataPage);
if(!newMain._leaf) { if(!newMain._leaf) {
// reparent the children pages of the new page // reparent the children pages of the new page
reparentChildren(newDataPage); reparentChildren(newDataPage);
* split. * split.
* *
* @param rootDataPage the root data page * @param rootDataPage the root data page
*
*
* @return the newly created page nested under the root page * @return the newly created page nested under the root page
*/ */
private CacheDataPage nestRootDataPage(CacheDataPage rootDataPage) private CacheDataPage nestRootDataPage(CacheDataPage rootDataPage)
throw new IllegalArgumentException(withErrorContext( throw new IllegalArgumentException(withErrorContext(
"should be called with root, duh")); "should be called with root, duh"));
} }
CacheDataPage newDataPage = CacheDataPage newDataPage =
allocateNewCacheDataPage(rootMain._pageNumber, rootMain._leaf); allocateNewCacheDataPage(rootMain._pageNumber, rootMain._leaf);
DataPageMain newMain = newDataPage._main; DataPageMain newMain = newDataPage._main;
// we need to re-parent all the child pages // we need to re-parent all the child pages
reparentChildren(newDataPage); reparentChildren(newDataPage);
} }
// clear the root page // clear the root page
rootMain._leaf = false; rootMain._leaf = false;
rootMain._childTailPageNumber = INVALID_INDEX_PAGE_NUMBER; rootMain._childTailPageNumber = INVALID_INDEX_PAGE_NUMBER;


return newDataPage; return newDataPage;
} }
/** /**
* Allocates a new index page with the given parent page and type. * Allocates a new index page with the given parent page and type.
* *
newMain._nextPageNumber = origMain._pageNumber; newMain._nextPageNumber = origMain._pageNumber;
newMain._prevPageNumber = origMain._prevPageNumber; newMain._prevPageNumber = origMain._prevPageNumber;
origMain._prevPageNumber = newMain._pageNumber; origMain._prevPageNumber = newMain._pageNumber;
if(prevMain != null) { if(prevMain != null) {
setModified(new CacheDataPage(prevMain)); setModified(new CacheDataPage(prevMain));
prevMain._nextPageNumber = newMain._pageNumber; prevMain._nextPageNumber = newMain._pageNumber;
nextMain._prevPageNumber = INVALID_INDEX_PAGE_NUMBER; nextMain._prevPageNumber = INVALID_INDEX_PAGE_NUMBER;
dpMain._nextPageNumber = INVALID_INDEX_PAGE_NUMBER; dpMain._nextPageNumber = INVALID_INDEX_PAGE_NUMBER;
} }
/** /**
* Sets the parent info for the children of the given page to the given * Sets the parent info for the children of the given page to the given
* page. * page.
* @param cacheDataPage the page whose children need to be updated * @param cacheDataPage the page whose children need to be updated
*/ */
private void reparentChildren(CacheDataPage cacheDataPage) private void reparentChildren(CacheDataPage cacheDataPage)
throws IOException
{ {
DataPageMain dpMain = cacheDataPage._main; DataPageMain dpMain = cacheDataPage._main;
DataPageExtra dpExtra = cacheDataPage._extra; DataPageExtra dpExtra = cacheDataPage._extra;
DataPageExtra dpExtra = cacheDataPage._extra; DataPageExtra dpExtra = cacheDataPage._extra;


setModified(cacheDataPage); setModified(cacheDataPage);
DataPageMain tailMain = dpMain.getChildTailPage(); DataPageMain tailMain = dpMain.getChildTailPage();
CacheDataPage tailDataPage = new CacheDataPage(tailMain); CacheDataPage tailDataPage = new CacheDataPage(tailMain);


Entry tailEntry = dpExtra._entryView.demoteTail(); Entry tailEntry = dpExtra._entryView.demoteTail();
dpExtra._totalEntrySize += tailEntry.size(); dpExtra._totalEntrySize += tailEntry.size();
dpExtra._entryPrefix = EMPTY_PREFIX; dpExtra._entryPrefix = EMPTY_PREFIX;
tailMain.setParentPage(dpMain._pageNumber, false); tailMain.setParentPage(dpMain._pageNumber, false);
} }
/** /**
* Makes the last normal entry of the given page the tail entry on that * Makes the last normal entry of the given page the tail entry on that
* page, done when there are multiple entries on a page and no tail entry. * page, done when there are multiple entries on a page and no tail entry.
DataPageExtra dpExtra = cacheDataPage._extra; DataPageExtra dpExtra = cacheDataPage._extra;


setModified(cacheDataPage); setModified(cacheDataPage);
DataPageMain lastMain = dpMain.getChildPage(dpExtra._entryView.getLast()); DataPageMain lastMain = dpMain.getChildPage(dpExtra._entryView.getLast());
CacheDataPage lastDataPage = new CacheDataPage(lastMain); CacheDataPage lastDataPage = new CacheDataPage(lastMain);




lastMain.setParentPage(dpMain._pageNumber, true); lastMain.setParentPage(dpMain._pageNumber, true);
} }
/** /**
* Finds the index page on which the given entry does or should reside. * Finds the index page on which the given entry does or should reside.
* *
// nowhere to go from here // nowhere to go from here
return new CacheDataPage(curPage); return new CacheDataPage(curPage);
} }
DataPageExtra extra = curPage.getExtra(); DataPageExtra extra = curPage.getExtra();


// need to descend // need to descend
{ {
byte[] b1 = e1.getEntryBytes(); byte[] b1 = e1.getEntryBytes();
byte[] b2 = e2.getEntryBytes(); byte[] b2 = e2.getEntryBytes();
int maxLen = b1.length; int maxLen = b1.length;
byte[] prefix = b1; byte[] prefix = b1;
if(b1.length > b2.length) { if(b1.length > b2.length) {
maxLen = b2.length; maxLen = b2.length;
prefix = b2; prefix = b2;
} }
int len = 0; int len = 0;
while((len < maxLen) && (b1[len] == b2[len])) { while((len < maxLen) && (b1[len] == b2[len])) {
++len; ++len;
} }
if(len < prefix.length) { if(len < prefix.length) {
if(len == 0) { if(len == 0) {
return EMPTY_PREFIX; return EMPTY_PREFIX;
} }
// need new prefix // need new prefix
prefix = ByteUtil.copyOf(prefix, len); prefix = ByteUtil.copyOf(prefix, len);
} }
/** /**
* Used by unit tests to validate the internal status of the index. * Used by unit tests to validate the internal status of the index.
*/ */
void validate() throws IOException {
// copy the values as the validation methods might trigger map updates
for(DataPageMain dpMain : new ArrayList<DataPageMain>(_dataPages.values())) {
DataPageExtra dpExtra = dpMain.getExtra();
validateEntries(dpExtra);
validateChildren(dpMain, dpExtra);
validatePeers(dpMain);
}
}

/**
* Validates the entries for an index page
*
* @param dpExtra the entries to validate
*/
private void validateEntries(DataPageExtra dpExtra) throws IOException {
int entrySize = 0;
Entry prevEntry = FIRST_ENTRY;
for(Entry e : dpExtra._entries) {
entrySize += e.size();
if(prevEntry.compareTo(e) >= 0) {
throw new IOException(withErrorContext(
"Unexpected order in index entries, " + prevEntry + " >= " + e));
}
prevEntry = e;
}
if(entrySize != dpExtra._totalEntrySize) {
throw new IllegalStateException(withErrorContext(
"Expected size " + entrySize +
" but was " + dpExtra._totalEntrySize));
}
}

/**
* Validates the children for an index page
*
* @param dpMain the index page
* @param dpExtra the child entries to validate
*/
private void validateChildren(DataPageMain dpMain,
DataPageExtra dpExtra) throws IOException {
int childTailPageNumber = dpMain._childTailPageNumber;
if(dpMain._leaf) {
if(childTailPageNumber != INVALID_INDEX_PAGE_NUMBER) {
throw new IllegalStateException(withErrorContext(
"Leaf page has tail " + dpMain));
}
return;
}
if((dpExtra._entryView.size() == 1) && dpMain.hasChildTail()) {
throw new IllegalStateException(withErrorContext(
"Single child is tail " + dpMain));
}
for(Entry e : dpExtra._entryView) {
validateEntryForPage(dpMain, e);
Integer subPageNumber = e.getSubPageNumber();
DataPageMain childMain = _dataPages.get(subPageNumber);
if(childMain != null) {
if(childMain._parentPageNumber != null) {
if(childMain._parentPageNumber != dpMain._pageNumber) {
throw new IllegalStateException(withErrorContext(
"Child's parent is incorrect " + childMain));
}
boolean expectTail = (subPageNumber == childTailPageNumber);
if(expectTail != childMain._tail) {
throw new IllegalStateException(withErrorContext(
"Child tail status incorrect " + childMain));
}
}
Entry lastEntry = childMain.getExtra()._entryView.getLast();
if(e.compareTo(lastEntry) != 0) {
throw new IllegalStateException(withErrorContext(
"Invalid entry " + e + " but child is " + lastEntry));
}
}
}
}

/**
* Validates the peer pages for an index page.
*
* @param dpMain the index page
*/
private void validatePeers(DataPageMain dpMain) throws IOException {
DataPageMain prevMain = _dataPages.get(dpMain._prevPageNumber);
if(prevMain != null) {
if(prevMain._nextPageNumber != dpMain._pageNumber) {
throw new IllegalStateException(withErrorContext(
"Prev page " + prevMain + " does not ref " + dpMain));
}
validatePeerStatus(dpMain, prevMain);
}
DataPageMain nextMain = _dataPages.get(dpMain._nextPageNumber);
if(nextMain != null) {
if(nextMain._prevPageNumber != dpMain._pageNumber) {
throw new IllegalStateException(withErrorContext(
"Next page " + nextMain + " does not ref " + dpMain));
}
validatePeerStatus(dpMain, nextMain);
}
void validate(boolean forceLoad) throws IOException {
new Validator(forceLoad).validate();
} }


/**
* Validates the given peer page against the given index page
*
* @param dpMain the index page
* @param peerMain the peer index page
*/
private void validatePeerStatus(DataPageMain dpMain, DataPageMain peerMain)
throws IOException
{
if(dpMain._leaf != peerMain._leaf) {
throw new IllegalStateException(withErrorContext(
"Mismatched peer status " + dpMain._leaf + " " + peerMain._leaf));
}
if(!dpMain._leaf) {
if((dpMain._parentPageNumber != null) &&
(peerMain._parentPageNumber != null) &&
((int)dpMain._parentPageNumber != (int)peerMain._parentPageNumber)) {
throw new IllegalStateException(withErrorContext(
"Mismatched node parents " + dpMain._parentPageNumber + " " +
peerMain._parentPageNumber));
}
}
}
/** /**
* Collects all the cache pages in the cache. * Collects all the cache pages in the cache.
* *
iter.remove(); iter.remove();
if(_dataPages.size() <= MAX_CACHE_SIZE) { if(_dataPages.size() <= MAX_CACHE_SIZE) {
break; break;
}
}
} }
} }
} }
@Override @Override
public String toString() { public String toString() {
ToStringBuilder sb = CustomToStringStyle.builder(this); ToStringBuilder sb = CustomToStringStyle.builder(this);
if(_rootPage == null) { if(_rootPage == null) {
sb.append("pages", "(uninitialized)"); sb.append("pages", "(uninitialized)");
} else {
} else {
sb.append("pages", collectPages(new ArrayList<Object>(), _rootPage)); sb.append("pages", collectPages(new ArrayList<Object>(), _rootPage));
} }
return sb.toString(); return sb.toString();
private String withErrorContext(String msg) { private String withErrorContext(String msg) {
return _indexData.withErrorContext(msg); return _indexData.withErrorContext(msg);
} }


/** /**
* Keeps track of the main info for an index page. * Keeps track of the main info for an index page.
public IndexPageCache getCache() { public IndexPageCache getCache() {
return IndexPageCache.this; return IndexPageCache.this;
} }
public boolean isRoot() { public boolean isRoot() {
return(this == _rootPage); return(this == _rootPage);
} }
public boolean isTail() throws IOException public boolean isTail() throws IOException
{ {
resolveParent(); resolveParent();
public boolean isChildTailPageNumber(int pageNumber) { public boolean isChildTailPageNumber(int pageNumber) {
return(_childTailPageNumber == pageNumber); return(_childTailPageNumber == pageNumber);
} }
public DataPageMain getParentPage() throws IOException public DataPageMain getParentPage() throws IOException
{ {
resolveParent(); resolveParent();
setParentPage(parentPageNumber, isTail); setParentPage(parentPageNumber, isTail);
} }
} }
public void setParentPage(Integer parentPageNumber, boolean isTail) { public void setParentPage(Integer parentPageNumber, boolean isTail) {
_parentPageNumber = parentPageNumber; _parentPageNumber = parentPageNumber;
_tail = isTail; _tail = isTail;
{ {
return IndexPageCache.this.getDataPage(_prevPageNumber); return IndexPageCache.this.getDataPage(_prevPageNumber);
} }
public DataPageMain getNextPage() throws IOException public DataPageMain getNextPage() throws IOException
{ {
return IndexPageCache.this.getDataPage(_nextPageNumber); return IndexPageCache.this.getDataPage(_nextPageNumber);
} }
public DataPageMain getChildPage(Entry e) throws IOException public DataPageMain getChildPage(Entry e) throws IOException
{ {
Integer childPageNumber = e.getSubPageNumber(); Integer childPageNumber = e.getSubPageNumber();
return getChildPage(childPageNumber, return getChildPage(childPageNumber,
isChildTailPageNumber(childPageNumber)); isChildTailPageNumber(childPageNumber));
} }
public DataPageMain getChildTailPage() throws IOException public DataPageMain getChildTailPage() throws IOException
{ {
return getChildPage(_childTailPageNumber, true); return getChildPage(_childTailPageNumber, true);
} }
return child; return child;
} }
public DataPageExtra getExtra() throws IOException public DataPageExtra getExtra() throws IOException
{ {
DataPageExtra extra = _extra.get(); DataPageExtra extra = _extra.get();
extra = readDataPage(_pageNumber)._extra; extra = readDataPage(_pageNumber)._extra;
setExtra(extra); setExtra(extra);
} }
return extra; return extra;
} }
public void setExtra(DataPageExtra extra) throws IOException public void setExtra(DataPageExtra extra) throws IOException
{ {
extra.setEntryView(this); extra.setEntryView(this);
public void setEntryView(DataPageMain main) throws IOException { public void setEntryView(DataPageMain main) throws IOException {
_entryView = new EntryListView(main, this); _entryView = new EntryListView(main, this);
} }
public void updateEntryPrefix() { public void updateEntryPrefix() {
if(_entryPrefix.length == 0) { if(_entryPrefix.length == 0) {
// prefix is only related to *real* entries, tail not included // prefix is only related to *real* entries, tail not included
_entries.get(_entries.size() - 1)); _entries.get(_entries.size() - 1));
} }
} }
@Override @Override
public String toString() { public String toString() {
return CustomToStringStyle.builder("DPExtra") return CustomToStringStyle.builder("DPExtra")
private CacheDataPage(DataPageMain dataPage) throws IOException { private CacheDataPage(DataPageMain dataPage) throws IOException {
this(dataPage, dataPage.getExtra()); this(dataPage, dataPage.getExtra());
} }
private CacheDataPage(DataPageMain dataPage, DataPageExtra extra) { private CacheDataPage(DataPageMain dataPage, DataPageExtra extra) {
_main = dataPage; _main = dataPage;
_extra = extra; _extra = extra;
public int getPageNumber() { public int getPageNumber() {
return _main._pageNumber; return _main._pageNumber;
} }
@Override @Override
public boolean isLeaf() { public boolean isLeaf() {
return _main._leaf; return _main._leaf;
_main._childTailPageNumber = pageNumber; _main._childTailPageNumber = pageNumber;
} }


@Override @Override
public int getTotalEntrySize() { public int getTotalEntrySize() {
return _extra._totalEntrySize; return _extra._totalEntrySize;
public void addEntry(int idx, Entry entry) throws IOException { public void addEntry(int idx, Entry entry) throws IOException {
_main.getCache().addEntry(this, idx, entry); _main.getCache().addEntry(this, idx, entry);
} }
@Override @Override
public Entry removeEntry(int idx) throws IOException { public Entry removeEntry(int idx) throws IOException {
return _main.getCache().removeEntry(this, idx); return _main.getCache().removeEntry(this, idx);
} }
} }


/** /**
private List<Entry> getEntries() { private List<Entry> getEntries() {
return _extra._entries; return _extra._entries;
} }
@Override @Override
public int size() { public int size() {
int size = getEntries().size(); int size = getEntries().size();
setChildTailEntry(newEntry) : setChildTailEntry(newEntry) :
getEntries().set(idx, newEntry)); getEntries().set(idx, newEntry));
} }
@Override @Override
public void add(int idx, Entry newEntry) { public void add(int idx, Entry newEntry) {
// note, we will never add to the "tail" entry, that will always be // note, we will never add to the "tail" entry, that will always be
// handled through promoteTail // handled through promoteTail
getEntries().add(idx, newEntry); getEntries().add(idx, newEntry);
} }
@Override @Override
public Entry remove(int idx) { public Entry remove(int idx) {
return (isCurrentChildTailIndex(idx) ? return (isCurrentChildTailIndex(idx) ?
setChildTailEntry(null) : setChildTailEntry(null) :
getEntries().remove(idx)); getEntries().remove(idx));
} }
public Entry setChildTailEntry(Entry newEntry) { public Entry setChildTailEntry(Entry newEntry) {
Entry old = _childTailEntry; Entry old = _childTailEntry;
_childTailEntry = newEntry; _childTailEntry = newEntry;
return old; return old;
} }
private boolean hasChildTail() { private boolean hasChildTail() {
return(_childTailEntry != null); return(_childTailEntry != null);
} }
private boolean isCurrentChildTailIndex(int idx) { private boolean isCurrentChildTailIndex(int idx) {
return(idx == getEntries().size()); return(idx == getEntries().size());
} }
getEntries().add(tail); getEntries().add(tail);
return tail; return tail;
} }
public Entry promoteTail() { public Entry promoteTail() {
Entry last = getEntries().remove(getEntries().size() - 1); Entry last = getEntries().remove(getEntries().size() - 1);
_childTailEntry = last; _childTailEntry = last;
return last; return last;
} }
public int find(Entry e) { public int find(Entry e) {
return Collections.binarySearch(this, e); return Collections.binarySearch(this, e);
} }


} }


/**
* Utility class for running index validation.
*/
private final class Validator {
private final boolean _forceLoad;
private final Map<Integer,DataPageMain> _knownPages = new HashMap<>();
private final Queue<DataPageMain> _pendingPages = new LinkedList<>();

private Validator(boolean forceLoad) {
_forceLoad = forceLoad;
_knownPages.putAll(_dataPages);
_pendingPages.addAll(_knownPages.values());
}

void validate() throws IOException {
DataPageMain dpMain = null;
while((dpMain = _pendingPages.poll()) != null) {
DataPageExtra dpExtra = dpMain.getExtra();
validateEntries(dpExtra);
validateChildren(dpMain, dpExtra);
validatePeers(dpMain);
}
}

/**
* Validates the entries for an index page
*
* @param dpExtra the entries to validate
*/
private void validateEntries(DataPageExtra dpExtra) throws IOException {
int entrySize = 0;
Entry prevEntry = FIRST_ENTRY;
for(Entry e : dpExtra._entries) {
entrySize += e.size();
if(prevEntry.compareTo(e) >= 0) {
throw new IOException(withErrorContext(
"Unexpected order in index entries, " + prevEntry +
" >= " + e));
}
prevEntry = e;
}
if(entrySize != dpExtra._totalEntrySize) {
throw new IllegalStateException(withErrorContext(
"Expected size " + entrySize +
" but was " + dpExtra._totalEntrySize));
}
}

/**
* Validates the children for an index page
*
* @param dpMain the index page
* @param dpExtra the child entries to validate
*/
private void validateChildren(DataPageMain dpMain,
DataPageExtra dpExtra) throws IOException {
int childTailPageNumber = dpMain._childTailPageNumber;
if(dpMain._leaf) {
if(childTailPageNumber != INVALID_INDEX_PAGE_NUMBER) {
throw new IllegalStateException(
withErrorContext("Leaf page has tail " + dpMain));
}
return;
}
if((dpExtra._entryView.size() == 1) && dpMain.hasChildTail()) {
throw new IllegalStateException(
withErrorContext("Single child is tail " + dpMain));
}
Integer prevPageNumber = null;
Integer nextPageNumber = null;
for(Entry e : dpExtra._entryView) {
validateEntryForPage(dpMain, e);
Integer subPageNumber = e.getSubPageNumber();
DataPageMain childMain = getPageForValidate(subPageNumber);
if(childMain != null) {
if((prevPageNumber != null) &&
((int)childMain._prevPageNumber != prevPageNumber)) {
throw new IllegalStateException(withErrorContext(
"Child's prevPageNumber is not the previous child for " +
childMain + " " + dpExtra._entryView + " " +
prevPageNumber));
}
if((nextPageNumber != null) &&
(childMain._pageNumber != nextPageNumber)) {
throw new IllegalStateException(withErrorContext(
"Child's pageNumber is not the expected next child for " +
childMain));
}
if(childMain._parentPageNumber != null) {
if(childMain._parentPageNumber != dpMain._pageNumber) {
throw new IllegalStateException(
withErrorContext("Child's parent is incorrect " + childMain));
}
boolean expectTail = (subPageNumber == childTailPageNumber);
if(expectTail != childMain._tail) {
throw new IllegalStateException(withErrorContext(
"Child tail status incorrect " + childMain));
}
}
Entry lastEntry = childMain.getExtra()._entryView.getLast();
if(e.compareTo(lastEntry) != 0) {
throw new IllegalStateException(withErrorContext(
"Invalid entry " + e + " but child is " + lastEntry));
}
nextPageNumber = childMain._nextPageNumber;
prevPageNumber = childMain._pageNumber;
} else {
// if we aren't force loading, we may have gaps in the children so we
// can't validate these for the current child
nextPageNumber = null;
prevPageNumber = null;
}
}
}

/**
* Validates the peer pages for an index page.
*
* @param dpMain the index page
*/
private void validatePeers(DataPageMain dpMain)
throws IOException {

DataPageMain prevMain = getPageForValidate(dpMain._prevPageNumber);
if(prevMain != null) {
if(prevMain._nextPageNumber != dpMain._pageNumber) {
throw new IllegalStateException(withErrorContext(
"Prev page " + prevMain + " does not ref " + dpMain));
}
validatePeerStatus(dpMain, prevMain);
}

DataPageMain nextMain =
getPageForValidate(dpMain._nextPageNumber);
if(nextMain != null) {
if(nextMain._prevPageNumber != dpMain._pageNumber) {
throw new IllegalStateException(withErrorContext(
"Next page " + nextMain + " does not ref " + dpMain));
}
validatePeerStatus(dpMain, nextMain);
}
}

/**
* Validates the given peer page against the given index page
*
* @param dpMain the index page
* @param peerMain the peer index page
*/
private void validatePeerStatus(DataPageMain dpMain, DataPageMain peerMain)
{
if(dpMain._leaf != peerMain._leaf) {
throw new IllegalStateException(withErrorContext(
"Mismatched peer status " + dpMain._leaf + " " +
peerMain._leaf));
}
if(!dpMain._leaf) {
if((dpMain._parentPageNumber != null) &&
(peerMain._parentPageNumber != null) &&
((int)dpMain._parentPageNumber != (int)peerMain._parentPageNumber)) {
throw new IllegalStateException(withErrorContext(
"Mismatched node parents " + dpMain._parentPageNumber + " " +
peerMain._parentPageNumber));
}
}
}

private DataPageMain getPageForValidate(
Integer pageNumber) throws IOException {
DataPageMain dpMain = _knownPages.get(pageNumber);
if((dpMain == null) && _forceLoad &&
(pageNumber != INVALID_INDEX_PAGE_NUMBER)) {
dpMain = getDataPage(pageNumber);
if(dpMain != null) {
_knownPages.put(pageNumber, dpMain);
_pendingPages.add(dpMain);
}
}
return dpMain;
}
}

} }

+ 1
- 1
src/main/java/com/healthmarketscience/jackcess/util/CustomLinkResolver.java View File

throws IOException throws IOException
{ {
super(file, channel, true, false, fileFormat, null, null, null, super(file, channel, true, false, fileFormat, null, null, null,
readOnly);
readOnly, false);
_resolver = resolver; _resolver = resolver;
_customFile = customFile; _customFile = customFile;
} }

+ 8
- 8
src/test/java/com/healthmarketscience/jackcess/BigIndexTest.java View File

public BigIndexTest(String name) { public BigIndexTest(String name) {
super(name); super(name);
} }
public void testComplexIndex() throws Exception public void testComplexIndex() throws Exception
{ {
for (final TestDB testDB : TestDB.getSupportedForBasename(Basename.COMP_INDEX, true)) { for (final TestDB testDB : TestDB.getSupportedForBasename(Basename.COMP_INDEX, true)) {
} }
} }


index.getIndexData().validate();
index.getIndexData().validate(false);


db.flush(); db.flush();
t = null; t = null;
System.gc(); System.gc();
t = (TableImpl)db.getTable("Table1"); t = (TableImpl)db.getTable("Table1");
index = t.getIndex("col1"); index = t.getIndex("col1");




assertEquals(2000, rowCount); assertEquals(2000, rowCount);


index.getIndexData().validate();
index.getIndexData().validate(false);


// delete an entry in the middle // delete an entry in the middle
Cursor cursor = CursorBuilder.createCursor(index); Cursor cursor = CursorBuilder.createCursor(index);
cursor.deleteCurrentRow(); cursor.deleteCurrentRow();
} }


index.getIndexData().validate();
index.getIndexData().validate(false);


List<String> found = new ArrayList<String>(); List<String> found = new ArrayList<String>();
for(Row row : CursorBuilder.createCursor(index)) { for(Row row : CursorBuilder.createCursor(index)) {
assertFalse(cursor.moveToNextRow()); assertFalse(cursor.moveToNextRow());
assertFalse(cursor.moveToPreviousRow()); assertFalse(cursor.moveToPreviousRow());


index.getIndexData().validate();
index.getIndexData().validate(false);


// add 50 (pseudo) random entries to the table // add 50 (pseudo) random entries to the table
rand = new Random(42L); rand = new Random(42L);
t.addRow(nextVal, "this is some row data " + nextInt); t.addRow(nextVal, "this is some row data " + nextInt);
} }


index.getIndexData().validate();
index.getIndexData().validate(false);


cursor = CursorBuilder.createCursor(index); cursor = CursorBuilder.createCursor(index);
while(cursor.moveToNextRow()) { while(cursor.moveToNextRow()) {
cursor.deleteCurrentRow(); cursor.deleteCurrentRow();
} }


index.getIndexData().validate();
index.getIndexData().validate(false);


db.close(); db.close();



+ 25
- 0
src/test/java/com/healthmarketscience/jackcess/DatabaseTest.java View File

assertEquals(expectedUpdateDate, table.getUpdatedDate().toString()); assertEquals(expectedUpdateDate, table.getUpdatedDate().toString());
} }
} }

public void testBrokenIndex() throws Exception {
TestDB testDb = TestDB.getSupportedForBasename(Basename.TEST).get(0);
try (Database db = new DatabaseBuilder(testDb.getFile())
.setReadOnly(true).setIgnoreBrokenSystemCatalogIndex(true).open()) {
Table test = db.getTable("Table1");
assertNotNull(test);
verifyFinderType(db, "FallbackTableFinder");
}
try (Database db = openMem(testDb)) {
Table test = db.getTable("Table1");
assertNotNull(test);
verifyFinderType(db, "DefaultTableFinder");
}
}

private static void verifyFinderType(Database db, String clazzName)
throws Exception{
java.lang.reflect.Field f = db.getClass().getDeclaredField("_tableFinder");
f.setAccessible(true);
Object finder = f.get(db);
assertNotNull(finder);
assertEquals(clazzName, finder.getClass().getSimpleName());
}

private static void checkRawValue(String expected, Object val) private static void checkRawValue(String expected, Object val)
{ {
if(expected != null) { if(expected != null) {

+ 1
- 1
src/test/java/com/healthmarketscience/jackcess/IndexTest.java View File

if(expectedSuccess) { if(expectedSuccess) {
assertNull(failure); assertNull(failure);
} else { } else {
assertTrue(failure != null);
assertNotNull(failure);
assertTrue(failure.getMessage().contains("uniqueness")); assertTrue(failure.getMessage().contains("uniqueness"));
} }
} }

+ 8
- 4
src/test/java/com/healthmarketscience/jackcess/TestUtil.java View File

final Database db = new DatabaseBuilder(file).setReadOnly(readOnly) final Database db = new DatabaseBuilder(file).setReadOnly(readOnly)
.setAutoSync(getTestAutoSync()).setChannel(channel) .setAutoSync(getTestAutoSync()).setChannel(channel)
.setCharset(charset).open(); .setCharset(charset).open();
Assert.assertEquals("Wrong JetFormat.",
DatabaseImpl.getFileFormatDetails(fileFormat).getFormat(),
((DatabaseImpl)db).getFormat());
Assert.assertEquals("Wrong FileFormat.", fileFormat, db.getFileFormat());
if(fileFormat != null) {
Assert.assertEquals(
"Wrong JetFormat.",
DatabaseImpl.getFileFormatDetails(fileFormat).getFormat(),
((DatabaseImpl)db).getFormat());
Assert.assertEquals(
"Wrong FileFormat.", fileFormat, db.getFileFormat());
}
return db; return db;
} }



Loading…
Cancel
Save