From 1bffc624a7ce3b78109c57ff1149375b8af248c8 Mon Sep 17 00:00:00 2001 From: Nick Burch Date: Mon, 27 Dec 2010 06:50:05 +0000 Subject: [PATCH] Start to support the MiniStream, by extracting out the BlockStore superclass and implementing a SBAT/Stream based mini block version git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1053007 13f79535-47bb-0310-9956-ffa450edef68 --- .../poi/poifs/common/POIFSConstants.java | 4 + .../poi/poifs/filesystem/BlockStore.java | 105 ++++++++++ .../poifs/filesystem/NPOIFSFileSystem.java | 98 +++++---- .../poi/poifs/filesystem/NPOIFSMiniStore.java | 195 ++++++++++++++++++ .../poi/poifs/filesystem/NPOIFSStream.java | 47 +++-- .../apache/poi/poifs/storage/BATBlock.java | 21 +- 6 files changed, 407 insertions(+), 63 deletions(-) create mode 100644 src/java/org/apache/poi/poifs/filesystem/BlockStore.java create mode 100644 src/java/org/apache/poi/poifs/filesystem/NPOIFSMiniStore.java diff --git a/src/java/org/apache/poi/poifs/common/POIFSConstants.java b/src/java/org/apache/poi/poifs/common/POIFSConstants.java index 74bc037d13..b732db45d9 100644 --- a/src/java/org/apache/poi/poifs/common/POIFSConstants.java +++ b/src/java/org/apache/poi/poifs/common/POIFSConstants.java @@ -33,6 +33,10 @@ public interface POIFSConstants public static final POIFSBigBlockSize LARGER_BIG_BLOCK_SIZE_DETAILS = new POIFSBigBlockSize(LARGER_BIG_BLOCK_SIZE, (short)12); + /** How big a block in the small block stream is. Fixed size */ + public static final int SMALL_BLOCK_SIZE = 0x0040; + + /** How big a single property is */ public static final int PROPERTY_SIZE = 0x0080; /** diff --git a/src/java/org/apache/poi/poifs/filesystem/BlockStore.java b/src/java/org/apache/poi/poifs/filesystem/BlockStore.java new file mode 100644 index 0000000000..1da9b6a965 --- /dev/null +++ b/src/java/org/apache/poi/poifs/filesystem/BlockStore.java @@ -0,0 +1,105 @@ + +/* ==================================================================== + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +==================================================================== */ + +package org.apache.poi.poifs.filesystem; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.poi.poifs.storage.BATBlock.BATBlockAndIndex; + +/** + * This abstract class describes a way to read, store, chain + * and free a series of blocks (be they Big or Small ones) + */ +public abstract class BlockStore { + /** + * Returns the size of the blocks managed through the block store. + */ + protected abstract int getBlockStoreBlockSize(); + + /** + * Load the block at the given offset. + */ + protected abstract ByteBuffer getBlockAt(final int offset) throws IOException; + + /** + * Extends the file if required to hold blocks up to + * the specified offset, and return the block from there. + */ + protected abstract ByteBuffer createBlockIfNeeded(final int offset) throws IOException; + + /** + * Returns the BATBlock that handles the specified offset, + * and the relative index within it + */ + protected abstract BATBlockAndIndex getBATBlockAndIndex(final int offset); + + /** + * Works out what block follows the specified one. + */ + protected abstract int getNextBlock(final int offset); + + /** + * Changes the record of what block follows the specified one. + */ + protected abstract void setNextBlock(final int offset, final int nextBlock); + + /** + * Finds a free block, and returns its offset. + * This method will extend the file/stream if needed, and if doing + * so, allocate new FAT blocks to address the extra space. + */ + protected abstract int getFreeBlock() throws IOException; + + /** + * Creates a Detector for loops in the chain + */ + protected abstract ChainLoopDetector getChainLoopDetector() throws IOException; + + /** + * Used to detect if a chain has a loop in it, so + * we can bail out with an error rather than + * spinning away for ever... + */ + protected class ChainLoopDetector { + private boolean[] used_blocks; + protected ChainLoopDetector(long rawSize) { + int numBlocks = (int)Math.ceil( rawSize / getBlockStoreBlockSize() ); + used_blocks = new boolean[numBlocks]; + } + protected void claim(int offset) { + if(offset >= used_blocks.length) { + // They're writing, and have had new blocks requested + // for the write to proceed. That means they're into + // blocks we've allocated for them, so are safe + return; + } + + // Claiming an existing block, ensure there's no loop + if(used_blocks[offset]) { + throw new IllegalStateException( + "Potential loop detected - Block " + offset + + " was already claimed but was just requested again" + ); + } + used_blocks[offset] = true; + } + } +} + diff --git a/src/java/org/apache/poi/poifs/filesystem/NPOIFSFileSystem.java b/src/java/org/apache/poi/poifs/filesystem/NPOIFSFileSystem.java index 3caf3b8dd7..8147e149c3 100644 --- a/src/java/org/apache/poi/poifs/filesystem/NPOIFSFileSystem.java +++ b/src/java/org/apache/poi/poifs/filesystem/NPOIFSFileSystem.java @@ -66,7 +66,7 @@ import org.apache.poi.util.POILogger; * This is the new NIO version */ -public class NPOIFSFileSystem +public class NPOIFSFileSystem extends BlockStore implements POIFSViewable { private static final POILogger _logger = @@ -79,10 +79,11 @@ public class NPOIFSFileSystem return new CloseIgnoringInputStream(is); } + private NPOIFSMiniStore _mini_store; private NPropertyTable _property_table; - private List _bat_blocks; - private HeaderBlock _header; - private DirectoryNode _root; + private List _bat_blocks; + private HeaderBlock _header; + private DirectoryNode _root; private DataSource _data; @@ -102,6 +103,7 @@ public class NPOIFSFileSystem { _header = new HeaderBlock(bigBlockSize); _property_table = new NPropertyTable(_header); + _mini_store = new NPOIFSMiniStore(this, _property_table.getRoot(), new ArrayList(), _header); _bat_blocks = new ArrayList(); _root = null; } @@ -264,7 +266,7 @@ public class NPOIFSFileSystem // Each block should only ever be used by one of the // FAT, XFAT or Property Table. Ensure it does - ChainLoopDetector loopDetector = new ChainLoopDetector(); + ChainLoopDetector loopDetector = getChainLoopDetector(); // Read the FAT blocks for(int fatAt : _header.getBATArray()) { @@ -291,6 +293,20 @@ public class NPOIFSFileSystem // We're now able to load steams // Use this to read in the properties _property_table = new NPropertyTable(_header, this); + + // Finally read the Small Stream FAT (SBAT) blocks + BATBlock sfat; + List sbats = new ArrayList(); + _mini_store = new NPOIFSMiniStore(this, _property_table.getRoot(), sbats, _header); + nextAt = _header.getSBATStart(); + for(int i=0; i<_header.getSBATCount(); i++) { + loopDetector.claim(nextAt); + ByteBuffer fatData = getBlockAt(nextAt); + sfat = BATBlock.createBATBlock(bigBlockSize, fatData); + sfat.setOurBlockIndex(nextAt); + sbats.add(sfat); + nextAt = getNextBlock(nextAt); + } } /** @@ -302,6 +318,24 @@ public class NPOIFSFileSystem return _data.read(bigBlockSize.getBigBlockSize(), startAt); } + /** + * Load the block at the given offset, + * extending the file if needed + */ + protected ByteBuffer createBlockIfNeeded(final int offset) throws IOException { + try { + return getBlockAt(offset); + } catch(IndexOutOfBoundsException e) { + // The header block doesn't count, so add one + long startAt = (offset+1) * bigBlockSize.getBigBlockSize(); + // Allocate and write + ByteBuffer buffer = ByteBuffer.allocate(getBigBlockSize()); + _data.write(buffer, startAt); + // Retrieve the properly backed block + return getBlockAt(offset); + } + } + /** * Returns the BATBlock that handles the specified offset, * and the relative index within it @@ -409,14 +443,27 @@ public class NPOIFSFileSystem // The first offset stores us, but the 2nd is free return offset+1; } + + @Override + protected ChainLoopDetector getChainLoopDetector() throws IOException { + return new ChainLoopDetector(_data.size()); + } - /** + /** * For unit testing only! Returns the underlying * properties table */ NPropertyTable _get_property_table() { return _property_table; } + + /** + * Returns the MiniStore, which performs a similar low + * level function to this, except for the small blocks. + */ + public NPOIFSMiniStore getMiniStore() { + return _mini_store; + } /** * Create a new document to be added to the root directory @@ -725,36 +772,6 @@ public class NPOIFSFileSystem } } - /** - * Used to detect if a chain has a loop in it, so - * we can bail out with an error rather than - * spinning away for ever... - */ - protected class ChainLoopDetector { - private boolean[] used_blocks; - protected ChainLoopDetector() throws IOException { - int numBlocks = (int)Math.ceil(_data.size()/bigBlockSize.getBigBlockSize()); - used_blocks = new boolean[numBlocks]; - } - protected void claim(int offset) { - if(offset >= used_blocks.length) { - // They're writing, and have had new blocks requested - // for the write to proceed. That means they're into - // blocks we've allocated for them, so are safe - return; - } - - // Claiming an existing block, ensure there's no loop - if(used_blocks[offset]) { - throw new IllegalStateException( - "Potential loop detected - Block " + offset + - " was already claimed but was just requested again" - ); - } - used_blocks[offset] = true; - } - } - /* ********** START begin implementation of POIFSViewable ********** */ /** @@ -815,11 +832,13 @@ public class NPOIFSFileSystem return "POIFS FileSystem"; } + /* ********** END begin implementation of POIFSViewable ********** */ + /** * @return The Big Block size, normally 512 bytes, sometimes 4096 bytes */ public int getBigBlockSize() { - return bigBlockSize.getBigBlockSize(); + return bigBlockSize.getBigBlockSize(); } /** * @return The Big Block size, normally 512 bytes, sometimes 4096 bytes @@ -827,7 +846,8 @@ public class NPOIFSFileSystem public POIFSBigBlockSize getBigBlockSizeDetails() { return bigBlockSize; } - - /* ********** END begin implementation of POIFSViewable ********** */ + protected int getBlockStoreBlockSize() { + return getBigBlockSize(); + } } diff --git a/src/java/org/apache/poi/poifs/filesystem/NPOIFSMiniStore.java b/src/java/org/apache/poi/poifs/filesystem/NPOIFSMiniStore.java new file mode 100644 index 0000000000..a34e97efd0 --- /dev/null +++ b/src/java/org/apache/poi/poifs/filesystem/NPOIFSMiniStore.java @@ -0,0 +1,195 @@ + +/* ==================================================================== + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +==================================================================== */ + + +package org.apache.poi.poifs.filesystem; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.List; + +import org.apache.poi.poifs.common.POIFSConstants; +import org.apache.poi.poifs.property.RootProperty; +import org.apache.poi.poifs.storage.BATBlock; +import org.apache.poi.poifs.storage.HeaderBlock; +import org.apache.poi.poifs.storage.BATBlock.BATBlockAndIndex; + +/** + * This class handles the MiniStream (small block store) + * in the NIO case for {@link NPOIFSFileSystem} + */ +public class NPOIFSMiniStore extends BlockStore +{ + private NPOIFSFileSystem _filesystem; + private NPOIFSStream _mini_stream; + private List _sbat_blocks; + private HeaderBlock _header; + private RootProperty _root; + + protected NPOIFSMiniStore(NPOIFSFileSystem filesystem, RootProperty root, + List sbats, HeaderBlock header) + { + this._filesystem = filesystem; + this._sbat_blocks = sbats; + this._header = header; + this._root = root; + + this._mini_stream = new NPOIFSStream(filesystem); + } + + /** + * Load the block at the given offset. + */ + protected ByteBuffer getBlockAt(final int offset) throws IOException { + // Which big block is this? + int byteOffset = offset * POIFSConstants.SMALL_BLOCK_SIZE; + int bigBlockNumber = byteOffset / _filesystem.getBigBlockSize(); + int bigBlockOffset = byteOffset % _filesystem.getBigBlockSize(); + + // Now locate the data block for it + Iterator it = _mini_stream.getBlockIterator(); + for(int i=0; i { - private NPOIFSFileSystem filesystem; + private BlockStore blockStore; private int startBlock; /** @@ -53,8 +53,8 @@ public class NPOIFSStream implements Iterable * to know how to get the start block (eg from a * {@link HeaderBlock} or a {@link Property}) */ - public NPOIFSStream(NPOIFSFileSystem filesystem, int startBlock) { - this.filesystem = filesystem; + public NPOIFSStream(BlockStore blockStore, int startBlock) { + this.blockStore = blockStore; this.startBlock = startBlock; } @@ -62,8 +62,8 @@ public class NPOIFSStream implements Iterable * Constructor for a new stream. A start block won't * be allocated until you begin writing to it. */ - public NPOIFSStream(NPOIFSFileSystem filesystem) { - this.filesystem = filesystem; + public NPOIFSStream(BlockStore blockStore) { + this.blockStore = blockStore; this.startBlock = POIFSConstants.END_OF_CHAIN; } @@ -101,51 +101,56 @@ public class NPOIFSStream implements Iterable */ public void updateContents(byte[] contents) throws IOException { // How many blocks are we going to need? - int blocks = (int)Math.ceil(contents.length / filesystem.getBigBlockSize()); + int blockSize = blockStore.getBlockStoreBlockSize(); + int blocks = (int)Math.ceil(contents.length / blockSize); // Make sure we don't encounter a loop whilst overwriting // the existing blocks - ChainLoopDetector loopDetector = filesystem.new ChainLoopDetector(); + ChainLoopDetector loopDetector = blockStore.getChainLoopDetector(); // Start writing int prevBlock = POIFSConstants.END_OF_CHAIN; int nextBlock = startBlock; for(int i=0; i protected StreamBlockByteBufferIterator(int firstBlock) { this.nextBlock = firstBlock; try { - this.loopDetector = filesystem.new ChainLoopDetector(); + this.loopDetector = blockStore.getChainLoopDetector(); } catch(IOException e) { throw new RuntimeException(e); } @@ -180,8 +185,8 @@ public class NPOIFSStream implements Iterable try { loopDetector.claim(nextBlock); - ByteBuffer data = filesystem.getBlockAt(nextBlock); - nextBlock = filesystem.getNextBlock(nextBlock); + ByteBuffer data = blockStore.getBlockAt(nextBlock); + nextBlock = blockStore.getNextBlock(nextBlock); return data; } catch(IOException e) { throw new RuntimeException(e); diff --git a/src/java/org/apache/poi/poifs/storage/BATBlock.java b/src/java/org/apache/poi/poifs/storage/BATBlock.java index 636a46f2be..a48b43aa2c 100644 --- a/src/java/org/apache/poi/poifs/storage/BATBlock.java +++ b/src/java/org/apache/poi/poifs/storage/BATBlock.java @@ -257,7 +257,7 @@ public final class BATBlock extends BigBlock { * The List of BATBlocks must be in sequential order */ public static BATBlockAndIndex getBATBlockAndIndex(final int offset, - final HeaderBlock header, final List blocks) { + final HeaderBlock header, final List bats) { POIFSBigBlockSize bigBlockSize = header.getBigBlockSize(); // Are we in the BAT or XBAT range @@ -267,7 +267,7 @@ public final class BATBlock extends BigBlock { if(offset < batRangeEndsAt) { int whichBAT = (int)Math.floor(offset / bigBlockSize.getBATEntriesPerBlock()); int index = offset % bigBlockSize.getBATEntriesPerBlock(); - return new BATBlockAndIndex( index, blocks.get(whichBAT) ); + return new BATBlockAndIndex( index, bats.get(whichBAT) ); } // XBATs hold slightly less @@ -276,10 +276,25 @@ public final class BATBlock extends BigBlock { int index = relOffset % bigBlockSize.getXBATEntriesPerBlock(); return new BATBlockAndIndex( index, - blocks.get(header.getBATCount() + whichXBAT) + bats.get(header.getBATCount() + whichXBAT) ); } + /** + * Returns the BATBlock that handles the specified offset, + * and the relative index within it, for the mini stream. + * The List of BATBlocks must be in sequential order + */ + public static BATBlockAndIndex getSBATBlockAndIndex(final int offset, + final HeaderBlock header, final List sbats) { + POIFSBigBlockSize bigBlockSize = header.getBigBlockSize(); + + // SBATs are so much easier, as they're chained streams + int whichSBAT = (int)Math.floor(offset / bigBlockSize.getBATEntriesPerBlock()); + int index = offset % bigBlockSize.getBATEntriesPerBlock(); + return new BATBlockAndIndex( index, sbats.get(whichSBAT) ); + } + private void setXBATChain(final POIFSBigBlockSize bigBlockSize, int chainIndex) { int _entries_per_xbat_block = bigBlockSize.getXBATEntriesPerBlock(); -- 2.39.5