From aaddcfc655dcf8a428c169f0952c574a4ead3903 Mon Sep 17 00:00:00 2001 From: Nick Burch Date: Sun, 19 Dec 2010 08:53:36 +0000 Subject: [PATCH] Change how the NIO block read works, to re-use the byte array for the from-InputStream case. Also start on reading the FAT blocks for NPOIFSFileSystem git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1050775 13f79535-47bb-0310-9956-ffa450edef68 --- .../poifs/filesystem/NPOIFSFileSystem.java | 52 ++++++-- .../poifs/nio/ByteArrayBackedDataSource.java | 8 +- .../org/apache/poi/poifs/nio/DataSource.java | 8 +- .../poi/poifs/nio/FileBackedDataSource.java | 13 +- src/java/org/apache/poi/util/IOUtils.java | 2 +- .../filesystem/TestNPOIFSFileSystem.java | 27 ++++- .../apache/poi/poifs/nio/TestDataSource.java | 113 +++++++++--------- 7 files changed, 145 insertions(+), 78 deletions(-) diff --git a/src/java/org/apache/poi/poifs/filesystem/NPOIFSFileSystem.java b/src/java/org/apache/poi/poifs/filesystem/NPOIFSFileSystem.java index eea94f8679..d859139d3c 100644 --- a/src/java/org/apache/poi/poifs/filesystem/NPOIFSFileSystem.java +++ b/src/java/org/apache/poi/poifs/filesystem/NPOIFSFileSystem.java @@ -184,12 +184,18 @@ public class NPOIFSFileSystem _header = new HeaderBlock(headerBuffer); // We need to buffer the whole file into memory when - // working with an InputStream. Do so now - int maxSize = _header.getBATCount() * + // working with an InputStream. + // The max possible size is when each BAT block entry is used + int maxSize = + _header.getBATCount() * _header.getBigBlockSize().getBATEntriesPerBlock() * - _header.getBigBlockSize().getBigBlockSize(); + _header.getBigBlockSize().getBigBlockSize() + ; ByteBuffer data = ByteBuffer.allocate(maxSize); + // Copy in the header data.put(headerBuffer); + data.position(_header.getBigBlockSize().getBigBlockSize()); + // Now read the rest of the stream IOUtils.readFully(channel, data); success = true; @@ -260,14 +266,46 @@ public class NPOIFSFileSystem // Grab the block size bigBlockSize = _header.getBigBlockSize(); - // Read the properties - // TODO - // Read the FAT blocks - // TODO + for(int fatAT : _header.getBATArray()) { + ByteBuffer fatData = getBlockAt(fatAT); + _blocks.add(BATBlock.createBATBlock(bigBlockSize, fatData)); + } // Now read the XFAT blocks +// TODO Corrupt / Loop checking + BATBlock xfat; + int nextAt = _header.getXBATIndex(); + for(int i=0; i<_header.getXBATCount(); i++) { + ByteBuffer fatData = getBlockAt(nextAt); + xfat = BATBlock.createBATBlock(bigBlockSize, fatData); + nextAt = xfat.getValueAt(bigBlockSize.getNextXBATChainOffset()); + + _blocks.add(xfat); + } + + // We're now able to load steams + // Use this to read in the properties + // TODO +// TODO With loop checking + } + + /** + * Load the block at the given offset. + */ + protected ByteBuffer getBlockAt(final int offset) throws IOException { + ByteBuffer data = ByteBuffer.allocate(bigBlockSize.getBigBlockSize()); + + // The header block doesn't count, so add one + long startAt = (offset+1) * bigBlockSize.getBigBlockSize(); + return _data.read(bigBlockSize.getBigBlockSize(), startAt); + } + /** + * Works out what block follows the specified one. + */ + protected int getNextBlock(final int offset) { // TODO + return -1; } /** diff --git a/src/java/org/apache/poi/poifs/nio/ByteArrayBackedDataSource.java b/src/java/org/apache/poi/poifs/nio/ByteArrayBackedDataSource.java index 823b84e939..1df2b1b064 100644 --- a/src/java/org/apache/poi/poifs/nio/ByteArrayBackedDataSource.java +++ b/src/java/org/apache/poi/poifs/nio/ByteArrayBackedDataSource.java @@ -34,16 +34,16 @@ public class ByteArrayBackedDataSource extends DataSource { this(data, data.length); } - public void read(ByteBuffer dst, long position) { + public ByteBuffer read(int length, long position) { if(position >= size) { throw new IndexOutOfBoundsException( - "Unable to read " + dst.capacity() + " bytes from " + + "Unable to read " + length + " bytes from " + position + " in stream of length " + size ); } - int toRead = (int)Math.min(dst.capacity(), size - position); - dst.put(buffer, (int)position, toRead); + int toRead = (int)Math.min(length, size - position); + return ByteBuffer.wrap(buffer, (int)position, toRead); } public void write(ByteBuffer src, long position) { diff --git a/src/java/org/apache/poi/poifs/nio/DataSource.java b/src/java/org/apache/poi/poifs/nio/DataSource.java index 1264b0922d..4d56525fd2 100644 --- a/src/java/org/apache/poi/poifs/nio/DataSource.java +++ b/src/java/org/apache/poi/poifs/nio/DataSource.java @@ -24,8 +24,8 @@ import java.nio.ByteBuffer; * Common definition of how we read and write bytes */ public abstract class DataSource { - abstract void read(ByteBuffer dst, long position) throws IOException; - abstract void write(ByteBuffer src, long position) throws IOException; - abstract long size() throws IOException; - abstract void close() throws IOException; + public abstract ByteBuffer read(int length, long position) throws IOException; + public abstract void write(ByteBuffer src, long position) throws IOException; + public abstract long size() throws IOException; + public abstract void close() throws IOException; } diff --git a/src/java/org/apache/poi/poifs/nio/FileBackedDataSource.java b/src/java/org/apache/poi/poifs/nio/FileBackedDataSource.java index 19c6a30300..bba58efeaa 100644 --- a/src/java/org/apache/poi/poifs/nio/FileBackedDataSource.java +++ b/src/java/org/apache/poi/poifs/nio/FileBackedDataSource.java @@ -42,17 +42,26 @@ public class FileBackedDataSource extends DataSource { this.channel = channel; } - public void read(ByteBuffer dst, long position) throws IOException { + public ByteBuffer read(int length, long position) throws IOException { if(position >= size()) { throw new IllegalArgumentException("Position " + position + " past the end of the file"); } - + + // Read channel.position(position); + ByteBuffer dst = ByteBuffer.allocate(length); int worked = IOUtils.readFully(channel, dst); + // Check if(worked == -1) { throw new IllegalArgumentException("Position " + position + " past the end of the file"); } + + // Ready it for reading + dst.position(0); + + // All done + return dst; } public void write(ByteBuffer src, long position) throws IOException { diff --git a/src/java/org/apache/poi/util/IOUtils.java b/src/java/org/apache/poi/util/IOUtils.java index 719330f122..c3c8fa1ac9 100644 --- a/src/java/org/apache/poi/util/IOUtils.java +++ b/src/java/org/apache/poi/util/IOUtils.java @@ -105,7 +105,7 @@ public final class IOUtils { public static int readFully(ReadableByteChannel channel, ByteBuffer b) throws IOException { int total = 0; while (true) { - int got = channel.read(b); + int got = channel.read(b); if (got < 0) { return (total == 0) ? -1 : total; } diff --git a/src/testcases/org/apache/poi/poifs/filesystem/TestNPOIFSFileSystem.java b/src/testcases/org/apache/poi/poifs/filesystem/TestNPOIFSFileSystem.java index 387d15b800..66194c8b0f 100644 --- a/src/testcases/org/apache/poi/poifs/filesystem/TestNPOIFSFileSystem.java +++ b/src/testcases/org/apache/poi/poifs/filesystem/TestNPOIFSFileSystem.java @@ -55,5 +55,30 @@ public final class TestNPOIFSFileSystem extends TestCase { assertEquals(4096, fs.getBigBlockSize()); } } - + + public void testPropertiesAndFatOnRead() throws Exception { + NPOIFSFileSystem fsA, fsB; + + // With a simple 512 block file + fsA = new NPOIFSFileSystem(_inst.getFile("BlockSize512.zvi")); + fsB = new NPOIFSFileSystem(_inst.openResourceAsStream("BlockSize512.zvi")); + for(NPOIFSFileSystem fs : new NPOIFSFileSystem[] {fsA,fsB}) { + // Check the FAT was properly processed + // TODO + + // Check the properties + // TODO + } + + // Now with a simple 4096 block file + fsA = new NPOIFSFileSystem(_inst.getFile("BlockSize4096.zvi")); + fsB = new NPOIFSFileSystem(_inst.openResourceAsStream("BlockSize4096.zvi")); + for(NPOIFSFileSystem fs : new NPOIFSFileSystem[] {fsA,fsB}) { + // Check the FAT was properly processed + // TODO + + // Check the properties + // TODO + } + } } diff --git a/src/testcases/org/apache/poi/poifs/nio/TestDataSource.java b/src/testcases/org/apache/poi/poifs/nio/TestDataSource.java index 6eb0098cb6..f3ffd50f06 100644 --- a/src/testcases/org/apache/poi/poifs/nio/TestDataSource.java +++ b/src/testcases/org/apache/poi/poifs/nio/TestDataSource.java @@ -20,6 +20,7 @@ package org.apache.poi.poifs.nio; import java.io.File; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import org.apache.poi.POIDataSamples; @@ -40,20 +41,23 @@ public class TestDataSource extends TestCase assertEquals(8192, ds.size()); // Start of file - ByteBuffer bs = ByteBuffer.allocate(4); - ds.read(bs, 0); + ByteBuffer bs; + bs = ds.read(4, 0); assertEquals(4, bs.capacity()); - assertEquals(4, bs.position()); + assertEquals(0, bs.position()); assertEquals(0xd0-256, bs.get(0)); assertEquals(0xcf-256, bs.get(1)); assertEquals(0x11-000, bs.get(2)); assertEquals(0xe0-256, bs.get(3)); + assertEquals(0xd0-256, bs.get()); + assertEquals(0xcf-256, bs.get()); + assertEquals(0x11-000, bs.get()); + assertEquals(0xe0-256, bs.get()); // Mid way through - bs = ByteBuffer.allocate(8); - ds.read(bs, 0x400); + bs = ds.read(8, 0x400); assertEquals(8, bs.capacity()); - assertEquals(8, bs.position()); + assertEquals(0, bs.position()); assertEquals((byte)'R', bs.get(0)); assertEquals(0, bs.get(1)); assertEquals((byte)'o', bs.get(2)); @@ -64,14 +68,12 @@ public class TestDataSource extends TestCase assertEquals(0, bs.get(7)); // Can go to the end, but not past it - bs.clear(); - ds.read(bs, 8190); - assertEquals(2, bs.position()); + bs = ds.read(8, 8190); + assertEquals(0, bs.position()); // TODO How best to warn of a short read? // Can't go off the end try { - bs.clear(); - ds.read(bs, 8192); + bs = ds.read(4, 8192); fail("Shouldn't be able to read off the end of the file"); } catch(IllegalArgumentException e) {} } @@ -87,70 +89,64 @@ public class TestDataSource extends TestCase ByteArrayBackedDataSource ds = new ByteArrayBackedDataSource(data); // Start - ByteBuffer bs = ByteBuffer.allocate(4); - ds.read(bs, 0); - assertEquals(4, bs.capacity()); - assertEquals(4, bs.position()); - assertEquals(0x00, bs.get(0)); - assertEquals(0x01, bs.get(1)); - assertEquals(0x02, bs.get(2)); - assertEquals(0x03, bs.get(3)); + ByteBuffer bs; + bs = ds.read(4, 0); + assertEquals(0, bs.position()); + assertEquals(0x00, bs.get()); + assertEquals(0x01, bs.get()); + assertEquals(0x02, bs.get()); + assertEquals(0x03, bs.get()); // Middle - bs.clear(); - ds.read(bs, 100); - assertEquals(4, bs.capacity()); - assertEquals(4, bs.position()); - assertEquals(100, bs.get(0)); - assertEquals(101, bs.get(1)); - assertEquals(102, bs.get(2)); - assertEquals(103, bs.get(3)); + bs = ds.read(4, 100); + assertEquals(100, bs.position()); + assertEquals(100, bs.get()); + assertEquals(101, bs.get()); + assertEquals(102, bs.get()); + assertEquals(103, bs.get()); // End - bs.clear(); - ds.read(bs, 252); - assertEquals(4, bs.capacity()); - assertEquals(4, bs.position()); - assertEquals(-4, bs.get(0)); - assertEquals(-3, bs.get(1)); - assertEquals(-2, bs.get(2)); - assertEquals(-1, bs.get(3)); + bs = ds.read(4, 252); + assertEquals(-4, bs.get()); + assertEquals(-3, bs.get()); + assertEquals(-2, bs.get()); + assertEquals(-1, bs.get()); // Off the end - bs.clear(); - ds.read(bs, 254); - assertEquals(4, bs.capacity()); - assertEquals(2, bs.position()); - assertEquals(-2, bs.get(0)); - assertEquals(-1, bs.get(1)); + bs = ds.read(4, 254); + assertEquals(-2, bs.get()); + assertEquals(-1, bs.get()); + try { + bs.get(); + fail("Shouldn't be able to read off the end"); + } catch(BufferUnderflowException e) {} // Past the end - bs.clear(); try { - ds.read(bs, 256); + bs = ds.read(4, 256); fail("Shouldn't be able to read off the end"); } catch(IndexOutOfBoundsException e) {} // Overwrite - bs.clear(); + bs = ByteBuffer.allocate(4); bs.put(0, (byte)-55); bs.put(1, (byte)-54); bs.put(2, (byte)-53); bs.put(3, (byte)-52); + assertEquals(256, ds.size()); ds.write(bs, 40); - bs.clear(); - ds.read(bs, 40); + assertEquals(256, ds.size()); + bs = ds.read(4, 40); - assertEquals(4, bs.position()); - assertEquals(-55, bs.get(0)); - assertEquals(-54, bs.get(1)); - assertEquals(-53, bs.get(2)); - assertEquals(-52, bs.get(3)); + assertEquals(-55, bs.get()); + assertEquals(-54, bs.get()); + assertEquals(-53, bs.get()); + assertEquals(-52, bs.get()); // Append - bs.clear(); + bs = ByteBuffer.allocate(4); bs.put(0, (byte)-55); bs.put(1, (byte)-54); bs.put(2, (byte)-53); @@ -160,12 +156,11 @@ public class TestDataSource extends TestCase ds.write(bs, 256); assertEquals(260, ds.size()); - bs.clear(); - ds.read(bs, 256); - assertEquals(4, bs.position()); - assertEquals(-55, bs.get(0)); - assertEquals(-54, bs.get(1)); - assertEquals(-53, bs.get(2)); - assertEquals(-52, bs.get(3)); + bs = ds.read(4, 256); + assertEquals(256, bs.position()); + assertEquals(-55, bs.get()); + assertEquals(-54, bs.get()); + assertEquals(-53, bs.get()); + assertEquals(-52, bs.get()); } } -- 2.39.5