From 741093e3a25d8baab985984b4973e67427e54b7b Mon Sep 17 00:00:00 2001 From: James Ahlborn Date: Tue, 4 Jun 2013 03:10:46 +0000 Subject: [PATCH] add attachment encoding support git-svn-id: https://svn.code.sf.net/p/jackcess/code/jackcess/trunk@734 f203690c-595d-4dc9-a70b-905162fa7fd2 --- .../jackcess/ByteUtil.java | 11 +- .../jackcess/complex/Attachment.java | 2 +- .../complex/AttachmentColumnInfo.java | 107 +++++++++++++++--- .../jackcess/complex/ComplexColumnInfo.java | 4 +- .../complex/ComplexValueForeignKey.java | 6 +- .../complex/MultiValueColumnInfo.java | 2 +- .../complex/UnsupportedColumnInfo.java | 4 +- .../complex/VersionHistoryColumnInfo.java | 2 +- .../jackcess/ComplexColumnTest.java | 5 + 9 files changed, 120 insertions(+), 23 deletions(-) diff --git a/src/java/com/healthmarketscience/jackcess/ByteUtil.java b/src/java/com/healthmarketscience/jackcess/ByteUtil.java index b500268..b46a44b 100644 --- a/src/java/com/healthmarketscience/jackcess/ByteUtil.java +++ b/src/java/com/healthmarketscience/jackcess/ByteUtil.java @@ -29,6 +29,7 @@ package com.healthmarketscience.jackcess; import java.io.FileWriter; import java.io.IOException; +import java.io.OutputStream; import java.io.PrintWriter; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -609,7 +610,7 @@ public final class ByteUtil { * Utility byte stream similar to ByteArrayOutputStream but with extended * accessibility to the bytes. */ - public static class ByteStream + public static class ByteStream extends OutputStream { private byte[] _bytes; private int _length; @@ -641,15 +642,18 @@ public final class ByteUtil { } } + @Override public void write(int b) { ensureNewCapacity(1); _bytes[_length++] = (byte)b; } + @Override public void write(byte[] b) { write(b, 0, b.length); } + @Override public void write(byte[] b, int offset, int length) { ensureNewCapacity(length); System.arraycopy(b, offset, _bytes, _length, length); @@ -671,6 +675,11 @@ public final class ByteUtil { Arrays.fill(_bytes, oldLength, _length, b); } + public void skip(int n) { + ensureNewCapacity(n); + _length += n; + } + public void writeTo(ByteStream out) { out.write(_bytes, 0, _length); } diff --git a/src/java/com/healthmarketscience/jackcess/complex/Attachment.java b/src/java/com/healthmarketscience/jackcess/complex/Attachment.java index be7e302..18c361e 100644 --- a/src/java/com/healthmarketscience/jackcess/complex/Attachment.java +++ b/src/java/com/healthmarketscience/jackcess/complex/Attachment.java @@ -29,7 +29,7 @@ import java.util.Date; */ public interface Attachment extends ComplexValue { - public byte[] getFileData(); + public byte[] getFileData() throws IOException; public void setFileData(byte[] data); diff --git a/src/java/com/healthmarketscience/jackcess/complex/AttachmentColumnInfo.java b/src/java/com/healthmarketscience/jackcess/complex/AttachmentColumnInfo.java index eb3ca73..1d0595c 100644 --- a/src/java/com/healthmarketscience/jackcess/complex/AttachmentColumnInfo.java +++ b/src/java/com/healthmarketscience/jackcess/complex/AttachmentColumnInfo.java @@ -23,14 +23,21 @@ import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; import com.healthmarketscience.jackcess.ByteUtil; import com.healthmarketscience.jackcess.Column; +import com.healthmarketscience.jackcess.JetFormat; import com.healthmarketscience.jackcess.PageChannel; import com.healthmarketscience.jackcess.Table; @@ -42,12 +49,21 @@ import com.healthmarketscience.jackcess.Table; */ public class AttachmentColumnInfo extends ComplexColumnInfo { + /** some file formats which may not be worth re-compressing */ + private static final Set COMPRESSED_FORMATS = new HashSet( + Arrays.asList("jpg", "zip", "gz", "bz2", "z", "7z", "cab", "rar", + "mp3", "mpg")); + private static final String FILE_NAME_COL_NAME = "FileName"; private static final String FILE_TYPE_COL_NAME = "FileType"; private static final int DATA_TYPE_RAW = 0; private static final int DATA_TYPE_COMPRESSED = 1; + private static final int UNKNOWN_HEADER_VAL = 1; + private static final int WRAPPER_HEADER_SIZE = 8; + private static final int CONTENT_HEADER_SIZE = 12; + private final Column _fileUrlCol; private final Column _fileNameCol; private final Column _fileTypeCol; @@ -155,7 +171,9 @@ public class AttachmentColumnInfo extends ComplexColumnInfo } @Override - protected Object[] asRow(Object[] row, Attachment attachment) { + protected Object[] asRow(Object[] row, Attachment attachment) + throws IOException + { super.asRow(row, attachment); getFileUrlColumn().setRowValue(row, attachment.getFileUrl()); getFileNameColumn().setRowValue(row, attachment.getFileName()); @@ -286,7 +304,7 @@ public class AttachmentColumnInfo extends ComplexColumnInfo _decodedData = decodedData; } - public byte[] getFileData() { + public byte[] getFileData() throws IOException { if((_data == null) && (_decodedData != null)) { _data = encodeData(); } @@ -359,12 +377,19 @@ public class AttachmentColumnInfo extends ComplexColumnInfo } @Override - public String toString() - { + public String toString() { + + String dataStr = null; + try { + dataStr = ByteUtil.toHexString(getFileData()); + } catch(IOException e) { + dataStr = e.toString(); + } + return "Attachment(" + getComplexValueForeignKey() + "," + getId() + ") " + getFileUrl() + ", " + getFileName() + ", " + getFileType() + ", " + getFileTimeStamp() + ", " + getFileFlags() + ", " + - ByteUtil.toHexString(getFileData()); + dataStr; } /** @@ -372,7 +397,7 @@ public class AttachmentColumnInfo extends ComplexColumnInfo */ private byte[] decodeData() throws IOException { - if(_data.length < 8) { + if(_data.length < WRAPPER_HEADER_SIZE) { // nothing we can do throw new IOException("Unknown encoded attachment data format"); } @@ -385,7 +410,7 @@ public class AttachmentColumnInfo extends ComplexColumnInfo DataInputStream contentStream = null; try { InputStream bin = new ByteArrayInputStream( - _data, 8, _data.length - 8); + _data, WRAPPER_HEADER_SIZE, _data.length - WRAPPER_HEADER_SIZE); if(typeFlag == DATA_TYPE_RAW) { // nothing else to do @@ -399,9 +424,9 @@ public class AttachmentColumnInfo extends ComplexColumnInfo contentStream = new DataInputStream(bin); - // header is the "file extension" of the data. no clue why we need - // that again since it's already a separate field in the attachment - // table. just skip it + // header is an unknown flag followed by the "file extension" of the + // data (no clue why we need that again since it's already a separate + // field in the attachment table). just skip all of it byte[] tmpBytes = new byte[4]; contentStream.readFully(tmpBytes); int headerLen = PageChannel.wrap(tmpBytes).getInt(); @@ -428,9 +453,65 @@ public class AttachmentColumnInfo extends ComplexColumnInfo /** * Encodes the actual attachment file data to get the raw, stored format. */ - private byte[] encodeData() { - // FIXME, writeme - throw new UnsupportedOperationException(); + private byte[] encodeData() throws IOException { + + // possibly compress data based on file type + String type = ((_type != null) ? _type.toLowerCase() : ""); + boolean shouldCompress = !COMPRESSED_FORMATS.contains(type); + + // encode extension, which ends w/ a null byte + type += '\0'; + ByteBuffer typeBytes = Column.encodeUncompressedText( + type, JetFormat.VERSION_12.CHARSET); + int headerLen = typeBytes.remaining() + CONTENT_HEADER_SIZE; + + int dataLen = _decodedData.length; + ByteUtil.ByteStream dataStream = new ByteUtil.ByteStream( + WRAPPER_HEADER_SIZE + headerLen + dataLen); + + // write the wrapper header info + ByteBuffer bb = PageChannel.wrap(dataStream.getBytes()); + bb.putInt(shouldCompress ? DATA_TYPE_COMPRESSED : DATA_TYPE_RAW); + bb.putInt(dataLen + headerLen); + dataStream.skip(WRAPPER_HEADER_SIZE); + + OutputStream contentStream = dataStream; + Deflater deflater = null; + try { + + if(shouldCompress) { + contentStream = new DeflaterOutputStream( + contentStream, deflater = new Deflater(3)); + } + + // write the header w/ the file extension + byte[] tmpBytes = new byte[CONTENT_HEADER_SIZE]; + PageChannel.wrap(tmpBytes) + .putInt(headerLen) + .putInt(UNKNOWN_HEADER_VAL) + .putInt(type.length()); + contentStream.write(tmpBytes); + contentStream.write(typeBytes.array(), 0, typeBytes.remaining()); + + // write the _actual_ contents + contentStream.write(_decodedData); + contentStream.close(); + contentStream = null; + + return dataStream.toByteArray(); + + } finally { + if(contentStream != null) { + try { + contentStream.close(); + } catch(IOException e) { + // ignored + } + } + if(deflater != null) { + deflater.end(); + } + } } } diff --git a/src/java/com/healthmarketscience/jackcess/complex/ComplexColumnInfo.java b/src/java/com/healthmarketscience/jackcess/complex/ComplexColumnInfo.java index 0a4b255..3dac47c 100644 --- a/src/java/com/healthmarketscience/jackcess/complex/ComplexColumnInfo.java +++ b/src/java/com/healthmarketscience/jackcess/complex/ComplexColumnInfo.java @@ -349,7 +349,9 @@ public abstract class ComplexColumnInfo _pkCursor.deleteCurrentRow(); } - protected Object[] asRow(Object[] row, V value) { + protected Object[] asRow(Object[] row, V value) + throws IOException + { int id = value.getId(); _pkCol.setRowValue(row, ((id != INVALID_ID) ? id : Column.AUTO_NUMBER)); int cId = value.getComplexValueForeignKey().get(); diff --git a/src/java/com/healthmarketscience/jackcess/complex/ComplexValueForeignKey.java b/src/java/com/healthmarketscience/jackcess/complex/ComplexValueForeignKey.java index 170a92f..13e6b7a 100644 --- a/src/java/com/healthmarketscience/jackcess/complex/ComplexValueForeignKey.java +++ b/src/java/com/healthmarketscience/jackcess/complex/ComplexValueForeignKey.java @@ -327,9 +327,7 @@ public class ComplexValueForeignKey extends Number } @Override - public String toString() - { + public String toString() { return String.valueOf(_value); - } - + } } diff --git a/src/java/com/healthmarketscience/jackcess/complex/MultiValueColumnInfo.java b/src/java/com/healthmarketscience/jackcess/complex/MultiValueColumnInfo.java index b1bec20..efbd8b0 100644 --- a/src/java/com/healthmarketscience/jackcess/complex/MultiValueColumnInfo.java +++ b/src/java/com/healthmarketscience/jackcess/complex/MultiValueColumnInfo.java @@ -73,7 +73,7 @@ public class MultiValueColumnInfo extends ComplexColumnInfo } @Override - protected Object[] asRow(Object[] row, SingleValue value) { + protected Object[] asRow(Object[] row, SingleValue value) throws IOException { super.asRow(row, value); getValueColumn().setRowValue(row, value.get()); return row; diff --git a/src/java/com/healthmarketscience/jackcess/complex/UnsupportedColumnInfo.java b/src/java/com/healthmarketscience/jackcess/complex/UnsupportedColumnInfo.java index 03bd8b1..0eda7f7 100644 --- a/src/java/com/healthmarketscience/jackcess/complex/UnsupportedColumnInfo.java +++ b/src/java/com/healthmarketscience/jackcess/complex/UnsupportedColumnInfo.java @@ -68,7 +68,9 @@ public class UnsupportedColumnInfo extends ComplexColumnInfo } @Override - protected Object[] asRow(Object[] row, UnsupportedValue value) { + protected Object[] asRow(Object[] row, UnsupportedValue value) + throws IOException + { super.asRow(row, value); Map values = value.getValues(); diff --git a/src/java/com/healthmarketscience/jackcess/complex/VersionHistoryColumnInfo.java b/src/java/com/healthmarketscience/jackcess/complex/VersionHistoryColumnInfo.java index 8fc5622..c8df424 100644 --- a/src/java/com/healthmarketscience/jackcess/complex/VersionHistoryColumnInfo.java +++ b/src/java/com/healthmarketscience/jackcess/complex/VersionHistoryColumnInfo.java @@ -135,7 +135,7 @@ public class VersionHistoryColumnInfo extends ComplexColumnInfo } @Override - protected Object[] asRow(Object[] row, Version version) { + protected Object[] asRow(Object[] row, Version version) throws IOException { super.asRow(row, version); getValueColumn().setRowValue(row, version.getValue()); getModifiedDateColumn().setRowValue(row, version.getModifiedDate()); diff --git a/test/src/java/com/healthmarketscience/jackcess/ComplexColumnTest.java b/test/src/java/com/healthmarketscience/jackcess/ComplexColumnTest.java index 6dbcbed..2122b53 100644 --- a/test/src/java/com/healthmarketscience/jackcess/ComplexColumnTest.java +++ b/test/src/java/com/healthmarketscience/jackcess/ComplexColumnTest.java @@ -189,6 +189,11 @@ public class ComplexColumnTest extends TestCase row8ValFk.addAttachment(null, "test_data.txt", "txt", getFileBytes("test_data.txt"), null, null); checkAttachments(row8ValFk.get(), row8ValFk, "test_data.txt"); + row8ValFk.addDecodedAttachment(null, "test_data2.txt", "txt", + getDecodedFileBytes("test_data2.txt"), null, + null); + checkAttachments(row8ValFk.get(), row8ValFk, "test_data.txt", + "test_data2.txt"); Cursor cursor = Cursor.createCursor(t1); assertTrue(cursor.findFirstRow(t1.getColumn("id"), "row4")); -- 2.39.5