Browse Source

Support creating the working directory difference

If the iterators passed into a diff formatter are working tree
iterators, we should enable ignoring files that are ignored, as
well as actually pull up the current content from the working tree
rather than getting it from the repository.

Because we abstract away the working directory access logic,
we can now actually support rename detection between the working
directory and the local repository when using a DiffFormatter.
This means its possible for an application to show an unstaged
delete-add pair as a rename if the add path is not ignored.
(Because the ignored file wouldn't show up in our difference output.)

Even more interesting is we can now do rename detection between any
two working trees, if both input iterators are WorkingTreeIterators.

Unfortunately we don't (yet) optimize for comparing the working
tree with the index involved so we can take advantage of cached
stat data to rule out non-dirty paths.

Change-Id: I4c0598afe48d8f99257266bf447a0ecd23ca7f5e
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
tags/v0.9.1
Shawn O. Pearce 13 years ago
parent
commit
59a262d5d2

+ 337
- 0
org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java View File

@@ -0,0 +1,337 @@
/*
* Copyright (C) 2010, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

package org.eclipse.jgit.diff;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;

import org.eclipse.jgit.errors.LargeObjectException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.ObjectStream;
import org.eclipse.jgit.treewalk.FileTreeIterator;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.WorkingTreeIterator;
import org.eclipse.jgit.treewalk.filter.PathFilter;

/**
* Supplies the content of a file for {@link DiffFormatter}.
*
* A content source is not thread-safe. Sources may contain state, including
* information about the last ObjectLoader they returned. Callers must be
* careful to ensure there is no more than one ObjectLoader pending on any
* source, at any time.
*/
public abstract class ContentSource {
/**
* Construct a content source for an ObjectReader.
*
* @param reader
* the reader to obtain blobs from.
* @return a source wrapping the reader.
*/
public static ContentSource create(ObjectReader reader) {
return new ObjectReaderSource(reader);
}

/**
* Construct a content source for a working directory.
*
* If the iterator is a {@link FileTreeIterator} an optimized version is
* used that doesn't require seeking through a TreeWalk.
*
* @param iterator
* the iterator to obtain source files through.
* @return a content source wrapping the iterator.
*/
public static ContentSource create(WorkingTreeIterator iterator) {
if (iterator instanceof FileTreeIterator) {
FileTreeIterator i = (FileTreeIterator) iterator;
return new FileSource(i.getDirectory());
}
return new WorkingTreeSource(iterator);
}

/**
* Determine the size of the object.
*
* @param path
* the path of the file, relative to the root of the repository.
* @param id
* blob id of the file, if known.
* @return the size in bytes.
* @throws IOException
* the file cannot be accessed.
*/
public abstract long size(String path, ObjectId id) throws IOException;

/**
* Open the object.
*
* @param path
* the path of the file, relative to the root of the repository.
* @param id
* blob id of the file, if known.
* @return a loader that can supply the content of the file. The loader must
* be used before another loader can be obtained from this same
* source.
* @throws IOException
* the file cannot be accessed.
*/
public abstract ObjectLoader open(String path, ObjectId id)
throws IOException;

private static class ObjectReaderSource extends ContentSource {
private final ObjectReader reader;

ObjectReaderSource(ObjectReader reader) {
this.reader = reader;
}

@Override
public long size(String path, ObjectId id) throws IOException {
return reader.getObjectSize(id, Constants.OBJ_BLOB);
}

@Override
public ObjectLoader open(String path, ObjectId id) throws IOException {
return reader.open(id, Constants.OBJ_BLOB);
}
}

private static class WorkingTreeSource extends ContentSource {
private final TreeWalk tw;

private final WorkingTreeIterator iterator;

private String current;

private WorkingTreeIterator ptr;

WorkingTreeSource(WorkingTreeIterator iterator) {
this.tw = new TreeWalk((ObjectReader) null);
this.iterator = iterator;
}

@Override
public long size(String path, ObjectId id) throws IOException {
seek(path);
return ptr.getEntryLength();
}

@Override
public ObjectLoader open(String path, ObjectId id) throws IOException {
seek(path);
return new ObjectLoader() {
@Override
public long getSize() {
return ptr.getEntryLength();
}

@Override
public int getType() {
return ptr.getEntryFileMode().getObjectType();
}

@Override
public ObjectStream openStream() throws MissingObjectException,
IOException {
InputStream in = ptr.openEntryStream();
in = new BufferedInputStream(in);
return new ObjectStream.Filter(getType(), getSize(), in);
}

@Override
public boolean isLarge() {
return true;
}

@Override
public byte[] getCachedBytes() throws LargeObjectException {
throw new LargeObjectException();
}
};
}

private void seek(String path) throws IOException {
if (!path.equals(current)) {
iterator.reset();
tw.reset();
tw.addTree(iterator);
tw.setFilter(PathFilter.create(path));
current = path;
if (!tw.next())
throw new FileNotFoundException(path);
ptr = tw.getTree(0, WorkingTreeIterator.class);
if (ptr == null)
throw new FileNotFoundException(path);
}
}
}

private static class FileSource extends ContentSource {
private final File root;

FileSource(File root) {
this.root = root;
}

@Override
public long size(String path, ObjectId id) throws IOException {
return new File(root, path).length();
}

@Override
public ObjectLoader open(String path, ObjectId id) throws IOException {
final File p = new File(root, path);
if (!p.isFile())
throw new FileNotFoundException(path);
return new ObjectLoader() {
@Override
public long getSize() {
return p.length();
}

@Override
public int getType() {
return Constants.OBJ_BLOB;
}

@Override
public ObjectStream openStream() throws MissingObjectException,
IOException {
final FileInputStream in = new FileInputStream(p);
final long sz = in.getChannel().size();
final int type = getType();
final BufferedInputStream b = new BufferedInputStream(in);
return new ObjectStream.Filter(type, sz, b);
}

@Override
public boolean isLarge() {
return true;
}

@Override
public byte[] getCachedBytes() throws LargeObjectException {
throw new LargeObjectException();
}
};
}
}

/** A pair of sources to access the old and new sides of a DiffEntry. */
public static final class Pair {
private final ContentSource oldSource;

private final ContentSource newSource;

/**
* Construct a pair of sources.
*
* @param oldSource
* source to read the old side of a DiffEntry.
* @param newSource
* source to read the new side of a DiffEntry.
*/
public Pair(ContentSource oldSource, ContentSource newSource) {
this.oldSource = oldSource;
this.newSource = newSource;
}

/**
* Determine the size of the object.
*
* @param side
* which side of the entry to read (OLD or NEW).
* @param ent
* the entry to examine.
* @return the size in bytes.
* @throws IOException
* the file cannot be accessed.
*/
public long size(DiffEntry.Side side, DiffEntry ent) throws IOException {
switch (side) {
case OLD:
return oldSource.size(ent.oldPath, ent.oldId.toObjectId());
case NEW:
return newSource.size(ent.newPath, ent.newId.toObjectId());
default:
throw new IllegalArgumentException();
}
}

/**
* Open the object.
*
* @param side
* which side of the entry to read (OLD or NEW).
* @param ent
* the entry to examine.
* @return a loader that can supply the content of the file. The loader
* must be used before another loader can be obtained from this
* same source.
* @throws IOException
* the file cannot be accessed.
*/
public ObjectLoader open(DiffEntry.Side side, DiffEntry ent)
throws IOException {
switch (side) {
case OLD:
return oldSource.open(ent.oldPath, ent.oldId.toObjectId());
case NEW:
return newSource.open(ent.newPath, ent.newId.toObjectId());
default:
throw new IllegalArgumentException();
}
}
}
}

+ 42
- 0
org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffEntry.java View File

@@ -82,6 +82,15 @@ public class DiffEntry {
COPY;
}

/** Specify the old or new side for more generalized access. */
public static enum Side {
/** The old side of a DiffEntry. */
OLD,

/** The new side of a DiffEntry. */
NEW;
}

/**
* Create an empty DiffEntry
*/
@@ -284,6 +293,17 @@ public class DiffEntry {
return newPath;
}

/**
* Get the path associated with this file.
*
* @param side
* which path to obtain.
* @return name for this file.
*/
public String getPath(Side side) {
return side == Side.OLD ? getOldPath() : getNewPath();
}

/** @return the old file mode, if described in the patch */
public FileMode getOldMode() {
return oldMode;
@@ -294,6 +314,17 @@ public class DiffEntry {
return newMode;
}

/**
* Get the mode associated with this file.
*
* @param side
* which mode to obtain.
* @return the mode.
*/
public FileMode getMode(Side side) {
return side == Side.OLD ? getOldMode() : getNewMode();
}

/** @return the type of change this patch makes on {@link #getNewPath()} */
public ChangeType getChangeType() {
return changeType;
@@ -326,6 +357,17 @@ public class DiffEntry {
return newId;
}

/**
* Get the object id.
*
* @param side
* the side of the id to get.
* @return the object id; null if there is no index line
*/
public AbbreviatedObjectId getId(Side side) {
return side == Side.OLD ? getOldId() : getNewId();
}

@Override
public String toString() {
StringBuilder buf = new StringBuilder();

+ 51
- 22
org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java View File

@@ -49,6 +49,8 @@ import static org.eclipse.jgit.diff.DiffEntry.ChangeType.COPY;
import static org.eclipse.jgit.diff.DiffEntry.ChangeType.DELETE;
import static org.eclipse.jgit.diff.DiffEntry.ChangeType.MODIFY;
import static org.eclipse.jgit.diff.DiffEntry.ChangeType.RENAME;
import static org.eclipse.jgit.diff.DiffEntry.Side.NEW;
import static org.eclipse.jgit.diff.DiffEntry.Side.OLD;
import static org.eclipse.jgit.lib.Constants.encode;
import static org.eclipse.jgit.lib.Constants.encodeASCII;
import static org.eclipse.jgit.lib.FileMode.GITLINK;
@@ -85,7 +87,9 @@ import org.eclipse.jgit.storage.pack.PackConfig;
import org.eclipse.jgit.treewalk.AbstractTreeIterator;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.WorkingTreeIterator;
import org.eclipse.jgit.treewalk.filter.AndTreeFilter;
import org.eclipse.jgit.treewalk.filter.NotIgnoredFilter;
import org.eclipse.jgit.treewalk.filter.TreeFilter;
import org.eclipse.jgit.util.QuotedString;
import org.eclipse.jgit.util.io.DisabledOutputStream;
@@ -128,6 +132,8 @@ public class DiffFormatter {

private ProgressMonitor progressMonitor;

private ContentSource.Pair source;

/**
* Create a new formatter with a default level of context.
*
@@ -161,6 +167,9 @@ public class DiffFormatter {
db = repository;
reader = db.newObjectReader();

ContentSource cs = ContentSource.create(reader);
source = new ContentSource.Pair(cs, cs);

DiffConfig dc = db.getConfig().get(DiffConfig.KEY);
if (dc.isNoPrefix()) {
setOldPrefix("");
@@ -406,14 +415,17 @@ public class DiffFormatter {
walk.addTree(b);
walk.setRecursive(true);

if (pathFilter == TreeFilter.ALL) {
walk.setFilter(TreeFilter.ANY_DIFF);
} else if (pathFilter instanceof FollowFilter) {
walk.setFilter(pathFilter);
} else {
walk.setFilter(AndTreeFilter
.create(pathFilter, TreeFilter.ANY_DIFF));
}
TreeFilter filter = pathFilter;

if (a instanceof WorkingTreeIterator)
filter = AndTreeFilter.create(filter, new NotIgnoredFilter(0));
if (b instanceof WorkingTreeIterator)
filter = AndTreeFilter.create(filter, new NotIgnoredFilter(1));
if (!(pathFilter instanceof FollowFilter))
filter = AndTreeFilter.create(filter, TreeFilter.ANY_DIFF);
walk.setFilter(filter);

source = new ContentSource.Pair(source(a), source(b));

List<DiffEntry> files = DiffEntry.scan(walk);
if (pathFilter instanceof FollowFilter && isAdd(files)) {
@@ -426,7 +438,13 @@ public class DiffFormatter {
walk.reset();
walk.addTree(a);
walk.addTree(b);
walk.setFilter(TreeFilter.ANY_DIFF);

filter = TreeFilter.ANY_DIFF;
if (a instanceof WorkingTreeIterator)
filter = AndTreeFilter.create(new NotIgnoredFilter(0), filter);
if (b instanceof WorkingTreeIterator)
filter = AndTreeFilter.create(new NotIgnoredFilter(1), filter);
walk.setFilter(filter);

if (renameDetector == null)
setDetectRenames(true);
@@ -438,6 +456,12 @@ public class DiffFormatter {
return files;
}

private ContentSource source(AbstractTreeIterator iterator) {
if (iterator instanceof WorkingTreeIterator)
return ContentSource.create((WorkingTreeIterator) iterator);
return ContentSource.create(reader);
}

private List<DiffEntry> detectRenames(List<DiffEntry> files)
throws IOException {
renameDetector.reset();
@@ -835,12 +859,8 @@ public class DiffFormatter {
} else {
assertHaveRepository();

byte[] aRaw = open(ent.getOldPath(), //
ent.getOldMode(), //
ent.getOldId());
byte[] bRaw = open(ent.getNewPath(), //
ent.getNewMode(), //
ent.getNewId());
byte[] aRaw = open(OLD, ent);
byte[] bRaw = open(NEW, ent);

if (aRaw == BINARY || bRaw == BINARY //
|| RawText.isBinary(aRaw) || RawText.isBinary(bRaw)) {
@@ -878,29 +898,38 @@ public class DiffFormatter {
throw new IllegalStateException(JGitText.get().repositoryIsRequired);
}

private byte[] open(String path, FileMode mode, AbbreviatedObjectId id)
private byte[] open(DiffEntry.Side side, DiffEntry entry)
throws IOException {
if (mode == FileMode.MISSING)
if (entry.getMode(side) == FileMode.MISSING)
return EMPTY;

if (mode.getObjectType() != Constants.OBJ_BLOB)
if (entry.getMode(side).getObjectType() != Constants.OBJ_BLOB)
return EMPTY;

if (isBinary(path))
if (isBinary(entry.getPath(side)))
return BINARY;

AbbreviatedObjectId id = entry.getId(side);
if (!id.isComplete()) {
Collection<ObjectId> ids = reader.resolve(id);
if (ids.size() == 1)
if (ids.size() == 1) {
id = AbbreviatedObjectId.fromObjectId(ids.iterator().next());
else if (ids.size() == 0)
switch (side) {
case OLD:
entry.oldId = id;
break;
case NEW:
entry.newId = id;
break;
}
} else if (ids.size() == 0)
throw new MissingObjectException(id, Constants.OBJ_BLOB);
else
throw new AmbiguousObjectException(id, ids);
}

try {
ObjectLoader ldr = reader.open(id.toObjectId());
ObjectLoader ldr = source.open(side, entry);
return ldr.getBytes(binaryFileThreshold);

} catch (LargeObjectException.ExceedsLimit overLimit) {

+ 27
- 6
org.eclipse.jgit/src/org/eclipse/jgit/diff/RenameDetector.java View File

@@ -43,6 +43,9 @@

package org.eclipse.jgit.diff;

import static org.eclipse.jgit.diff.DiffEntry.Side.NEW;
import static org.eclipse.jgit.diff.DiffEntry.Side.OLD;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -55,7 +58,6 @@ import java.util.List;
import org.eclipse.jgit.JGitText;
import org.eclipse.jgit.diff.DiffEntry.ChangeType;
import org.eclipse.jgit.lib.AbbreviatedObjectId;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.ObjectReader;
@@ -331,6 +333,24 @@ public class RenameDetector {
*/
public List<DiffEntry> compute(ObjectReader reader, ProgressMonitor pm)
throws IOException {
final ContentSource cs = ContentSource.create(reader);
return compute(new ContentSource.Pair(cs, cs), pm);
}

/**
* Detect renames in the current file set.
*
* @param reader
* reader to obtain objects from the repository with.
* @param pm
* report progress during the detection phases.
* @return an unmodifiable list of {@link DiffEntry}s representing all files
* that have been changed.
* @throws IOException
* file contents cannot be read from the repository.
*/
public List<DiffEntry> compute(ContentSource.Pair reader, ProgressMonitor pm)
throws IOException {
if (!done) {
done = true;

@@ -360,7 +380,7 @@ public class RenameDetector {
done = false;
}

private void breakModifies(ObjectReader reader, ProgressMonitor pm)
private void breakModifies(ContentSource.Pair reader, ProgressMonitor pm)
throws IOException {
if (breakScore <= 0)
return;
@@ -423,19 +443,20 @@ public class RenameDetector {
deleted = new ArrayList<DiffEntry>(nameMap.values());
}

private int calculateModifyScore(ObjectReader reader, DiffEntry d)
private int calculateModifyScore(ContentSource.Pair reader, DiffEntry d)
throws IOException {
SimilarityIndex src = new SimilarityIndex();
src.hash(reader.open(d.oldId.toObjectId(), Constants.OBJ_BLOB));
src.hash(reader.open(OLD, d));
src.sort();

SimilarityIndex dst = new SimilarityIndex();
dst.hash(reader.open(d.newId.toObjectId(), Constants.OBJ_BLOB));
dst.hash(reader.open(NEW, d));
dst.sort();
return src.score(dst, 100);
}

private void findContentRenames(ObjectReader reader, ProgressMonitor pm)
private void findContentRenames(ContentSource.Pair reader,
ProgressMonitor pm)
throws IOException {
int cnt = Math.max(added.size(), deleted.size());
if (cnt == 0)

+ 14
- 13
org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java View File

@@ -43,6 +43,9 @@

package org.eclipse.jgit.diff;

import static org.eclipse.jgit.diff.DiffEntry.Side.NEW;
import static org.eclipse.jgit.diff.DiffEntry.Side.OLD;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -50,11 +53,8 @@ import java.util.List;

import org.eclipse.jgit.JGitText;
import org.eclipse.jgit.diff.DiffEntry.ChangeType;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.ProgressMonitor;

class SimilarityRenameDetector {
@@ -72,7 +72,7 @@ class SimilarityRenameDetector {

private static final int SCORE_SHIFT = 2 * BITS_PER_INDEX;

private ObjectReader reader;
private ContentSource.Pair reader;

/**
* All sources to consider for copies or renames.
@@ -112,7 +112,7 @@ class SimilarityRenameDetector {

private List<DiffEntry> out;

SimilarityRenameDetector(ObjectReader reader, List<DiffEntry> srcs,
SimilarityRenameDetector(ContentSource.Pair reader, List<DiffEntry> srcs,
List<DiffEntry> dsts) {
this.reader = reader;
this.srcs = srcs;
@@ -226,7 +226,7 @@ class SimilarityRenameDetector {
continue;
}

SimilarityIndex s = hash(srcEnt.oldId.toObjectId());
SimilarityIndex s = hash(OLD, srcEnt);
for (int dstIdx = 0; dstIdx < dsts.size(); dstIdx++) {
DiffEntry dstEnt = dsts.get(dstIdx);

@@ -242,13 +242,13 @@ class SimilarityRenameDetector {

long srcSize = srcSizes[srcIdx];
if (srcSize < 0) {
srcSize = size(srcEnt.oldId.toObjectId());
srcSize = size(OLD, srcEnt);
srcSizes[srcIdx] = srcSize;
}

long dstSize = dstSizes[dstIdx];
if (dstSize < 0) {
dstSize = size(dstEnt.newId.toObjectId());
dstSize = size(NEW, dstEnt);
dstSizes[dstIdx] = dstSize;
}

@@ -260,7 +260,7 @@ class SimilarityRenameDetector {
continue;
}

SimilarityIndex d = hash(dstEnt.newId.toObjectId());
SimilarityIndex d = hash(NEW, dstEnt);
int contentScore = s.score(d, 10000);

// nameScore returns a value between 0 and 100, but we want it
@@ -335,15 +335,16 @@ class SimilarityRenameDetector {
return (((dirScoreLtr + dirScoreRtl) * 25) + (fileScore * 50)) / 100;
}

private SimilarityIndex hash(ObjectId objectId) throws IOException {
private SimilarityIndex hash(DiffEntry.Side side, DiffEntry ent)
throws IOException {
SimilarityIndex r = new SimilarityIndex();
r.hash(reader.open(objectId));
r.hash(reader.open(side, ent));
r.sort();
return r;
}

private long size(ObjectId objectId) throws IOException {
return reader.getObjectSize(objectId, Constants.OBJ_BLOB);
private long size(DiffEntry.Side side, DiffEntry ent) throws IOException {
return reader.size(side, ent);
}

private static int score(long value) {

Loading…
Cancel
Save