Add tests for racy git / Index state encoding

In order to test racy git situations we have to be able to control
the last-modification timestamps of the filesystem. Since we already
access the modification timestamps of files through an abstraction
(the WorkingTreeIterator) I add a new implementation of this iterator
which allows to map timestamp-ranges to single constant timestamps.
For users of this iterator it looks like all files in that range
have been modified at exactly the same time.
With the help of this iterator a test has been writting which
checkes for racy git handling (smudging, unsmudging, dirty-detection).

Additionally add a method to RepositoryTestCase which encodes the
current index state in one String. This should include info about
pathes, file/index modtime, smudgeState, clean-state. Make
sure timestamps are presented in a way that it is easy to
write assertions against this strings (no concrete milliseconds
but t0,t1,...).

These two topics depend circulary on each other: thats why they have
been squashed in one commit.

Change-Id: I115c3f2f20fca9b481830bdc6b9d1ade2c3abdcf
Signed-off-by: Christian Halstrick <christian.halstrick@sap.com>

org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java

@@ -0,0 +1,225 @@
+/*
+ * Copyright (C) 2010, Christian Halstrick <christian.halstrick@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.lib;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.TreeSet;
+
+import org.eclipse.jgit.dircache.DirCache;
+import org.eclipse.jgit.dircache.DirCacheBuilder;
+import org.eclipse.jgit.dircache.DirCacheEntry;
+import org.eclipse.jgit.treewalk.FileTreeIterator;
+import org.eclipse.jgit.treewalk.FileTreeIteratorWithTimeControl;
+import org.eclipse.jgit.treewalk.NameConflictTreeWalk;
+
+public class RacyGitTests extends RepositoryTestCase {
+	public void testIterator() throws IllegalStateException, IOException,
+			InterruptedException {
+		TreeSet<Long> modTimes = new TreeSet<Long>();
+		File lastFile = null;
+		for (int i = 0; i < 10; i++) {
+			lastFile = new File(db.getWorkTree(), "0." + i);
+			lastFile.createNewFile();
+			if (i == 5)
+				fsTick(lastFile);
+		}
+		modTimes.add(fsTick(lastFile));
+		for (int i = 0; i < 10; i++) {
+			lastFile = new File(db.getWorkTree(), "1." + i);
+			lastFile.createNewFile();
+		}
+		modTimes.add(fsTick(lastFile));
+		for (int i = 0; i < 10; i++) {
+			lastFile = new File(db.getWorkTree(), "2." + i);
+			lastFile.createNewFile();
+			if (i % 4 == 0)
+				fsTick(lastFile);
+		}
+		FileTreeIteratorWithTimeControl fileIt = new FileTreeIteratorWithTimeControl(
+				db, modTimes);
+		NameConflictTreeWalk tw = new NameConflictTreeWalk(db);
+		tw.reset();
+		tw.addTree(fileIt);
+		tw.setRecursive(true);
+		FileTreeIterator t;
+		long t0 = 0;
+		for (int i = 0; i < 10; i++) {
+			assertTrue(tw.next());
+			t = tw.getTree(0, FileTreeIterator.class);
+			if (i == 0)
+				t0 = t.getEntryLastModified();
+			else
+				assertEquals(t0, t.getEntryLastModified());
+		}
+		long t1 = 0;
+		for (int i = 0; i < 10; i++) {
+			assertTrue(tw.next());
+			t = tw.getTree(0, FileTreeIterator.class);
+			if (i == 0) {
+				t1 = t.getEntryLastModified();
+				assertTrue(t1 > t0);
+			} else
+				assertEquals(t1, t.getEntryLastModified());
+		}
+		long t2 = 0;
+		for (int i = 0; i < 10; i++) {
+			assertTrue(tw.next());
+			t = tw.getTree(0, FileTreeIterator.class);
+			if (i == 0) {
+				t2 = t.getEntryLastModified();
+				assertTrue(t2 > t1);
+			} else
+				assertEquals(t2, t.getEntryLastModified());
+		}
+	}
+
+	public void testRacyGitDetection() throws IOException,
+			IllegalStateException, InterruptedException {
+		DirCache dc;
+		TreeSet<Long> modTimes = new TreeSet<Long>();
+		File lastFile;
+
+		// wait to ensure that modtimes of the file doesn't match last index
+		// file modtime
+		modTimes.add(fsTick(db.getIndexFile()));
+
+		// create two files
+		addToWorkDir("a", "a");
+		lastFile = addToWorkDir("b", "b");
+
+		// wait to ensure that file-modTimes and therefore index entry modTime
+		// doesn't match the modtime of index-file after next persistance
+		modTimes.add(fsTick(lastFile));
+
+		// now add both files to the index. No racy git expected
+		addToIndex(modTimes);
+
+		assertEquals("[[a, modTime(index/file): t0/t0], [b, modTime(index/file): t0/t0]]", indexState(modTimes));
+
+		// Remember the last modTime of index file. All modifications times of
+		// further modification are translated to this value so it looks that
+		// files have been modified in the same time slot as the index file
+		modTimes.add(Long.valueOf(db.getIndexFile().lastModified()));
+
+		// modify one file
+		addToWorkDir("a", "a2");
+		// now update the index the index. 'a' has to be racily clean -- because
+		// it's modification time is exactly the same as the previous index file
+		// mod time.
+		addToIndex(modTimes);
+
+		dc = db.readDirCache();
+		assertTrue(dc.getEntryCount() == 2);
+		assertTrue(dc.getEntry("a").isSmudged());
+		assertFalse(dc.getEntry("b").isSmudged());
+
+		// although racily clean a should not be reported as beeing dirty
+		assertEquals("[[a, modTime(index/file): t0/t0, unsmudged], [b, modTime(index/file): t1/t1]]", indexState(modTimes));
+		assertEquals("[[a, modTime(index/file): t0/t0, unsmudged], [b, modTime(index/file): t1/t1]]", indexState(modTimes));
+
+	}
+
+	/**
+	 * Waits until it is guaranteed that the filesystem timer (used e.g. for
+	 * lastModified) has a value greater than the lastmodified time of the given
+	 * file. This is done by touch a file, reading the lastmodified and sleeping
+	 * attribute sleeping
+	 *
+	 * @param lastFile
+	 * @return return the last measured value of the filesystem timer which is
+	 *         greater than then the lastmodification time of lastfile.
+	 * @throws InterruptedException
+	 * @throws IOException
+	 */
+	public static long fsTick(File lastFile) throws InterruptedException,
+			IOException {
+		long sleepTime = 1;
+		File tmp = File.createTempFile("FileTreeIteratorWithTimeControl", null);
+		try {
+			long startTime = (lastFile == null) ? tmp.lastModified() : lastFile
+					.lastModified();
+			long actTime = tmp.lastModified();
+			while (actTime <= startTime) {
+				Thread.sleep(sleepTime);
+				sleepTime *= 5;
+				tmp.setLastModified(System.currentTimeMillis());
+				actTime = tmp.lastModified();
+			}
+			return actTime;
+		} finally {
+			tmp.delete();
+		}
+	}
+
+	private void addToIndex(TreeSet<Long> modTimes)
+			throws FileNotFoundException, IOException {
+		DirCacheBuilder builder = db.lockDirCache().builder();
+		FileTreeIterator fIt = new FileTreeIteratorWithTimeControl(
+				db, modTimes);
+		DirCacheEntry dce;
+		while (!fIt.eof()) {
+			dce = new DirCacheEntry(fIt.getEntryPathString());
+			dce.setFileMode(fIt.getEntryFileMode());
+			dce.setLastModified(fIt.getEntryLastModified());
+			dce.setLength((int) fIt.getEntryLength());
+			dce.setObjectId(fIt.getEntryObjectId());
+			builder.add(dce);
+			fIt.next(1);
+		}
+		builder.commit();
+	}
+
+	private File addToWorkDir(String path, String content) throws IOException {
+		File f = new File(db.getWorkTree(), path);
+		FileOutputStream fos = new FileOutputStream(f);
+		try {
+			fos.write(content.getBytes(Constants.CHARACTER_ENCODING));
+			return f;
+		} finally {
+			fos.close();
+		}
+	}
+}

org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RepositoryTestCase.java

@@ -52,9 +52,20 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
 
+import org.eclipse.jgit.dircache.DirCache;
+import org.eclipse.jgit.dircache.DirCacheIterator;
+import org.eclipse.jgit.errors.IncorrectObjectTypeException;
+import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase;
 import org.eclipse.jgit.storage.file.FileRepository;
+import org.eclipse.jgit.treewalk.FileTreeIteratorWithTimeControl;
+import org.eclipse.jgit.treewalk.NameConflictTreeWalk;
 
 /**
  * Base class for most JGit unit tests.
@@ -114,4 +125,77 @@ public abstract class RepositoryTestCase extends LocalDiskRepositoryTestCase {
 		db = createWorkRepository();
 		trash = db.getWorkTree();
 	}
+
+	public String indexState(TreeSet<Long> modTimes)
+			throws IllegalStateException, MissingObjectException,
+			IncorrectObjectTypeException, IOException {
+		DirCache dc = db.readDirCache();
+		Map lookup = new HashMap();
+		List ret = new ArrayList(dc.getEntryCount());
+		NameConflictTreeWalk tw = new NameConflictTreeWalk(db);
+		tw.reset();
+		tw.addTree(new FileTreeIteratorWithTimeControl(db, modTimes));
+		tw.addTree(new DirCacheIterator(dc));
+		boolean smudgedBefore;
+		while (tw.next()) {
+			List entry = new ArrayList(4);
+			FileTreeIteratorWithTimeControl fIt = tw.getTree(0,
+					FileTreeIteratorWithTimeControl.class);
+			DirCacheIterator dcIt = tw.getTree(1, DirCacheIterator.class);
+			entry.add(tw.getPathString());
+			entry.add("modTime(index/file): "
+					+ ((dcIt == null) ? "null" : lookup(Long.valueOf(dcIt
+							.getDirCacheEntry().getLastModified()), "t%n",
+							lookup))
+					+ "/"
+					+ ((fIt == null) ? "null" : lookup(
+							Long.valueOf(fIt.getEntryLastModified()), "t%n",
+							lookup)));
+			smudgedBefore = (dcIt == null) ? false : dcIt.getDirCacheEntry()
+					.isSmudged();
+			if (fIt != null
+					&& dcIt != null
+					&& fIt.isModified(dcIt.getDirCacheEntry(), true, true,
+							db.getFS()))
+				entry.add("dirty");
+			if (dcIt != null && dcIt.getDirCacheEntry().isSmudged())
+				entry.add("smudged");
+			else if (smudgedBefore)
+				entry.add("unsmudged");
+			ret.add(entry);
+		}
+		return ret.toString();
+	}
+
+	/**
+	 * Helper method to map arbitrary objects to user-defined names. This can be
+	 * used create short names for objects to produce small and stable debug
+	 * output. It is guaranteed that when you lookup the same object multiple
+	 * times even with different nameTemplates this method will always return
+	 * the same name which was derived from the first nameTemplate.
+	 * nameTemplates can contain "%n" which will be replaced by a running number
+	 * before used as a name.
+	 *
+	 * @param l
+	 *            the object to lookup
+	 * @param nameTemplate
+	 *            the name for that object. Can contain "%n" which will be
+	 *            replaced by a running number before used as a name. If the
+	 *            lookup table already contains the object this parameter will
+	 *            be ignored
+	 * @param lookupTable
+	 *            a table storing object-name mappings.
+	 * @return a name of that object. Is not guaranteed to be unique. Use
+	 *         nameTemplates containing "%n" to always have uniqe names
+	 */
+	public static String lookup(Object l, String nameTemplate,
+			Map<Object, String> lookupTable) {
+		String name = lookupTable.get(l);
+		if (name == null) {
+			name = nameTemplate.replaceAll("%n",
+					Integer.toString(lookupTable.size()));
+			lookupTable.put(l, name);
+		}
+		return name;
+	}
 }

org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorWithTimeControl.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright (C) 2010, Christian Halstrick <christian.halstrick@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.treewalk;
+
+import java.io.File;
+import java.util.TreeSet;
+
+import org.eclipse.jgit.lib.ObjectReader;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.util.FS;
+
+/**
+ * A {@link FileTreeIterator} used in tests which allows to specify explicitly
+ * what will be returned by {@link #getEntryLastModified()}. This allows to
+ * write tests where certain files have to have the same modification time.
+ * <p>
+ * This iterator is configured by a list of strictly increasing long values
+ * t(0), t(1), ..., t(n). For each file with a modification between t(x) and
+ * t(x+1) [ t(x) <= time < t(x+1) ] this iterator will report t(x). For files
+ * with a modification time smaller t(0) a modification time of 0 is returned.
+ * For files with a modification time greater or equal t(n) t(n) will be
+ * returned.
+ * <p>
+ * This class was written especially to test racy-git problems
+ */
+public class FileTreeIteratorWithTimeControl extends FileTreeIterator {
+	private TreeSet<Long> modTimes = new TreeSet<Long>();
+
+	public FileTreeIteratorWithTimeControl(FileTreeIterator p, Repository repo,
+			TreeSet<Long> modTimes) {
+		super(p, repo.getWorkTree(), repo.getFS());
+		this.modTimes = modTimes;
+	}
+
+	public FileTreeIteratorWithTimeControl(FileTreeIterator p, File f, FS fs,
+			TreeSet<Long> modTimes) {
+		super(p, f, fs);
+		this.modTimes = modTimes;
+	}
+
+	public FileTreeIteratorWithTimeControl(Repository repo,
+			TreeSet<Long> modTimes) {
+		super(repo);
+		this.modTimes = modTimes;
+	}
+
+	public FileTreeIteratorWithTimeControl(File f, FS fs,
+			TreeSet<Long> modTimes) {
+		super(f, fs);
+		this.modTimes = modTimes;
+	}
+
+	@Override
+	public AbstractTreeIterator createSubtreeIterator(final ObjectReader reader) {
+		return new FileTreeIteratorWithTimeControl(this,
+				((FileEntry) current()).file, fs, modTimes);
+	}
+
+	@Override
+	public long getEntryLastModified() {
+		Long cutOff = modTimes
+				.floor(Long.valueOf(super.getEntryLastModified()));
+		return (cutOff == null) ? 0 : cutOff;
+	}
+}