From eda2e95fa8748813cbcfc7b7fbedefb002999cfa Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 10 Jul 2019 16:22:15 +0200 Subject: Measure filesystem timestamp resolution already in test setup This helps to avoid some time critical tests can't prepare the test fixture intended since measuring timestamp resolution takes time. Change-Id: Ib34023e682a106070ca97e98ef16789a4dfb97b4 Signed-off-by: Matthias Sohn --- .../tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit') diff --git a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java index 10823b8788..63af6eb52b 100644 --- a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java +++ b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java @@ -82,6 +82,7 @@ import org.eclipse.jgit.lfs.lib.LongObjectId; import org.eclipse.jgit.lfs.server.LargeFileRepository; import org.eclipse.jgit.lfs.server.LfsProtocolServlet; import org.eclipse.jgit.lfs.test.LongObjectIdTestUtils; +import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.IO; import org.junit.After; @@ -119,6 +120,11 @@ public abstract class LfsServerTest { @Before public void setup() throws Exception { tmp = Files.createTempDirectory("jgit_test_"); + + // measure timer resolution before the test to avoid time critical tests + // are affected by time needed for measurement + FS.getFsTimerResolution(tmp.getParent()); + server = new AppServer(); ServletContextHandler app = server.addContext("/lfs"); dir = Paths.get(tmp.toString(), "lfs"); -- cgit v1.2.3 From 5911521ba6d47868871c4b5f240c71af827b7aa2 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 15 Jul 2019 15:00:09 +0200 Subject: Measure minimum racy interval to auto-configure FileSnapshot By running FileSnapshotTest#detectFileModified we found that the sum of measured filesystem timestamp resolution and measured clock resolution may yield a too small interval after a file has been modified which we need to consider racily clean. In our tests we didn't find this behavior on all systems we tested on, e.g. on MacOS using APFS and Java 8 and 11 this effect was not observed. On Linux (SLES 15, kernel 4.12.14-150.22-default) we collected the following test results using Java 8 and 11: In 23-98% of 10000 test runs (depending on filesystem type and Java version) the test failed, which means the effective interval which needs to be considered racily clean after a file was modified is larger than the measured file timestamp resolution. "delta" is the observed interval after a file has been modified but FileSnapshot did not yet detect the modification: "resolution" is the measured sum of file timestamp resolution and clock resolution seen in Java. Java version filesystem failures resolution min delta max delta 1.8.0_212-b04 btrfs 98.6% 1 ms 3.6 ms 6.6 ms 1.8.0_212-b04 ext4 82.6% 3 ms 1.1 ms 4.1 ms 1.8.0_212-b04 xfs 23.8% 4 ms 3.7 ms 3.9 ms 1.8.0_212-b04 zfs 23.1% 3 ms 4.8 ms 5.0 ms 11.0.3+7 btrfs 98.1% 3 us 0.7 ms 4.7 ms 11.0.3+7 ext4 98.1% 6 us 0.7 ms 4.7 ms 11.0.3+7 xfs 98.5% 7 us 0.1 ms 8.0 ms 11.0.3+7 zfs 98.4% 7 us 0.7 ms 5.2 ms Mac OS 1.8.0_212 APFS 0% 1 s 11.0.3+7 APFS 0% 6 us The observed delta is not distributed according to a normal gaussian distribution but rather random in the observed range between "min delta" and "max delta". Run this test after measuring file timestamp resolution in FS.FileAttributeCache to auto-configure JGit since it's unclear what mechanism is causing this effect. In FileSnapshot#isRacyClean use the maximum of the measured timestamp resolution and the measured "delta" as explained above to decide if a given FileSnapshot is to be considered racily clean. Add a 30% safety margin to ensure we are on the safe side. Change-Id: I1c8bb59f6486f174b7bbdc63072777ddbe06694d Signed-off-by: Matthias Sohn --- .../eclipse/jgit/ant/tasks/GitCloneTaskTest.java | 2 +- .../jgit/junit/LocalDiskRepositoryTestCase.java | 2 +- .../org/eclipse/jgit/junit/RepositoryTestCase.java | 3 +- .../eclipse/jgit/lfs/server/fs/LfsServerTest.java | 2 +- org.eclipse.jgit.test/tst-rsrc/log4j.properties | 5 +- .../internal/storage/file/FileSnapshotTest.java | 28 ++- .../jgit/storage/file/FileBasedConfigTest.java | 2 +- .../tst/org/eclipse/jgit/util/FSTest.java | 3 +- org.eclipse.jgit/.settings/.api_filters | 24 +-- .../jgit/internal/storage/file/FileSnapshot.java | 48 +++-- .../src/org/eclipse/jgit/lib/Constants.java | 11 -- org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 201 ++++++++++++++++++--- 12 files changed, 245 insertions(+), 86 deletions(-) (limited to 'org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit') diff --git a/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java b/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java index 1d7187a312..9f9d459a6c 100644 --- a/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java +++ b/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java @@ -66,7 +66,7 @@ public class GitCloneTaskTest extends LocalDiskRepositoryTestCase { @Before public void before() throws IOException { dest = createTempFile(); - FS.getFsTimerResolution(dest.toPath().getParent()); + FS.getFileStoreAttributeCache(dest.toPath().getParent()); project = new Project(); project.init(); enableLogging(); diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java index 62dfc5d9c0..fb8295fa4b 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java @@ -130,7 +130,7 @@ public abstract class LocalDiskRepositoryTestCase { // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - FS.getFsTimerResolution(tmp.toPath().getParent()); + FS.getFileStoreAttributeCache(tmp.toPath().getParent()); mockSystemReader = new MockSystemReader(); mockSystemReader.userGitConfig = new FileBasedConfig(new File(tmp, diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java index 49f5c5febb..ebd13e4112 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java @@ -378,7 +378,8 @@ public abstract class RepositoryTestCase extends LocalDiskRepositoryTestCase { tmp = File.createTempFile("fsTickTmpFile", null, lastFile.getParentFile()); } - long res = FS.getFsTimerResolution(tmp.toPath()).toNanos(); + long res = FS.getFileStoreAttributeCache(tmp.toPath()) + .getFsTimestampResolution().toNanos(); long sleepTime = res / 10; try { Instant startTime = fs.lastModifiedInstant(lastFile); diff --git a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java index 63af6eb52b..92a6ec351f 100644 --- a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java +++ b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java @@ -123,7 +123,7 @@ public abstract class LfsServerTest { // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - FS.getFsTimerResolution(tmp.getParent()); + FS.getFileStoreAttributeCache(tmp.getParent()); server = new AppServer(); ServletContextHandler app = server.addContext("/lfs"); diff --git a/org.eclipse.jgit.test/tst-rsrc/log4j.properties b/org.eclipse.jgit.test/tst-rsrc/log4j.properties index a48a4022ff..ee1ac35158 100644 --- a/org.eclipse.jgit.test/tst-rsrc/log4j.properties +++ b/org.eclipse.jgit.test/tst-rsrc/log4j.properties @@ -8,4 +8,7 @@ log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n #log4j.appender.fileLogger.bufferedIO = true -#log4j.appender.fileLogger.bufferSize = 1024 \ No newline at end of file +#log4j.appender.fileLogger.bufferSize = 4096 + +#log4j.logger.org.eclipse.jgit.util.FS = DEBUG +#log4j.logger.org.eclipse.jgit.internal.storage.file.FileSnapshot = DEBUG diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index 9eb55db09c..012407f715 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -62,6 +62,7 @@ import java.util.ArrayList; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.FS.FileStoreAttributeCache; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.Stats; import org.eclipse.jgit.util.SystemReader; @@ -78,14 +79,15 @@ public class FileSnapshotTest { private Path trash; - private Duration fsTimerResolution; + private FileStoreAttributeCache fsAttrCache; @Before public void setUp() throws Exception { trash = Files.createTempDirectory("tmp_"); // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - fsTimerResolution = FS.getFsTimerResolution(trash.getParent()); + fsAttrCache = FS + .getFileStoreAttributeCache(trash.getParent()); } @Before @@ -131,11 +133,13 @@ public class FileSnapshotTest { // if filesystem timestamp resolution is high the snapshot won't be // racily clean Assume.assumeTrue( - fsTimerResolution.compareTo(Duration.ofMillis(10)) > 0); + fsAttrCache.getFsTimestampResolution() + .compareTo(Duration.ofMillis(10)) > 0); Path f1 = createFile("newfile"); waitNextTick(f1); FileSnapshot save = FileSnapshot.save(f1.toFile()); - TimeUnit.NANOSECONDS.sleep(fsTimerResolution.dividedBy(2).toNanos()); + TimeUnit.NANOSECONDS.sleep( + fsAttrCache.getFsTimestampResolution().dividedBy(2).toNanos()); assertTrue(save.isModified(f1.toFile())); } @@ -149,7 +153,8 @@ public class FileSnapshotTest { // if filesystem timestamp resolution is high the snapshot won't be // racily clean Assume.assumeTrue( - fsTimerResolution.compareTo(Duration.ofMillis(10)) > 0); + fsAttrCache.getFsTimestampResolution() + .compareTo(Duration.ofMillis(10)) > 0); Path f1 = createFile("newfile"); FileSnapshot save = FileSnapshot.save(f1.toFile()); assertTrue(save.isModified(f1.toFile())); @@ -230,7 +235,7 @@ public class FileSnapshotTest { write(f, "b"); if (!snapshot.isModified(f)) { deltas.add(snapshot.lastDelta()); - racyNanos = snapshot.lastRacyNanos(); + racyNanos = snapshot.lastRacyThreshold(); failures++; } assertEquals("file should contain 'b'", "b", read(f)); @@ -244,7 +249,7 @@ public class FileSnapshotTest { LOG.debug(String.format("%,d", d)); } LOG.error( - "count, failures, racy limit [ns], delta min [ns]," + "count, failures, eff. racy threshold [ns], delta min [ns]," + " delta max [ns], delta avg [ns]," + " delta stddev [ns]"); LOG.error(String.format( @@ -253,7 +258,14 @@ public class FileSnapshotTest { stats.avg(), stats.stddev())); } assertTrue( - "FileSnapshot: number of failures to detect file modifications should be 0", + String.format( + "FileSnapshot: failures to detect file modifications" + + " %d out of %d\n" + + "timestamp resolution %d µs" + + " min racy threshold %d µs" + , failures, COUNT, + fsAttrCache.getFsTimestampResolution().toNanos() / 1000, + fsAttrCache.getMinimalRacyInterval().toNanos() / 1000), failures == 0); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java index d3686285e3..77f5febc17 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java @@ -83,7 +83,7 @@ public class FileBasedConfigTest { @Before public void setUp() throws Exception { trash = Files.createTempDirectory("tmp_"); - FS.getFsTimerResolution(trash.getParent()); + FS.getFileStoreAttributeCache(trash.getParent()); } @After diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java index bde8a8a6b3..63e295ec83 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java @@ -203,7 +203,8 @@ public class FSTest { .ofPattern("uuuu-MMM-dd HH:mm:ss.nnnnnnnnn", Locale.ENGLISH) .withZone(ZoneId.systemDefault()); Path dir = Files.createTempDirectory("probe-filesystem"); - Duration resolution = FS.getFsTimerResolution(dir); + Duration resolution = FS.getFileStoreAttributeCache(dir) + .getFsTimestampResolution(); long resolutionNs = resolution.toNanos(); assertTrue(resolutionNs > 0); for (int i = 0; i < 10; i++) { diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 59bafc52e0..a027caaf02 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -56,14 +56,6 @@ - - - - - - - - @@ -179,6 +171,12 @@ + + + + + + @@ -203,18 +201,20 @@ + + - - + + - + - + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 2a490a4a1f..aa9f1cc45b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.internal.storage.file; -import static org.eclipse.jgit.lib.Constants.FALLBACK_TIMESTAMP_RESOLUTION; +import static org.eclipse.jgit.util.FS.FileStoreAttributeCache.FALLBACK_FILESTORE_ATTRIBUTES; import java.io.File; import java.io.IOException; @@ -58,6 +58,7 @@ import java.util.concurrent.TimeUnit; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.FS.FileStoreAttributeCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -213,8 +214,8 @@ public class FileSnapshot { * When set to {@link #UNKNOWN_SIZE} the size is not considered for modification checks. */ private final long size; - /** measured filesystem timestamp resolution */ - private Duration fsTimestampResolution; + /** measured FileStore attributes */ + private FileStoreAttributeCache fileStoreAttributeCache; /** * Object that uniquely identifies the given file, or {@code @@ -252,9 +253,9 @@ public class FileSnapshot { protected FileSnapshot(File file, boolean useConfig) { this.file = file; this.lastRead = Instant.now(); - this.fsTimestampResolution = useConfig - ? FS.getFsTimerResolution(file.toPath().getParent()) - : FALLBACK_TIMESTAMP_RESOLUTION; + this.fileStoreAttributeCache = useConfig + ? FS.getFileStoreAttributeCache(file.toPath().getParent()) + : FALLBACK_FILESTORE_ATTRIBUTES; BasicFileAttributes fileAttributes = null; try { fileAttributes = FS.DETECTED.fileAttributes(file); @@ -285,14 +286,15 @@ public class FileSnapshot { private long delta; - private long racyNanos; + private long racyThreshold; private FileSnapshot(Instant read, Instant modified, long size, @NonNull Duration fsTimestampResolution, @NonNull Object fileKey) { this.file = null; this.lastRead = read; this.lastModified = modified; - this.fsTimestampResolution = fsTimestampResolution; + this.fileStoreAttributeCache = new FileStoreAttributeCache( + fsTimestampResolution); this.size = size; this.fileKey = fileKey; } @@ -397,9 +399,10 @@ public class FileSnapshot { * if sleep was interrupted */ public void waitUntilNotRacy() throws InterruptedException { + long timestampResolution = fileStoreAttributeCache + .getFsTimestampResolution().toNanos(); while (isRacyClean(Instant.now())) { - TimeUnit.NANOSECONDS - .sleep((fsTimestampResolution.toNanos() + 1) * 11 / 10); + TimeUnit.NANOSECONDS.sleep(timestampResolution); } } @@ -474,15 +477,16 @@ public class FileSnapshot { * @return the delta in nanoseconds between lastModified and lastRead during * last racy check */ - long lastDelta() { + public long lastDelta() { return delta; } /** - * @return the racyNanos threshold in nanoseconds during last racy check + * @return the racyLimitNanos threshold in nanoseconds during last racy + * check */ - long lastRacyNanos() { - return racyNanos; + public long lastRacyThreshold() { + return racyThreshold; } /** {@inheritDoc} */ @@ -501,20 +505,28 @@ public class FileSnapshot { } private boolean isRacyClean(Instant read) { - // add a 10% safety margin - racyNanos = (fsTimestampResolution.toNanos() + 1) * 11 / 10; + racyThreshold = getEffectiveRacyThreshold(); delta = Duration.between(lastModified, read).toNanos(); - wasRacyClean = delta <= racyNanos; + wasRacyClean = delta <= racyThreshold; if (LOG.isDebugEnabled()) { LOG.debug( "file={}, isRacyClean={}, read={}, lastModified={}, delta={} ns, racy<={} ns", //$NON-NLS-1$ file, Boolean.valueOf(wasRacyClean), dateFmt.format(read), dateFmt.format(lastModified), Long.valueOf(delta), - Long.valueOf(racyNanos)); + Long.valueOf(racyThreshold)); } return wasRacyClean; } + private long getEffectiveRacyThreshold() { + long timestampResolution = fileStoreAttributeCache + .getFsTimestampResolution().toNanos(); + long minRacyInterval = fileStoreAttributeCache.getMinimalRacyInterval() + .toNanos(); + // add a 30% safety margin + return Math.max(timestampResolution, minRacyInterval) * 13 / 10; + } + private boolean isModified(Instant currLastModified) { // Any difference indicates the path was modified. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 94fc100386..4c55196961 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -52,7 +52,6 @@ import java.nio.charset.Charset; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.MessageFormat; -import java.time.Duration; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.internal.JGitText; @@ -723,16 +722,6 @@ public final class Constants { */ public static final String LOCK_SUFFIX = ".lock"; //$NON-NLS-1$ - /** - * Fallback filesystem timestamp resolution used when we can't measure the - * resolution. The last modified time granularity of FAT filesystems is 2 - * seconds. - * - * @since 5.1.9 - */ - public static final Duration FALLBACK_TIMESTAMP_RESOLUTION = Duration - .ofMillis(2000); - private Constants() { // Hide the default constructor } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 081776f081..08dab3201d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -45,7 +45,6 @@ package org.eclipse.jgit.util; import static java.nio.charset.StandardCharsets.UTF_8; import static java.time.Instant.EPOCH; -import static org.eclipse.jgit.lib.Constants.FALLBACK_TIMESTAMP_RESOLUTION; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -55,7 +54,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.io.PrintStream; +import java.io.Writer; import java.nio.charset.Charset; import java.nio.file.AccessDeniedException; import java.nio.file.FileStore; @@ -68,6 +69,7 @@ import java.security.PrivilegedAction; import java.text.MessageFormat; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -94,6 +96,7 @@ import org.eclipse.jgit.errors.CommandFailedException; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.FileSnapshot; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Repository; @@ -200,11 +203,24 @@ public abstract class FS { } } - private static final class FileStoreAttributeCache { + /** + * Attributes of FileStores on this system + * + * @since 5.1.9 + */ + public final static class FileStoreAttributeCache { private static final Duration UNDEFINED_RESOLUTION = Duration .ofNanos(Long.MAX_VALUE); + /** + * Fallback FileStore attributes used when we can't measure the + * filesystem timestamp resolution. The last modified time granularity + * of FAT filesystems is 2 seconds. + */ + public static final FileStoreAttributeCache FALLBACK_FILESTORE_ATTRIBUTES = new FileStoreAttributeCache( + Duration.ofMillis(2000)); + private static final Map attributeCache = new ConcurrentHashMap<>(); private static AtomicBoolean background = new AtomicBoolean(); @@ -216,36 +232,58 @@ public abstract class FS { } private static final String javaVersionPrefix = System - .getProperty("java.vm.vendor") + '|' //$NON-NLS-1$ - + System.getProperty("java.vm.version") + '|'; //$NON-NLS-1$ + .getProperty("java.vendor") + '|' //$NON-NLS-1$ + + System.getProperty("java.version") + '|'; //$NON-NLS-1$ + + private static final Duration FALLBACK_MIN_RACY_INTERVAL = Duration + .ofMillis(10); - private static Duration getFsTimestampResolution(Path file) { - file = file.toAbsolutePath(); - Path dir = Files.isDirectory(file) ? file : file.getParent(); + /** + * @param path + * file residing in the FileStore to get attributes for + * @return FileStoreAttributeCache entry for the given path. + */ + public static FileStoreAttributeCache get(Path path) { + path = path.toAbsolutePath(); + Path dir = Files.isDirectory(path) ? path : path.getParent(); + return getFileAttributeCache(dir); + } + + private static FileStoreAttributeCache getFileAttributeCache(Path dir) { FileStore s; try { if (Files.exists(dir)) { s = Files.getFileStore(dir); FileStoreAttributeCache c = attributeCache.get(s); if (c != null) { - return c.getFsTimestampResolution(); + return c; } if (!Files.isWritable(dir)) { // cannot measure resolution in a read-only directory - return FALLBACK_TIMESTAMP_RESOLUTION; + LOG.debug( + "{}: cannot measure timestamp resolution in read-only directory {}", //$NON-NLS-1$ + Thread.currentThread(), dir); + return FALLBACK_FILESTORE_ATTRIBUTES; } } else { // cannot determine FileStore of an unborn directory - return FALLBACK_TIMESTAMP_RESOLUTION; + LOG.debug( + "{}: cannot measure timestamp resolution of unborn directory {}", //$NON-NLS-1$ + Thread.currentThread(), dir); + return FALLBACK_FILESTORE_ATTRIBUTES; } - CompletableFuture> f = CompletableFuture + CompletableFuture> f = CompletableFuture .supplyAsync(() -> { Lock lock = locks.computeIfAbsent(s, l -> new ReentrantLock()); if (!lock.tryLock()) { + LOG.debug( + "{}: couldn't get lock to measure timestamp resolution in {}", //$NON-NLS-1$ + Thread.currentThread(), dir); return Optional.empty(); } - Optional resolution; + Optional cache = Optional + .empty(); try { // Some earlier future might have set the value // and removed itself since we checked for the @@ -253,28 +291,36 @@ public abstract class FS { FileStoreAttributeCache c = attributeCache .get(s); if (c != null) { - return Optional - .of(c.getFsTimestampResolution()); + return Optional.of(c); } - resolution = measureFsTimestampResolution(s, - dir); + Optional resolution = measureFsTimestampResolution( + s, dir); if (resolution.isPresent()) { - FileStoreAttributeCache cache = new FileStoreAttributeCache( + c = new FileStoreAttributeCache( resolution.get()); - attributeCache.put(s, cache); + attributeCache.put(s, c); + // for high timestamp resolution measure + // minimal racy interval + if (c.fsTimestampResolution + .toNanos() < 100_000_000L) { + c.minimalRacyInterval = measureMinimalRacyInterval( + dir); + } if (LOG.isDebugEnabled()) { - LOG.debug(cache.toString()); + LOG.debug(c.toString()); } + cache = Optional.of(c); } } finally { lock.unlock(); locks.remove(s); } - return resolution; + return cache; }); // even if measuring in background wait a little - if the result // arrives, it's better than returning the large fallback - Optional d = f.get(background.get() ? 50 : 2000, + Optional d = f.get( + background.get() ? 100 : 5000, TimeUnit.MILLISECONDS); if (d.isPresent()) { return d.get(); @@ -286,11 +332,79 @@ public abstract class FS { } catch (TimeoutException | SecurityException e) { // use fallback } - return FALLBACK_TIMESTAMP_RESOLUTION; + LOG.debug("{}: use fallback timestamp resolution for directory {}", //$NON-NLS-1$ + Thread.currentThread(), dir); + return FALLBACK_FILESTORE_ATTRIBUTES; + } + + @SuppressWarnings("boxing") + private static Duration measureMinimalRacyInterval(Path dir) { + LOG.debug("{}: start measure minimal racy interval in {}", //$NON-NLS-1$ + Thread.currentThread(), dir); + int failures = 0; + long racyNanos = 0; + final int COUNT = 1000; + ArrayList deltas = new ArrayList<>(); + Path probe = dir.resolve(".probe-" + UUID.randomUUID()); //$NON-NLS-1$ + try { + Files.createFile(probe); + for (int i = 0; i < COUNT; i++) { + write(probe, "a"); //$NON-NLS-1$ + FileSnapshot snapshot = FileSnapshot.save(probe.toFile()); + read(probe); + write(probe, "b"); //$NON-NLS-1$ + if (!snapshot.isModified(probe.toFile())) { + deltas.add(Long.valueOf(snapshot.lastDelta())); + racyNanos = snapshot.lastRacyThreshold(); + failures++; + } + } + } catch (IOException e) { + LOG.error(e.getMessage(), e); + return FALLBACK_MIN_RACY_INTERVAL; + } finally { + deleteProbe(probe); + } + if (failures > 0) { + Stats stats = new Stats(); + for (Long d : deltas) { + stats.add(d); + } + LOG.debug( + "delta [ns] since modification FileSnapshot failed to detect\n" //$NON-NLS-1$ + + "count, failures, racy limit [ns], delta min [ns]," //$NON-NLS-1$ + + " delta max [ns], delta avg [ns]," //$NON-NLS-1$ + + " delta stddev [ns]\n" //$NON-NLS-1$ + + "{}, {}, {}, {}, {}, {}, {}", //$NON-NLS-1$ + COUNT, failures, racyNanos, stats.min(), stats.max(), + stats.avg(), stats.stddev()); + return Duration + .ofNanos(Double.valueOf(stats.max()).longValue()); + } + // since no failures occurred using the measured filesystem + // timestamp resolution there is no need for minimal racy interval + LOG.debug("{}: no failures when measuring minimal racy interval", //$NON-NLS-1$ + Thread.currentThread()); + return Duration.ZERO; + } + + private static void write(Path p, String body) throws IOException { + FileUtils.mkdirs(p.getParent().toFile(), true); + try (Writer w = new OutputStreamWriter(Files.newOutputStream(p), + UTF_8)) { + w.write(body); + } + } + + private static String read(Path p) throws IOException { + final byte[] body = IO.readFully(p.toFile()); + return new String(body, 0, body.length, UTF_8); } private static Optional measureFsTimestampResolution( FileStore s, Path dir) { + LOG.debug("{}: start measure timestamp resolution {} in {}", //$NON-NLS-1$ + Thread.currentThread(), s, dir); Duration configured = readFileTimeResolution(s); if (!UNDEFINED_RESOLUTION.equals(configured)) { return Optional.of(configured); @@ -310,6 +424,8 @@ public abstract class FS { Duration clockResolution = measureClockResolution(); fsResolution = fsResolution.plus(clockResolution); saveFileTimeResolution(s, fsResolution); + LOG.debug("{}: end measure timestamp resolution {} in {}", //$NON-NLS-1$ + Thread.currentThread(), s, dir); return Optional.of(fsResolution); } catch (AccessDeniedException e) { LOG.warn(e.getLocalizedMessage(), e); // see bug 548648 @@ -424,21 +540,45 @@ public abstract class FS { private final @NonNull Duration fsTimestampResolution; + private Duration minimalRacyInterval; + + /** + * @return the measured minimal interval after a file has been modified + * in which we cannot rely on lastModified to detect + * modifications + */ + public Duration getMinimalRacyInterval() { + return minimalRacyInterval; + } + + /** + * @return the measured filesystem timestamp resolution + */ @NonNull - Duration getFsTimestampResolution() { + public Duration getFsTimestampResolution() { return fsTimestampResolution; } - private FileStoreAttributeCache( + /** + * Construct a FileStoreAttributeCache entry for the given filesystem + * timestamp resolution + * + * @param fsTimestampResolution + */ + public FileStoreAttributeCache( @NonNull Duration fsTimestampResolution) { this.fsTimestampResolution = fsTimestampResolution; + this.minimalRacyInterval = Duration.ZERO; } - @SuppressWarnings("nls") + @SuppressWarnings({ "nls", "boxing" }) @Override public String toString() { - return "FileStoreAttributeCache [fsTimestampResolution=" - + fsTimestampResolution + "]"; + return String.format( + "FileStoreAttributeCache[fsTimestampResolution=%,d µs, " + + "minimalRacyInterval=%,d µs]", + fsTimestampResolution.toNanos() / 1000, + minimalRacyInterval.toNanos() / 1000); } } @@ -507,10 +647,11 @@ public abstract class FS { * the directory under which the probe file will be created to * measure the timer resolution. * @return measured filesystem timestamp resolution - * @since 5.2.3 + * @since 5.1.9 */ - public static Duration getFsTimerResolution(@NonNull Path dir) { - return FileStoreAttributeCache.getFsTimestampResolution(dir); + public static FileStoreAttributeCache getFileStoreAttributeCache( + @NonNull Path dir) { + return FileStoreAttributeCache.get(dir); } private volatile Holder userHome; -- cgit v1.2.3 From d45219baacab711abf3c4112146ca0522d984be2 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 17 Jul 2019 16:31:42 +0200 Subject: Persist minimal racy threshold and allow manual configuration To enable persisting the minimal racy threshold per FileStore add a new config option to the user global git configuration: - Config section is "filesystem" - Config subsection is concatenation of - Java vendor (system property "java.vendor") - Java version (system property "java.version") - FileStore's name, on Windows we use the attribute volume:vsn instead since the name is not necessarily unique. - separated by '|' e.g. "AdoptOpenJDK|1.8.0_212-b03|/dev/disk1s1" The same prefix is used as for filesystem timestamp resolution, so both values are stored in the same config section - The config key for minmal racy threshold is "minRacyThreshold" as a time value, supported time units are those supported by DefaultTypedConfigGetter#getTimeUnit - measure for 3 seconds to limit runtime which depends on hardware, OS and Java version being used If the minimal racy threshold is configured for a given FileStore the configured value is used instead of measuring it. When the minimal racy threshold was measured it is persisted in the user global git configuration. Rename FileStoreAttributeCache to FileStoreAttributes since this class is now declared public in order to enable exposing all attributes in one object. Example: [filesystem "AdoptOpenJDK|11.0.3|/dev/disk1s1"] timestampResolution = 7000 nanoseconds minRacyThreshold = 3440 microseconds Change-Id: I22195e488453aae8d011b0a8e3276fe3d99deaea Signed-off-by: Matthias Sohn Also-By: Marc Strapetz --- .../eclipse/jgit/ant/tasks/GitCloneTaskTest.java | 2 +- .../jgit/junit/LocalDiskRepositoryTestCase.java | 2 +- .../org/eclipse/jgit/junit/RepositoryTestCase.java | 2 +- .../eclipse/jgit/lfs/server/fs/LfsServerTest.java | 2 +- .../internal/storage/file/FileSnapshotTest.java | 6 +- .../jgit/storage/file/FileBasedConfigTest.java | 2 +- .../tst/org/eclipse/jgit/util/FSTest.java | 2 +- org.eclipse.jgit/.settings/.api_filters | 22 ++- .../jgit/internal/storage/file/FileSnapshot.java | 10 +- .../src/org/eclipse/jgit/lib/ConfigConstants.java | 7 + .../eclipse/jgit/storage/file/FileBasedConfig.java | 30 ++- org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 202 +++++++++++++-------- 12 files changed, 195 insertions(+), 94 deletions(-) (limited to 'org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit') diff --git a/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java b/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java index 9f9d459a6c..8043d2b183 100644 --- a/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java +++ b/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java @@ -66,7 +66,7 @@ public class GitCloneTaskTest extends LocalDiskRepositoryTestCase { @Before public void before() throws IOException { dest = createTempFile(); - FS.getFileStoreAttributeCache(dest.toPath().getParent()); + FS.getFileStoreAttributes(dest.toPath().getParent()); project = new Project(); project.init(); enableLogging(); diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java index fb8295fa4b..af23ad1e35 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java @@ -130,7 +130,7 @@ public abstract class LocalDiskRepositoryTestCase { // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - FS.getFileStoreAttributeCache(tmp.toPath().getParent()); + FS.getFileStoreAttributes(tmp.toPath().getParent()); mockSystemReader = new MockSystemReader(); mockSystemReader.userGitConfig = new FileBasedConfig(new File(tmp, diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java index ebd13e4112..5aacbbadec 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java @@ -378,7 +378,7 @@ public abstract class RepositoryTestCase extends LocalDiskRepositoryTestCase { tmp = File.createTempFile("fsTickTmpFile", null, lastFile.getParentFile()); } - long res = FS.getFileStoreAttributeCache(tmp.toPath()) + long res = FS.getFileStoreAttributes(tmp.toPath()) .getFsTimestampResolution().toNanos(); long sleepTime = res / 10; try { diff --git a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java index 92a6ec351f..ec44da4cac 100644 --- a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java +++ b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java @@ -123,7 +123,7 @@ public abstract class LfsServerTest { // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - FS.getFileStoreAttributeCache(tmp.getParent()); + FS.getFileStoreAttributes(tmp.getParent()); server = new AppServer(); ServletContextHandler app = server.addContext("/lfs"); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index 012407f715..40af9e2a00 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -62,7 +62,7 @@ import java.util.ArrayList; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.util.FS; -import org.eclipse.jgit.util.FS.FileStoreAttributeCache; +import org.eclipse.jgit.util.FS.FileStoreAttributes; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.Stats; import org.eclipse.jgit.util.SystemReader; @@ -79,7 +79,7 @@ public class FileSnapshotTest { private Path trash; - private FileStoreAttributeCache fsAttrCache; + private FileStoreAttributes fsAttrCache; @Before public void setUp() throws Exception { @@ -87,7 +87,7 @@ public class FileSnapshotTest { // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement fsAttrCache = FS - .getFileStoreAttributeCache(trash.getParent()); + .getFileStoreAttributes(trash.getParent()); } @Before diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java index 77f5febc17..1adddb5ea5 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java @@ -83,7 +83,7 @@ public class FileBasedConfigTest { @Before public void setUp() throws Exception { trash = Files.createTempDirectory("tmp_"); - FS.getFileStoreAttributeCache(trash.getParent()); + FS.getFileStoreAttributes(trash.getParent()); } @After diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java index 63e295ec83..2054e1efa8 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java @@ -203,7 +203,7 @@ public class FSTest { .ofPattern("uuuu-MMM-dd HH:mm:ss.nnnnnnnnn", Locale.ENGLISH) .withZone(ZoneId.systemDefault()); Path dir = Files.createTempDirectory("probe-filesystem"); - Duration resolution = FS.getFileStoreAttributeCache(dir) + Duration resolution = FS.getFileStoreAttributes(dir) .getFsTimestampResolution(); long resolutionNs = resolution.toNanos(); assertTrue(resolutionNs > 0); diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index a027caaf02..8277735e24 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -49,6 +49,12 @@ + + + + + + @@ -72,6 +78,14 @@ + + + + + + + + @@ -174,7 +188,7 @@ - + @@ -192,7 +206,7 @@ - + @@ -210,11 +224,11 @@ - + - + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index aa9f1cc45b..e81a88451b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.internal.storage.file; -import static org.eclipse.jgit.util.FS.FileStoreAttributeCache.FALLBACK_FILESTORE_ATTRIBUTES; +import static org.eclipse.jgit.util.FS.FileStoreAttributes.FALLBACK_FILESTORE_ATTRIBUTES; import java.io.File; import java.io.IOException; @@ -58,7 +58,7 @@ import java.util.concurrent.TimeUnit; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.util.FS; -import org.eclipse.jgit.util.FS.FileStoreAttributeCache; +import org.eclipse.jgit.util.FS.FileStoreAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -215,7 +215,7 @@ public class FileSnapshot { private final long size; /** measured FileStore attributes */ - private FileStoreAttributeCache fileStoreAttributeCache; + private FileStoreAttributes fileStoreAttributeCache; /** * Object that uniquely identifies the given file, or {@code @@ -254,7 +254,7 @@ public class FileSnapshot { this.file = file; this.lastRead = Instant.now(); this.fileStoreAttributeCache = useConfig - ? FS.getFileStoreAttributeCache(file.toPath().getParent()) + ? FS.getFileStoreAttributes(file.toPath().getParent()) : FALLBACK_FILESTORE_ATTRIBUTES; BasicFileAttributes fileAttributes = null; try { @@ -293,7 +293,7 @@ public class FileSnapshot { this.file = null; this.lastRead = read; this.lastModified = modified; - this.fileStoreAttributeCache = new FileStoreAttributeCache( + this.fileStoreAttributeCache = new FileStoreAttributes( fsTimestampResolution); this.size = size; this.fileKey = fileKey; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 4f636d4553..82ccd7b034 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -444,4 +444,11 @@ public final class ConfigConstants { * @since 5.1.9 */ public static final String CONFIG_KEY_TIMESTAMP_RESOLUTION = "timestampResolution"; + + /** + * The "minRacyThreshold" key + * + * @since 5.1.9 + */ + public static final String CONFIG_KEY_MIN_RACY_THRESHOLD = "minRacyThreshold"; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java index 3a41643e6e..633632dc01 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java @@ -149,13 +149,37 @@ public class FileBasedConfig extends StoredConfig { */ @Override public void load() throws IOException, ConfigInvalidException { + load(true); + } + + /** + * Load the configuration as a Git text style configuration file. + *

+ * If the file does not exist, this configuration is cleared, and thus + * behaves the same as though the file exists, but is empty. + * + * @param useFileSnapshotWithConfig + * if {@code true} use the FileSnapshot with config, otherwise + * use it without config + * @throws IOException + * if IO failed + * @throws ConfigInvalidException + * if config is invalid + * @since 5.1.9 + */ + public void load(boolean useFileSnapshotWithConfig) + throws IOException, ConfigInvalidException { final int maxStaleRetries = 5; int retries = 0; while (true) { final FileSnapshot oldSnapshot = snapshot; - // don't use config in this snapshot to avoid endless recursion - final FileSnapshot newSnapshot = FileSnapshot - .saveNoConfig(getFile()); + final FileSnapshot newSnapshot; + if (useFileSnapshotWithConfig) { + newSnapshot = FileSnapshot.save(getFile()); + } else { + // don't use config in this snapshot to avoid endless recursion + newSnapshot = FileSnapshot.saveNoConfig(getFile()); + } try { final byte[] in = IO.readFully(getFile()); final ObjectId newHash = hash(in); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 08dab3201d..16810e0dc3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -208,9 +208,9 @@ public abstract class FS { * * @since 5.1.9 */ - public final static class FileStoreAttributeCache { + public final static class FileStoreAttributes { - private static final Duration UNDEFINED_RESOLUTION = Duration + private static final Duration UNDEFINED_DURATION = Duration .ofNanos(Long.MAX_VALUE); /** @@ -218,10 +218,10 @@ public abstract class FS { * filesystem timestamp resolution. The last modified time granularity * of FAT filesystems is 2 seconds. */ - public static final FileStoreAttributeCache FALLBACK_FILESTORE_ATTRIBUTES = new FileStoreAttributeCache( + public static final FileStoreAttributes FALLBACK_FILESTORE_ATTRIBUTES = new FileStoreAttributes( Duration.ofMillis(2000)); - private static final Map attributeCache = new ConcurrentHashMap<>(); + private static final Map attributeCache = new ConcurrentHashMap<>(); private static AtomicBoolean background = new AtomicBoolean(); @@ -239,22 +239,24 @@ public abstract class FS { .ofMillis(10); /** + * Get the FileStoreAttributes for the given FileStore + * * @param path * file residing in the FileStore to get attributes for - * @return FileStoreAttributeCache entry for the given path. + * @return FileStoreAttributes for the given path. */ - public static FileStoreAttributeCache get(Path path) { + public static FileStoreAttributes get(Path path) { path = path.toAbsolutePath(); Path dir = Files.isDirectory(path) ? path : path.getParent(); - return getFileAttributeCache(dir); + return getFileStoreAttributes(dir); } - private static FileStoreAttributeCache getFileAttributeCache(Path dir) { + private static FileStoreAttributes getFileStoreAttributes(Path dir) { FileStore s; try { if (Files.exists(dir)) { s = Files.getFileStore(dir); - FileStoreAttributeCache c = attributeCache.get(s); + FileStoreAttributes c = attributeCache.get(s); if (c != null) { return c; } @@ -272,7 +274,8 @@ public abstract class FS { Thread.currentThread(), dir); return FALLBACK_FILESTORE_ATTRIBUTES; } - CompletableFuture> f = CompletableFuture + + CompletableFuture> f = CompletableFuture .supplyAsync(() -> { Lock lock = locks.computeIfAbsent(s, l -> new ReentrantLock()); @@ -282,21 +285,27 @@ public abstract class FS { Thread.currentThread(), dir); return Optional.empty(); } - Optional cache = Optional + Optional attributes = Optional .empty(); try { // Some earlier future might have set the value // and removed itself since we checked for the // value above. Hence check cache again. - FileStoreAttributeCache c = attributeCache + FileStoreAttributes c = attributeCache .get(s); if (c != null) { return Optional.of(c); } + attributes = readFromConfig(s); + if (attributes.isPresent()) { + attributeCache.put(s, attributes.get()); + return attributes; + } + Optional resolution = measureFsTimestampResolution( s, dir); if (resolution.isPresent()) { - c = new FileStoreAttributeCache( + c = new FileStoreAttributes( resolution.get()); attributeCache.put(s, c); // for high timestamp resolution measure @@ -304,24 +313,28 @@ public abstract class FS { if (c.fsTimestampResolution .toNanos() < 100_000_000L) { c.minimalRacyInterval = measureMinimalRacyInterval( - dir); + dir); } if (LOG.isDebugEnabled()) { LOG.debug(c.toString()); } - cache = Optional.of(c); + saveToConfig(s, c); } + attributes = Optional.of(c); } finally { lock.unlock(); locks.remove(s); } - return cache; + return attributes; }); + f.exceptionally(e -> { + LOG.error(e.getLocalizedMessage(), e); + return Optional.empty(); + }); // even if measuring in background wait a little - if the result // arrives, it's better than returning the large fallback - Optional d = f.get( - background.get() ? 100 : 5000, - TimeUnit.MILLISECONDS); + Optional d = background.get() ? f.get( + 100, TimeUnit.MILLISECONDS) : f.get(); if (d.isPresent()) { return d.get(); } @@ -341,14 +354,16 @@ public abstract class FS { private static Duration measureMinimalRacyInterval(Path dir) { LOG.debug("{}: start measure minimal racy interval in {}", //$NON-NLS-1$ Thread.currentThread(), dir); + int n = 0; int failures = 0; long racyNanos = 0; - final int COUNT = 1000; ArrayList deltas = new ArrayList<>(); Path probe = dir.resolve(".probe-" + UUID.randomUUID()); //$NON-NLS-1$ + Instant end = Instant.now().plusSeconds(3); try { Files.createFile(probe); - for (int i = 0; i < COUNT; i++) { + do { + n++; write(probe, "a"); //$NON-NLS-1$ FileSnapshot snapshot = FileSnapshot.save(probe.toFile()); read(probe); @@ -358,7 +373,7 @@ public abstract class FS { racyNanos = snapshot.lastRacyThreshold(); failures++; } - } + } while (Instant.now().compareTo(end) < 0); } catch (IOException e) { LOG.error(e.getMessage(), e); return FALLBACK_MIN_RACY_INTERVAL; @@ -376,7 +391,7 @@ public abstract class FS { + " delta max [ns], delta avg [ns]," //$NON-NLS-1$ + " delta stddev [ns]\n" //$NON-NLS-1$ + "{}, {}, {}, {}, {}, {}, {}", //$NON-NLS-1$ - COUNT, failures, racyNanos, stats.min(), stats.max(), + n, failures, racyNanos, stats.min(), stats.max(), stats.avg(), stats.stddev()); return Duration .ofNanos(Double.valueOf(stats.max()).longValue()); @@ -405,10 +420,6 @@ public abstract class FS { FileStore s, Path dir) { LOG.debug("{}: start measure timestamp resolution {} in {}", //$NON-NLS-1$ Thread.currentThread(), s, dir); - Duration configured = readFileTimeResolution(s); - if (!UNDEFINED_RESOLUTION.equals(configured)) { - return Optional.of(configured); - } Path probe = dir.resolve(".probe-" + UUID.randomUUID()); //$NON-NLS-1$ try { Files.createFile(probe); @@ -423,7 +434,6 @@ public abstract class FS { Duration fsResolution = Duration.between(t1.toInstant(), t2.toInstant()); Duration clockResolution = measureClockResolution(); fsResolution = fsResolution.plus(clockResolution); - saveFileTimeResolution(s, fsResolution); LOG.debug("{}: end measure timestamp resolution {} in {}", //$NON-NLS-1$ Thread.currentThread(), s, dir); return Optional.of(fsResolution); @@ -454,20 +464,20 @@ public abstract class FS { } private static void deleteProbe(Path probe) { - if (Files.exists(probe)) { - try { - Files.delete(probe); - } catch (IOException e) { - LOG.error(e.getLocalizedMessage(), e); - } + try { + FileUtils.delete(probe.toFile(), + FileUtils.SKIP_MISSING | FileUtils.RETRY); + } catch (IOException e) { + LOG.error(e.getMessage(), e); } } - private static Duration readFileTimeResolution(FileStore s) { + private static Optional readFromConfig( + FileStore s) { FileBasedConfig userConfig = SystemReader.getInstance() .openUserConfig(null, FS.DETECTED); try { - userConfig.load(); + userConfig.load(false); } catch (IOException e) { LOG.error(MessageFormat.format(JGitText.get().readConfigFailed, userConfig.getFile().getAbsolutePath()), e); @@ -477,49 +487,65 @@ public abstract class FS { userConfig.getFile().getAbsolutePath(), e.getMessage())); } - Duration configured = Duration - .ofNanos(userConfig.getTimeUnit( - ConfigConstants.CONFIG_FILESYSTEM_SECTION, - javaVersionPrefix + s.name(), - ConfigConstants.CONFIG_KEY_TIMESTAMP_RESOLUTION, - UNDEFINED_RESOLUTION.toNanos(), - TimeUnit.NANOSECONDS)); - return configured; + String key = getConfigKey(s); + Duration resolution = Duration.ofNanos(userConfig.getTimeUnit( + ConfigConstants.CONFIG_FILESYSTEM_SECTION, key, + ConfigConstants.CONFIG_KEY_TIMESTAMP_RESOLUTION, + UNDEFINED_DURATION.toNanos(), TimeUnit.NANOSECONDS)); + if (UNDEFINED_DURATION.equals(resolution)) { + return Optional.empty(); + } + Duration minRacyThreshold = Duration.ofNanos(userConfig.getTimeUnit( + ConfigConstants.CONFIG_FILESYSTEM_SECTION, key, + ConfigConstants.CONFIG_KEY_MIN_RACY_THRESHOLD, + UNDEFINED_DURATION.toNanos(), TimeUnit.NANOSECONDS)); + FileStoreAttributes c = new FileStoreAttributes(resolution); + if (!UNDEFINED_DURATION.equals(minRacyThreshold)) { + c.minimalRacyInterval = minRacyThreshold; + } + return Optional.of(c); } - private static void saveFileTimeResolution(FileStore s, - Duration resolution) { + private static void saveToConfig(FileStore s, + FileStoreAttributes c) { FileBasedConfig userConfig = SystemReader.getInstance() .openUserConfig(null, FS.DETECTED); - long nanos = resolution.toNanos(); - TimeUnit unit; - if (nanos < 200_000L) { - unit = TimeUnit.NANOSECONDS; - } else if (nanos < 200_000_000L) { - unit = TimeUnit.MICROSECONDS; - } else { - unit = TimeUnit.MILLISECONDS; - } + long resolution = c.getFsTimestampResolution().toNanos(); + TimeUnit resolutionUnit = getUnit(resolution); + long resolutionValue = resolutionUnit.convert(resolution, + TimeUnit.NANOSECONDS); + + long minRacyThreshold = c.getMinimalRacyInterval().toNanos(); + TimeUnit minRacyThresholdUnit = getUnit(minRacyThreshold); + long minRacyThresholdValue = minRacyThresholdUnit + .convert(minRacyThreshold, TimeUnit.NANOSECONDS); final int max_retries = 5; int retries = 0; boolean succeeded = false; - long value = unit.convert(nanos, TimeUnit.NANOSECONDS); + String key = getConfigKey(s); while (!succeeded && retries < max_retries) { try { - userConfig.load(); + userConfig.load(false); userConfig.setString( - ConfigConstants.CONFIG_FILESYSTEM_SECTION, - javaVersionPrefix + s.name(), + ConfigConstants.CONFIG_FILESYSTEM_SECTION, key, ConfigConstants.CONFIG_KEY_TIMESTAMP_RESOLUTION, String.format("%d %s", //$NON-NLS-1$ - Long.valueOf(value), - unit.name().toLowerCase())); + Long.valueOf(resolutionValue), + resolutionUnit.name().toLowerCase())); + userConfig.setString( + ConfigConstants.CONFIG_FILESYSTEM_SECTION, key, + ConfigConstants.CONFIG_KEY_MIN_RACY_THRESHOLD, + String.format("%d %s", //$NON-NLS-1$ + Long.valueOf(minRacyThresholdValue), + minRacyThresholdUnit.name().toLowerCase())); userConfig.save(); succeeded = true; } catch (LockFailedException e) { // race with another thread, wait a bit and try again try { + LOG.warn(MessageFormat.format(JGitText.get().cannotLock, + userConfig.getFile().getAbsolutePath())); retries++; Thread.sleep(20); } catch (InterruptedException e1) { @@ -538,6 +564,38 @@ public abstract class FS { } } + private static String getConfigKey(FileStore s) { + final String storeKey; + if (SystemReader.getInstance().isWindows()) { + Object attribute = null; + try { + attribute = s.getAttribute("volume:vsn"); //$NON-NLS-1$ + } catch (IOException ignored) { + // ignore + } + if (attribute instanceof Integer) { + storeKey = attribute.toString(); + } else { + storeKey = s.name(); + } + } else { + storeKey = s.name(); + } + return javaVersionPrefix + storeKey; + } + + private static TimeUnit getUnit(long nanos) { + TimeUnit unit; + if (nanos < 200_000L) { + unit = TimeUnit.NANOSECONDS; + } else if (nanos < 200_000_000L) { + unit = TimeUnit.MICROSECONDS; + } else { + unit = TimeUnit.MILLISECONDS; + } + return unit; + } + private final @NonNull Duration fsTimestampResolution; private Duration minimalRacyInterval; @@ -565,7 +623,7 @@ public abstract class FS { * * @param fsTimestampResolution */ - public FileStoreAttributeCache( + public FileStoreAttributes( @NonNull Duration fsTimestampResolution) { this.fsTimestampResolution = fsTimestampResolution; this.minimalRacyInterval = Duration.ZERO; @@ -575,7 +633,7 @@ public abstract class FS { @Override public String toString() { return String.format( - "FileStoreAttributeCache[fsTimestampResolution=%,d µs, " + "FileStoreAttributes[fsTimestampResolution=%,d µs, " + "minimalRacyInterval=%,d µs]", fsTimestampResolution.toNanos() / 1000, minimalRacyInterval.toNanos() / 1000); @@ -598,17 +656,16 @@ public abstract class FS { } /** - * Whether FileStore attribute cache entries should be determined - * asynchronously + * Whether FileStore attributes should be determined asynchronously * * @param asynch - * whether FileStore attribute cache entries should be determined + * whether FileStore attributes should be determined * asynchronously. If false access to cached attributes may block * for some seconds for the first call per FileStore * @since 5.1.9 */ - public static void setAsyncfileStoreAttrCache(boolean asynch) { - FileStoreAttributeCache.setBackground(asynch); + public static void setAsyncFileStoreAttributes(boolean asynch) { + FileStoreAttributes.setBackground(asynch); } /** @@ -639,9 +696,8 @@ public abstract class FS { } /** - * Get an estimate for the filesystem timestamp resolution from a cache of - * timestamp resolution per FileStore, if not yet available it is measured - * for a probe file under the given directory. + * Get cached FileStore attributes, if not yet available measure them using + * a probe file under the given directory. * * @param dir * the directory under which the probe file will be created to @@ -649,9 +705,9 @@ public abstract class FS { * @return measured filesystem timestamp resolution * @since 5.1.9 */ - public static FileStoreAttributeCache getFileStoreAttributeCache( + public static FileStoreAttributes getFileStoreAttributes( @NonNull Path dir) { - return FileStoreAttributeCache.get(dir); + return FileStoreAttributes.get(dir); } private volatile Holder userHome; -- cgit v1.2.3