From 5911521ba6d47868871c4b5f240c71af827b7aa2 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 15 Jul 2019 15:00:09 +0200 Subject: [PATCH] Measure minimum racy interval to auto-configure FileSnapshot By running FileSnapshotTest#detectFileModified we found that the sum of measured filesystem timestamp resolution and measured clock resolution may yield a too small interval after a file has been modified which we need to consider racily clean. In our tests we didn't find this behavior on all systems we tested on, e.g. on MacOS using APFS and Java 8 and 11 this effect was not observed. On Linux (SLES 15, kernel 4.12.14-150.22-default) we collected the following test results using Java 8 and 11: In 23-98% of 10000 test runs (depending on filesystem type and Java version) the test failed, which means the effective interval which needs to be considered racily clean after a file was modified is larger than the measured file timestamp resolution. "delta" is the observed interval after a file has been modified but FileSnapshot did not yet detect the modification: "resolution" is the measured sum of file timestamp resolution and clock resolution seen in Java. Java version filesystem failures resolution min delta max delta 1.8.0_212-b04 btrfs 98.6% 1 ms 3.6 ms 6.6 ms 1.8.0_212-b04 ext4 82.6% 3 ms 1.1 ms 4.1 ms 1.8.0_212-b04 xfs 23.8% 4 ms 3.7 ms 3.9 ms 1.8.0_212-b04 zfs 23.1% 3 ms 4.8 ms 5.0 ms 11.0.3+7 btrfs 98.1% 3 us 0.7 ms 4.7 ms 11.0.3+7 ext4 98.1% 6 us 0.7 ms 4.7 ms 11.0.3+7 xfs 98.5% 7 us 0.1 ms 8.0 ms 11.0.3+7 zfs 98.4% 7 us 0.7 ms 5.2 ms Mac OS 1.8.0_212 APFS 0% 1 s 11.0.3+7 APFS 0% 6 us The observed delta is not distributed according to a normal gaussian distribution but rather random in the observed range between "min delta" and "max delta". Run this test after measuring file timestamp resolution in FS.FileAttributeCache to auto-configure JGit since it's unclear what mechanism is causing this effect. In FileSnapshot#isRacyClean use the maximum of the measured timestamp resolution and the measured "delta" as explained above to decide if a given FileSnapshot is to be considered racily clean. Add a 30% safety margin to ensure we are on the safe side. Change-Id: I1c8bb59f6486f174b7bbdc63072777ddbe06694d Signed-off-by: Matthias Sohn --- .../jgit/ant/tasks/GitCloneTaskTest.java | 2 +- .../junit/LocalDiskRepositoryTestCase.java | 2 +- .../jgit/junit/RepositoryTestCase.java | 3 +- .../jgit/lfs/server/fs/LfsServerTest.java | 2 +- .../tst-rsrc/log4j.properties | 5 +- .../storage/file/FileSnapshotTest.java | 28 ++- .../storage/file/FileBasedConfigTest.java | 2 +- .../tst/org/eclipse/jgit/util/FSTest.java | 3 +- org.eclipse.jgit/.settings/.api_filters | 24 +-- .../internal/storage/file/FileSnapshot.java | 48 +++-- .../src/org/eclipse/jgit/lib/Constants.java | 11 - .../src/org/eclipse/jgit/util/FS.java | 201 +++++++++++++++--- 12 files changed, 245 insertions(+), 86 deletions(-) diff --git a/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java b/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java index 1d7187a312..9f9d459a6c 100644 --- a/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java +++ b/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java @@ -66,7 +66,7 @@ public class GitCloneTaskTest extends LocalDiskRepositoryTestCase { @Before public void before() throws IOException { dest = createTempFile(); - FS.getFsTimerResolution(dest.toPath().getParent()); + FS.getFileStoreAttributeCache(dest.toPath().getParent()); project = new Project(); project.init(); enableLogging(); diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java index 62dfc5d9c0..fb8295fa4b 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java @@ -130,7 +130,7 @@ public abstract class LocalDiskRepositoryTestCase { // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - FS.getFsTimerResolution(tmp.toPath().getParent()); + FS.getFileStoreAttributeCache(tmp.toPath().getParent()); mockSystemReader = new MockSystemReader(); mockSystemReader.userGitConfig = new FileBasedConfig(new File(tmp, diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java index 49f5c5febb..ebd13e4112 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java @@ -378,7 +378,8 @@ public abstract class RepositoryTestCase extends LocalDiskRepositoryTestCase { tmp = File.createTempFile("fsTickTmpFile", null, lastFile.getParentFile()); } - long res = FS.getFsTimerResolution(tmp.toPath()).toNanos(); + long res = FS.getFileStoreAttributeCache(tmp.toPath()) + .getFsTimestampResolution().toNanos(); long sleepTime = res / 10; try { Instant startTime = fs.lastModifiedInstant(lastFile); diff --git a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java index 63af6eb52b..92a6ec351f 100644 --- a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java +++ b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java @@ -123,7 +123,7 @@ public abstract class LfsServerTest { // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - FS.getFsTimerResolution(tmp.getParent()); + FS.getFileStoreAttributeCache(tmp.getParent()); server = new AppServer(); ServletContextHandler app = server.addContext("/lfs"); diff --git a/org.eclipse.jgit.test/tst-rsrc/log4j.properties b/org.eclipse.jgit.test/tst-rsrc/log4j.properties index a48a4022ff..ee1ac35158 100644 --- a/org.eclipse.jgit.test/tst-rsrc/log4j.properties +++ b/org.eclipse.jgit.test/tst-rsrc/log4j.properties @@ -8,4 +8,7 @@ log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n #log4j.appender.fileLogger.bufferedIO = true -#log4j.appender.fileLogger.bufferSize = 1024 \ No newline at end of file +#log4j.appender.fileLogger.bufferSize = 4096 + +#log4j.logger.org.eclipse.jgit.util.FS = DEBUG +#log4j.logger.org.eclipse.jgit.internal.storage.file.FileSnapshot = DEBUG diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index 9eb55db09c..012407f715 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -62,6 +62,7 @@ import java.util.ArrayList; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.FS.FileStoreAttributeCache; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.Stats; import org.eclipse.jgit.util.SystemReader; @@ -78,14 +79,15 @@ public class FileSnapshotTest { private Path trash; - private Duration fsTimerResolution; + private FileStoreAttributeCache fsAttrCache; @Before public void setUp() throws Exception { trash = Files.createTempDirectory("tmp_"); // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - fsTimerResolution = FS.getFsTimerResolution(trash.getParent()); + fsAttrCache = FS + .getFileStoreAttributeCache(trash.getParent()); } @Before @@ -131,11 +133,13 @@ public class FileSnapshotTest { // if filesystem timestamp resolution is high the snapshot won't be // racily clean Assume.assumeTrue( - fsTimerResolution.compareTo(Duration.ofMillis(10)) > 0); + fsAttrCache.getFsTimestampResolution() + .compareTo(Duration.ofMillis(10)) > 0); Path f1 = createFile("newfile"); waitNextTick(f1); FileSnapshot save = FileSnapshot.save(f1.toFile()); - TimeUnit.NANOSECONDS.sleep(fsTimerResolution.dividedBy(2).toNanos()); + TimeUnit.NANOSECONDS.sleep( + fsAttrCache.getFsTimestampResolution().dividedBy(2).toNanos()); assertTrue(save.isModified(f1.toFile())); } @@ -149,7 +153,8 @@ public class FileSnapshotTest { // if filesystem timestamp resolution is high the snapshot won't be // racily clean Assume.assumeTrue( - fsTimerResolution.compareTo(Duration.ofMillis(10)) > 0); + fsAttrCache.getFsTimestampResolution() + .compareTo(Duration.ofMillis(10)) > 0); Path f1 = createFile("newfile"); FileSnapshot save = FileSnapshot.save(f1.toFile()); assertTrue(save.isModified(f1.toFile())); @@ -230,7 +235,7 @@ public class FileSnapshotTest { write(f, "b"); if (!snapshot.isModified(f)) { deltas.add(snapshot.lastDelta()); - racyNanos = snapshot.lastRacyNanos(); + racyNanos = snapshot.lastRacyThreshold(); failures++; } assertEquals("file should contain 'b'", "b", read(f)); @@ -244,7 +249,7 @@ public class FileSnapshotTest { LOG.debug(String.format("%,d", d)); } LOG.error( - "count, failures, racy limit [ns], delta min [ns]," + "count, failures, eff. racy threshold [ns], delta min [ns]," + " delta max [ns], delta avg [ns]," + " delta stddev [ns]"); LOG.error(String.format( @@ -253,7 +258,14 @@ public class FileSnapshotTest { stats.avg(), stats.stddev())); } assertTrue( - "FileSnapshot: number of failures to detect file modifications should be 0", + String.format( + "FileSnapshot: failures to detect file modifications" + + " %d out of %d\n" + + "timestamp resolution %d µs" + + " min racy threshold %d µs" + , failures, COUNT, + fsAttrCache.getFsTimestampResolution().toNanos() / 1000, + fsAttrCache.getMinimalRacyInterval().toNanos() / 1000), failures == 0); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java index d3686285e3..77f5febc17 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java @@ -83,7 +83,7 @@ public class FileBasedConfigTest { @Before public void setUp() throws Exception { trash = Files.createTempDirectory("tmp_"); - FS.getFsTimerResolution(trash.getParent()); + FS.getFileStoreAttributeCache(trash.getParent()); } @After diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java index bde8a8a6b3..63e295ec83 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java @@ -203,7 +203,8 @@ public class FSTest { .ofPattern("uuuu-MMM-dd HH:mm:ss.nnnnnnnnn", Locale.ENGLISH) .withZone(ZoneId.systemDefault()); Path dir = Files.createTempDirectory("probe-filesystem"); - Duration resolution = FS.getFsTimerResolution(dir); + Duration resolution = FS.getFileStoreAttributeCache(dir) + .getFsTimestampResolution(); long resolutionNs = resolution.toNanos(); assertTrue(resolutionNs > 0); for (int i = 0; i < 10; i++) { diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 59bafc52e0..a027caaf02 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -56,14 +56,6 @@ - - - - - - - - @@ -179,6 +171,12 @@ + + + + + + @@ -203,18 +201,20 @@ + + - - + + - + - + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 2a490a4a1f..aa9f1cc45b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.internal.storage.file; -import static org.eclipse.jgit.lib.Constants.FALLBACK_TIMESTAMP_RESOLUTION; +import static org.eclipse.jgit.util.FS.FileStoreAttributeCache.FALLBACK_FILESTORE_ATTRIBUTES; import java.io.File; import java.io.IOException; @@ -58,6 +58,7 @@ import java.util.concurrent.TimeUnit; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.FS.FileStoreAttributeCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -213,8 +214,8 @@ public class FileSnapshot { * When set to {@link #UNKNOWN_SIZE} the size is not considered for modification checks. */ private final long size; - /** measured filesystem timestamp resolution */ - private Duration fsTimestampResolution; + /** measured FileStore attributes */ + private FileStoreAttributeCache fileStoreAttributeCache; /** * Object that uniquely identifies the given file, or {@code @@ -252,9 +253,9 @@ public class FileSnapshot { protected FileSnapshot(File file, boolean useConfig) { this.file = file; this.lastRead = Instant.now(); - this.fsTimestampResolution = useConfig - ? FS.getFsTimerResolution(file.toPath().getParent()) - : FALLBACK_TIMESTAMP_RESOLUTION; + this.fileStoreAttributeCache = useConfig + ? FS.getFileStoreAttributeCache(file.toPath().getParent()) + : FALLBACK_FILESTORE_ATTRIBUTES; BasicFileAttributes fileAttributes = null; try { fileAttributes = FS.DETECTED.fileAttributes(file); @@ -285,14 +286,15 @@ public class FileSnapshot { private long delta; - private long racyNanos; + private long racyThreshold; private FileSnapshot(Instant read, Instant modified, long size, @NonNull Duration fsTimestampResolution, @NonNull Object fileKey) { this.file = null; this.lastRead = read; this.lastModified = modified; - this.fsTimestampResolution = fsTimestampResolution; + this.fileStoreAttributeCache = new FileStoreAttributeCache( + fsTimestampResolution); this.size = size; this.fileKey = fileKey; } @@ -397,9 +399,10 @@ public class FileSnapshot { * if sleep was interrupted */ public void waitUntilNotRacy() throws InterruptedException { + long timestampResolution = fileStoreAttributeCache + .getFsTimestampResolution().toNanos(); while (isRacyClean(Instant.now())) { - TimeUnit.NANOSECONDS - .sleep((fsTimestampResolution.toNanos() + 1) * 11 / 10); + TimeUnit.NANOSECONDS.sleep(timestampResolution); } } @@ -474,15 +477,16 @@ public class FileSnapshot { * @return the delta in nanoseconds between lastModified and lastRead during * last racy check */ - long lastDelta() { + public long lastDelta() { return delta; } /** - * @return the racyNanos threshold in nanoseconds during last racy check + * @return the racyLimitNanos threshold in nanoseconds during last racy + * check */ - long lastRacyNanos() { - return racyNanos; + public long lastRacyThreshold() { + return racyThreshold; } /** {@inheritDoc} */ @@ -501,20 +505,28 @@ public class FileSnapshot { } private boolean isRacyClean(Instant read) { - // add a 10% safety margin - racyNanos = (fsTimestampResolution.toNanos() + 1) * 11 / 10; + racyThreshold = getEffectiveRacyThreshold(); delta = Duration.between(lastModified, read).toNanos(); - wasRacyClean = delta <= racyNanos; + wasRacyClean = delta <= racyThreshold; if (LOG.isDebugEnabled()) { LOG.debug( "file={}, isRacyClean={}, read={}, lastModified={}, delta={} ns, racy<={} ns", //$NON-NLS-1$ file, Boolean.valueOf(wasRacyClean), dateFmt.format(read), dateFmt.format(lastModified), Long.valueOf(delta), - Long.valueOf(racyNanos)); + Long.valueOf(racyThreshold)); } return wasRacyClean; } + private long getEffectiveRacyThreshold() { + long timestampResolution = fileStoreAttributeCache + .getFsTimestampResolution().toNanos(); + long minRacyInterval = fileStoreAttributeCache.getMinimalRacyInterval() + .toNanos(); + // add a 30% safety margin + return Math.max(timestampResolution, minRacyInterval) * 13 / 10; + } + private boolean isModified(Instant currLastModified) { // Any difference indicates the path was modified. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 94fc100386..4c55196961 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -52,7 +52,6 @@ import java.nio.charset.Charset; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.MessageFormat; -import java.time.Duration; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.internal.JGitText; @@ -723,16 +722,6 @@ public final class Constants { */ public static final String LOCK_SUFFIX = ".lock"; //$NON-NLS-1$ - /** - * Fallback filesystem timestamp resolution used when we can't measure the - * resolution. The last modified time granularity of FAT filesystems is 2 - * seconds. - * - * @since 5.1.9 - */ - public static final Duration FALLBACK_TIMESTAMP_RESOLUTION = Duration - .ofMillis(2000); - private Constants() { // Hide the default constructor } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 081776f081..08dab3201d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -45,7 +45,6 @@ package org.eclipse.jgit.util; import static java.nio.charset.StandardCharsets.UTF_8; import static java.time.Instant.EPOCH; -import static org.eclipse.jgit.lib.Constants.FALLBACK_TIMESTAMP_RESOLUTION; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -55,7 +54,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.io.PrintStream; +import java.io.Writer; import java.nio.charset.Charset; import java.nio.file.AccessDeniedException; import java.nio.file.FileStore; @@ -68,6 +69,7 @@ import java.security.PrivilegedAction; import java.text.MessageFormat; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -94,6 +96,7 @@ import org.eclipse.jgit.errors.CommandFailedException; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.FileSnapshot; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Repository; @@ -200,11 +203,24 @@ public abstract class FS { } } - private static final class FileStoreAttributeCache { + /** + * Attributes of FileStores on this system + * + * @since 5.1.9 + */ + public final static class FileStoreAttributeCache { private static final Duration UNDEFINED_RESOLUTION = Duration .ofNanos(Long.MAX_VALUE); + /** + * Fallback FileStore attributes used when we can't measure the + * filesystem timestamp resolution. The last modified time granularity + * of FAT filesystems is 2 seconds. + */ + public static final FileStoreAttributeCache FALLBACK_FILESTORE_ATTRIBUTES = new FileStoreAttributeCache( + Duration.ofMillis(2000)); + private static final Map attributeCache = new ConcurrentHashMap<>(); private static AtomicBoolean background = new AtomicBoolean(); @@ -216,36 +232,58 @@ public abstract class FS { } private static final String javaVersionPrefix = System - .getProperty("java.vm.vendor") + '|' //$NON-NLS-1$ - + System.getProperty("java.vm.version") + '|'; //$NON-NLS-1$ + .getProperty("java.vendor") + '|' //$NON-NLS-1$ + + System.getProperty("java.version") + '|'; //$NON-NLS-1$ + + private static final Duration FALLBACK_MIN_RACY_INTERVAL = Duration + .ofMillis(10); - private static Duration getFsTimestampResolution(Path file) { - file = file.toAbsolutePath(); - Path dir = Files.isDirectory(file) ? file : file.getParent(); + /** + * @param path + * file residing in the FileStore to get attributes for + * @return FileStoreAttributeCache entry for the given path. + */ + public static FileStoreAttributeCache get(Path path) { + path = path.toAbsolutePath(); + Path dir = Files.isDirectory(path) ? path : path.getParent(); + return getFileAttributeCache(dir); + } + + private static FileStoreAttributeCache getFileAttributeCache(Path dir) { FileStore s; try { if (Files.exists(dir)) { s = Files.getFileStore(dir); FileStoreAttributeCache c = attributeCache.get(s); if (c != null) { - return c.getFsTimestampResolution(); + return c; } if (!Files.isWritable(dir)) { // cannot measure resolution in a read-only directory - return FALLBACK_TIMESTAMP_RESOLUTION; + LOG.debug( + "{}: cannot measure timestamp resolution in read-only directory {}", //$NON-NLS-1$ + Thread.currentThread(), dir); + return FALLBACK_FILESTORE_ATTRIBUTES; } } else { // cannot determine FileStore of an unborn directory - return FALLBACK_TIMESTAMP_RESOLUTION; + LOG.debug( + "{}: cannot measure timestamp resolution of unborn directory {}", //$NON-NLS-1$ + Thread.currentThread(), dir); + return FALLBACK_FILESTORE_ATTRIBUTES; } - CompletableFuture> f = CompletableFuture + CompletableFuture> f = CompletableFuture .supplyAsync(() -> { Lock lock = locks.computeIfAbsent(s, l -> new ReentrantLock()); if (!lock.tryLock()) { + LOG.debug( + "{}: couldn't get lock to measure timestamp resolution in {}", //$NON-NLS-1$ + Thread.currentThread(), dir); return Optional.empty(); } - Optional resolution; + Optional cache = Optional + .empty(); try { // Some earlier future might have set the value // and removed itself since we checked for the @@ -253,28 +291,36 @@ public abstract class FS { FileStoreAttributeCache c = attributeCache .get(s); if (c != null) { - return Optional - .of(c.getFsTimestampResolution()); + return Optional.of(c); } - resolution = measureFsTimestampResolution(s, - dir); + Optional resolution = measureFsTimestampResolution( + s, dir); if (resolution.isPresent()) { - FileStoreAttributeCache cache = new FileStoreAttributeCache( + c = new FileStoreAttributeCache( resolution.get()); - attributeCache.put(s, cache); + attributeCache.put(s, c); + // for high timestamp resolution measure + // minimal racy interval + if (c.fsTimestampResolution + .toNanos() < 100_000_000L) { + c.minimalRacyInterval = measureMinimalRacyInterval( + dir); + } if (LOG.isDebugEnabled()) { - LOG.debug(cache.toString()); + LOG.debug(c.toString()); } + cache = Optional.of(c); } } finally { lock.unlock(); locks.remove(s); } - return resolution; + return cache; }); // even if measuring in background wait a little - if the result // arrives, it's better than returning the large fallback - Optional d = f.get(background.get() ? 50 : 2000, + Optional d = f.get( + background.get() ? 100 : 5000, TimeUnit.MILLISECONDS); if (d.isPresent()) { return d.get(); @@ -286,11 +332,79 @@ public abstract class FS { } catch (TimeoutException | SecurityException e) { // use fallback } - return FALLBACK_TIMESTAMP_RESOLUTION; + LOG.debug("{}: use fallback timestamp resolution for directory {}", //$NON-NLS-1$ + Thread.currentThread(), dir); + return FALLBACK_FILESTORE_ATTRIBUTES; + } + + @SuppressWarnings("boxing") + private static Duration measureMinimalRacyInterval(Path dir) { + LOG.debug("{}: start measure minimal racy interval in {}", //$NON-NLS-1$ + Thread.currentThread(), dir); + int failures = 0; + long racyNanos = 0; + final int COUNT = 1000; + ArrayList deltas = new ArrayList<>(); + Path probe = dir.resolve(".probe-" + UUID.randomUUID()); //$NON-NLS-1$ + try { + Files.createFile(probe); + for (int i = 0; i < COUNT; i++) { + write(probe, "a"); //$NON-NLS-1$ + FileSnapshot snapshot = FileSnapshot.save(probe.toFile()); + read(probe); + write(probe, "b"); //$NON-NLS-1$ + if (!snapshot.isModified(probe.toFile())) { + deltas.add(Long.valueOf(snapshot.lastDelta())); + racyNanos = snapshot.lastRacyThreshold(); + failures++; + } + } + } catch (IOException e) { + LOG.error(e.getMessage(), e); + return FALLBACK_MIN_RACY_INTERVAL; + } finally { + deleteProbe(probe); + } + if (failures > 0) { + Stats stats = new Stats(); + for (Long d : deltas) { + stats.add(d); + } + LOG.debug( + "delta [ns] since modification FileSnapshot failed to detect\n" //$NON-NLS-1$ + + "count, failures, racy limit [ns], delta min [ns]," //$NON-NLS-1$ + + " delta max [ns], delta avg [ns]," //$NON-NLS-1$ + + " delta stddev [ns]\n" //$NON-NLS-1$ + + "{}, {}, {}, {}, {}, {}, {}", //$NON-NLS-1$ + COUNT, failures, racyNanos, stats.min(), stats.max(), + stats.avg(), stats.stddev()); + return Duration + .ofNanos(Double.valueOf(stats.max()).longValue()); + } + // since no failures occurred using the measured filesystem + // timestamp resolution there is no need for minimal racy interval + LOG.debug("{}: no failures when measuring minimal racy interval", //$NON-NLS-1$ + Thread.currentThread()); + return Duration.ZERO; + } + + private static void write(Path p, String body) throws IOException { + FileUtils.mkdirs(p.getParent().toFile(), true); + try (Writer w = new OutputStreamWriter(Files.newOutputStream(p), + UTF_8)) { + w.write(body); + } + } + + private static String read(Path p) throws IOException { + final byte[] body = IO.readFully(p.toFile()); + return new String(body, 0, body.length, UTF_8); } private static Optional measureFsTimestampResolution( FileStore s, Path dir) { + LOG.debug("{}: start measure timestamp resolution {} in {}", //$NON-NLS-1$ + Thread.currentThread(), s, dir); Duration configured = readFileTimeResolution(s); if (!UNDEFINED_RESOLUTION.equals(configured)) { return Optional.of(configured); @@ -310,6 +424,8 @@ public abstract class FS { Duration clockResolution = measureClockResolution(); fsResolution = fsResolution.plus(clockResolution); saveFileTimeResolution(s, fsResolution); + LOG.debug("{}: end measure timestamp resolution {} in {}", //$NON-NLS-1$ + Thread.currentThread(), s, dir); return Optional.of(fsResolution); } catch (AccessDeniedException e) { LOG.warn(e.getLocalizedMessage(), e); // see bug 548648 @@ -424,21 +540,45 @@ public abstract class FS { private final @NonNull Duration fsTimestampResolution; + private Duration minimalRacyInterval; + + /** + * @return the measured minimal interval after a file has been modified + * in which we cannot rely on lastModified to detect + * modifications + */ + public Duration getMinimalRacyInterval() { + return minimalRacyInterval; + } + + /** + * @return the measured filesystem timestamp resolution + */ @NonNull - Duration getFsTimestampResolution() { + public Duration getFsTimestampResolution() { return fsTimestampResolution; } - private FileStoreAttributeCache( + /** + * Construct a FileStoreAttributeCache entry for the given filesystem + * timestamp resolution + * + * @param fsTimestampResolution + */ + public FileStoreAttributeCache( @NonNull Duration fsTimestampResolution) { this.fsTimestampResolution = fsTimestampResolution; + this.minimalRacyInterval = Duration.ZERO; } - @SuppressWarnings("nls") + @SuppressWarnings({ "nls", "boxing" }) @Override public String toString() { - return "FileStoreAttributeCache [fsTimestampResolution=" - + fsTimestampResolution + "]"; + return String.format( + "FileStoreAttributeCache[fsTimestampResolution=%,d µs, " + + "minimalRacyInterval=%,d µs]", + fsTimestampResolution.toNanos() / 1000, + minimalRacyInterval.toNanos() / 1000); } } @@ -507,10 +647,11 @@ public abstract class FS { * the directory under which the probe file will be created to * measure the timer resolution. * @return measured filesystem timestamp resolution - * @since 5.2.3 + * @since 5.1.9 */ - public static Duration getFsTimerResolution(@NonNull Path dir) { - return FileStoreAttributeCache.getFsTimestampResolution(dir); + public static FileStoreAttributeCache getFileStoreAttributeCache( + @NonNull Path dir) { + return FileStoreAttributeCache.get(dir); } private volatile Holder userHome; -- 2.39.5