From 5606a53151e8b9b4785e5c01513eb331998b8278 Mon Sep 17 00:00:00 2001 From: Kaushik Lingarkar Date: Thu, 18 Nov 2021 13:12:17 -0800 Subject: FileSnapshot: Lazy load file store attributes cache Doing a getFileStoreAttributes call even when the file doesn't exist is unnecessary. This call is particularly slow on some filesystems. Instead, do it only when the file exists and load the appropriate cache. This update can help speed up RefDirectory.exactRef when the ref is packed, but has a corresponding empty dir for it under 'refs/'. This scenario can happen when an atomic 'BatchRefUpdate' creates new sharded refs. For example, consider the case where we create 50k sharded refs in a new namespace called 'new-refs' using an atomic 'BatchRefUpdate'. The refs are named like 'refs/new-refs/01/1/1', 'refs/new-refs/01/1/2', 'refs/new-refs/01/1/3' and so on. After the refs are created, the 'new-refs' namespace looks like below: $ find refs/new-refs -type f | wc -l 0 $ find refs/new-refs -type d | wc -l 5101 At this point, an 'exactRef' call on each of the 50k refs without this change takes ~30s, where as with this change it takes ~2.5s. Change-Id: I4a5d4c6a652dbeed1f4bc3b4f2b2f1416f7ca0e7 Signed-off-by: Kaushik Lingarkar --- .../jgit/internal/storage/file/FileSnapshot.java | 27 ++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 976f946e5d..487ae06014 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -217,6 +217,12 @@ public class FileSnapshot { /** measured FileStore attributes */ private FileStoreAttributes fileStoreAttributeCache; + /** + * if {@code true} read filesystem time resolution from configuration file + * otherwise use fallback resolution + */ + private boolean useConfig; + /** * Object that uniquely identifies the given file, or {@code * null} if a file key is not available @@ -253,9 +259,7 @@ public class FileSnapshot { protected FileSnapshot(File file, boolean useConfig) { this.file = file; this.lastRead = Instant.now(); - this.fileStoreAttributeCache = useConfig - ? FS.getFileStoreAttributes(file.toPath().getParent()) - : FALLBACK_FILESTORE_ATTRIBUTES; + this.useConfig = useConfig; BasicFileAttributes fileAttributes = null; try { fileAttributes = FS.DETECTED.fileAttributes(file); @@ -399,7 +403,7 @@ public class FileSnapshot { * if sleep was interrupted */ public void waitUntilNotRacy() throws InterruptedException { - long timestampResolution = fileStoreAttributeCache + long timestampResolution = fileStoreAttributeCache() .getFsTimestampResolution().toNanos(); while (isRacyClean(Instant.now())) { TimeUnit.NANOSECONDS.sleep(timestampResolution); @@ -519,10 +523,10 @@ public class FileSnapshot { } private long getEffectiveRacyThreshold() { - long timestampResolution = fileStoreAttributeCache + long timestampResolution = fileStoreAttributeCache() .getFsTimestampResolution().toNanos(); - long minRacyInterval = fileStoreAttributeCache.getMinimalRacyInterval() - .toNanos(); + long minRacyInterval = fileStoreAttributeCache() + .getMinimalRacyInterval().toNanos(); long max = Math.max(timestampResolution, minRacyInterval); // safety margin: factor 2.5 below 100ms otherwise 1.25 return max < 100_000_000L ? max * 5 / 2 : max * 5 / 4; @@ -582,4 +586,13 @@ public class FileSnapshot { } return changed; } + + private FileStoreAttributes fileStoreAttributeCache() { + if (fileStoreAttributeCache == null) { + fileStoreAttributeCache = useConfig + ? FS.getFileStoreAttributes(file.toPath().getParent()) + : FALLBACK_FILESTORE_ATTRIBUTES; + } + return fileStoreAttributeCache; + } } -- cgit v1.2.3 From f829f5f838e0f9c17373ea6cb3407976a8f395ff Mon Sep 17 00:00:00 2001 From: Kaushik Lingarkar Date: Mon, 29 Nov 2021 12:14:56 -0800 Subject: RefDirectory.scanRef: Re-use file existence check done in snapshot creation Return immediately in scanRef if the loose ref was identified as missing when a snapshot was attempted for the ref. This will help performance of scanRef when the ref is packed but has a corresponding empty dir in 'refs/'. For example, consider the case where we create 50k sharded refs in a new namespace called 'new-refs' using an atomic 'BatchRefUpdate'. The refs are named like 'refs/new-refs/01/1/1', 'refs/new-refs/01/1/2', 'refs/new-refs/01/1/3' and so on. After the refs are created, the 'new-refs' namespace looks like below: $ find refs/new-refs -type f | wc -l 0 $ find refs/new-refs -type d | wc -l 5101 At this point, an 'exactRef' call on each of the 50k refs without this change takes ~2.5s, where as with this change it takes ~1.5s. Change-Id: I926bc41b9ae89a1a792b1b5ec9a17b05271c906b Signed-off-by: Kaushik Lingarkar --- .../src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java | 9 +++++++++ .../src/org/eclipse/jgit/internal/storage/file/RefDirectory.java | 4 ++++ 2 files changed, 13 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 487ae06014..654e98d817 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -439,6 +439,15 @@ public class FileSnapshot { return equals(other); } + /** + * Check if the file exists + * + * @return true if the file exists + */ + public boolean fileExists() { + return !MISSING_FILEKEY.equals(this.fileKey); + } + /** {@inheritDoc} */ @Override public int hashCode() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index f7a52a54b6..c15e26cd43 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -1103,6 +1103,10 @@ public class RefDirectory extends RefDatabase { final int limit = 4096; final byte[] buf; FileSnapshot otherSnapshot = FileSnapshot.save(path); + if (!otherSnapshot.fileExists()) { + return null; + } + try { buf = IO.readSome(path, limit); } catch (FileNotFoundException noFile) { -- cgit v1.2.3 From 38db89142dc2f75fc63fd316a3bf8b0f6d8655a2 Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Fri, 3 Dec 2021 10:05:53 -0700 Subject: storage: file: De-duplicate File.exists()+File.isFile() File.isFile() [1] checks if the file exists and is a normal file. [1] https://docs.oracle.com/javase/8/docs/api/java/io/File.html#isFile-- Change-Id: I0a883f2482ecc5ac58b270351b416742b568eb68 Signed-off-by: Nasser Grainawi --- .../eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java | 2 +- .../src/org/eclipse/jgit/internal/storage/file/RefDirectory.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index 6e8a15e86d..889a5108d4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -184,7 +184,7 @@ public class ObjectDirectoryPackParser extends PackParser { String p = pack.getAbsolutePath(); String i = p.substring(0, p.length() - ".pack".length()) + ".idx"; //$NON-NLS-1$ //$NON-NLS-2$ File idx = new File(i); - if (idx.exists() && idx.isFile()) + if (idx.isFile()) size += idx.length(); return size; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index c15e26cd43..c34be9cc0a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -1110,7 +1110,7 @@ public class RefDirectory extends RefDatabase { try { buf = IO.readSome(path, limit); } catch (FileNotFoundException noFile) { - if (path.exists() && path.isFile()) { + if (path.isFile()) { throw noFile; } return null; // doesn't exist or no file; not a reference. -- cgit v1.2.3