]> source.dussan.org Git - jgit.git/commitdiff
Pass PackConfig down to PackWriter when packing 90/1190/2
authorShawn O. Pearce <spearce@spearce.org>
Wed, 28 Jul 2010 17:48:53 +0000 (10:48 -0700)
committerShawn O. Pearce <spearce@spearce.org>
Wed, 28 Jul 2010 19:13:48 +0000 (12:13 -0700)
When we are creating a pack the higher level application should be able
to override the PackConfig used, allowing it to control the number of
threads used or how much memory is allocated per writer.

Change-Id: I47795987bb0d161d3642082acc2f617d7cb28d8c
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/CLIText.properties
org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/CLIText.java
org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java
org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackPushConnection.java
org.eclipse.jgit/src/org/eclipse/jgit/transport/BundleWriter.java
org.eclipse.jgit/src/org/eclipse/jgit/transport/Daemon.java
org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java
org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java
org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java

index e879d6b605cc113208a4c89c146dd925f82e5aa4..2fff6d4630a147fe41af848d3016df59830d9180 100644 (file)
@@ -14,6 +14,7 @@ branchCreatedFrom =branch: Created from {0}
 branchIsNotAnAncestorOfYourCurrentHEAD=The branch '{0}' is not an ancestor of your current HEAD.\nIf you are sure you want to delete it, run 'jgit branch -D {0}'.
 branchNotFound=branch '{0}' not found.
 cacheTreePathInfo="{0}": {1} entries, {2} children
+configFileNotFound=configuration file {0} not found
 cannotBeRenamed={0} cannot be renamed
 cannotChekoutNoHeadsAdvertisedByRemote=cannot checkout; no HEAD advertised by remote
 cannotCreateCommand=Cannot create command {0}
@@ -61,6 +62,7 @@ metaVar_bucket=BUCKET
 metaVar_command=command
 metaVar_commitOrTag=COMMIT|TAG
 metaVar_commitish=commit-ish
+metaVar_configFile=FILE
 metaVar_connProp=conn.prop
 metaVar_directory=DIRECTORY
 metaVar_file=FILE
@@ -138,6 +140,7 @@ usage_approveDestructionOfRepository=approve destruction of repository
 usage_beMoreVerbose=be more verbose
 usage_beVerbose=be verbose
 usage_cloneRepositoryIntoNewDir=Clone a repository into a new directory
+usage_configFile=configuration file
 usage_configureTheServiceInDaemonServicename=configure the service in daemon.servicename
 usage_deleteBranchEvenIfNotMerged=delete branch (even if not merged)
 usage_deleteFullyMergedBranch=delete fully merged branch
index bae895cc76e0daf95d4d036568bdbe76eafea747..14dcb1f50df58f66224e7b38ea19a91763bb5735 100644 (file)
@@ -67,6 +67,7 @@ public class CLIText extends TranslationBundle {
        /***/ public String branchIsNotAnAncestorOfYourCurrentHEAD;
        /***/ public String branchNotFound;
        /***/ public String cacheTreePathInfo;
+       /***/ public String configFileNotFound;
        /***/ public String cannotBeRenamed;
        /***/ public String cannotChekoutNoHeadsAdvertisedByRemote;
        /***/ public String cannotCreateCommand;
index f015a9e7bdbe7c444b9d2031cd546255ba0a5124..3cca87abe8fd9bb11320c36ec2596fc7058a8302 100644 (file)
@@ -48,13 +48,22 @@ import java.net.InetSocketAddress;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.Executors;
 
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.storage.file.WindowCache;
+import org.eclipse.jgit.storage.file.WindowCacheConfig;
+import org.eclipse.jgit.storage.pack.PackConfig;
+import org.eclipse.jgit.transport.DaemonService;
+import org.eclipse.jgit.util.FS;
 import org.kohsuke.args4j.Argument;
 import org.kohsuke.args4j.Option;
-import org.eclipse.jgit.transport.DaemonService;
 
 @Command(common = true, usage = "usage_exportRepositoriesOverGit")
 class Daemon extends TextBuiltin {
+       @Option(name = "--config-file", metaVar = "metaVar_configFile", usage = "usage_configFile")
+       File configFile;
+
        @Option(name = "--port", metaVar = "metaVar_port", usage = "usage_portNumberToListenOn")
        int port = org.eclipse.jgit.transport.Daemon.DEFAULT_PORT;
 
@@ -89,12 +98,38 @@ class Daemon extends TextBuiltin {
 
        @Override
        protected void run() throws Exception {
+               PackConfig packConfig = new PackConfig();
+
+               if (configFile != null) {
+                       if (!configFile.exists()) {
+                               throw die(MessageFormat.format(
+                                               CLIText.get().configFileNotFound, //
+                                               configFile.getAbsolutePath()));
+                       }
+
+                       FileBasedConfig cfg = new FileBasedConfig(configFile, FS.DETECTED);
+                       cfg.load();
+
+                       WindowCacheConfig wcc = new WindowCacheConfig();
+                       wcc.fromConfig(cfg);
+                       WindowCache.reconfigure(wcc);
+
+                       packConfig.fromConfig(cfg);
+               }
+
+               int threads = packConfig.getThreads();
+               if (threads <= 0)
+                       threads = Runtime.getRuntime().availableProcessors();
+               if (1 < threads)
+                       packConfig.setExecutor(Executors.newFixedThreadPool(threads));
+
                final org.eclipse.jgit.transport.Daemon d;
 
                d = new org.eclipse.jgit.transport.Daemon(
                                host != null ? new InetSocketAddress(host, port)
                                                : new InetSocketAddress(port));
                d.setExportAll(exportAll);
+               d.setPackConfig(packConfig);
                if (0 <= timeout)
                        d.setTimeout(timeout);
 
index 297105d468932b57e6be28b8c760a7feb5120f86..0838f29c1ad65ee5c625124c60e3292d6694b70d 100644 (file)
@@ -231,7 +231,8 @@ class BasePackPushConnection extends BasePackConnection implements
                List<ObjectId> newObjects = new ArrayList<ObjectId>(refUpdates.size());
 
                final long start;
-               final PackWriter writer = new PackWriter(local);
+               final PackWriter writer = new PackWriter(transport.getPackConfig(),
+                               local.newObjectReader());
                try {
 
                        for (final Ref r : getRefs())
index 79fa58c368338ef5e954fa0a9fd69fb9fa54ad74..b513412da2f267f299af589dd649ad39933ea724 100644 (file)
@@ -61,6 +61,7 @@ import org.eclipse.jgit.lib.ProgressMonitor;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.storage.pack.PackConfig;
 import org.eclipse.jgit.storage.pack.PackWriter;
 
 /**
@@ -81,12 +82,14 @@ import org.eclipse.jgit.storage.pack.PackWriter;
  * overall bundle size.
  */
 public class BundleWriter {
-       private final PackWriter packWriter;
+       private final Repository db;
 
        private final Map<String, ObjectId> include;
 
        private final Set<RevCommit> assume;
 
+       private PackConfig packConfig;
+
        /**
         * Create a writer for a bundle.
         *
@@ -94,11 +97,22 @@ public class BundleWriter {
         *            repository where objects are stored.
         */
        public BundleWriter(final Repository repo) {
-               packWriter = new PackWriter(repo);
+               db = repo;
                include = new TreeMap<String, ObjectId>();
                assume = new HashSet<RevCommit>();
        }
 
+       /**
+        * Set the configuration used by the pack generator.
+        *
+        * @param pc
+        *            configuration controlling packing parameters. If null the
+        *            source repository's settings will be used.
+        */
+       public void setPackConfig(PackConfig pc) {
+               this.packConfig = pc;
+       }
+
        /**
         * Include an object (and everything reachable from it) in the bundle.
         *
@@ -166,6 +180,10 @@ public class BundleWriter {
         */
        public void writeBundle(ProgressMonitor monitor, OutputStream os)
                        throws IOException {
+               PackConfig pc = packConfig;
+               if (pc == null)
+                       pc = new PackConfig(db);
+               PackWriter packWriter = new PackWriter(pc, db.newObjectReader());
                try {
                        final HashSet<ObjectId> inc = new HashSet<ObjectId>();
                        final HashSet<ObjectId> exc = new HashSet<ObjectId>();
index aa2e2521c2cb152373ee354775ffe1100f78c0ea..0bc5fb3a2b553daca3f1644bc65864c081d27979 100644 (file)
@@ -63,6 +63,7 @@ import org.eclipse.jgit.lib.PersonIdent;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.lib.RepositoryCache;
 import org.eclipse.jgit.lib.RepositoryCache.FileKey;
+import org.eclipse.jgit.storage.pack.PackConfig;
 import org.eclipse.jgit.util.FS;
 
 /** Basic daemon for the anonymous <code>git://</code> transport protocol. */
@@ -90,6 +91,8 @@ public class Daemon {
 
        private int timeout;
 
+       private PackConfig packConfig;
+
        /** Configure a daemon to listen on any available network port. */
        public Daemon() {
                this(null);
@@ -120,6 +123,7 @@ public class Daemon {
                                                final UploadPack rp = new UploadPack(db);
                                                final InputStream in = dc.getInputStream();
                                                rp.setTimeout(Daemon.this.getTimeout());
+                                               rp.setPackConfig(Daemon.this.packConfig);
                                                rp.upload(in, dc.getOutputStream(), null);
                                        }
                                }, new DaemonService("receive-pack", "receivepack") {
@@ -242,6 +246,17 @@ public class Daemon {
                timeout = seconds;
        }
 
+       /**
+        * Set the configuration used by the pack generator.
+        *
+        * @param pc
+        *            configuration controlling packing parameters. If null the
+        *            source repository's settings will be used.
+        */
+       public void setPackConfig(PackConfig pc) {
+               this.packConfig = pc;
+       }
+
        /**
         * Start this daemon on a background thread.
         *
index a8e47afd3816fac256a5381124b326a783301e24..500cf0cff5e103a735b5863e502c12b172f7c5ad 100644 (file)
@@ -66,6 +66,7 @@ import org.eclipse.jgit.lib.ProgressMonitor;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.lib.TransferConfig;
+import org.eclipse.jgit.storage.pack.PackConfig;
 import org.eclipse.jgit.util.FS;
 
 /**
@@ -554,6 +555,9 @@ public abstract class Transport {
        /** Timeout in seconds to wait before aborting an IO read or write. */
        private int timeout;
 
+       /** Pack configuration used by this transport to make pack file. */
+       private PackConfig packConfig;
+
        /**
         * Create a new transport instance.
         *
@@ -791,6 +795,32 @@ public abstract class Transport {
                timeout = seconds;
        }
 
+       /**
+        * Get the configuration used by the pack generator to make packs.
+        *
+        * If {@link #setPackConfig(PackConfig)} was previously given null a new
+        * PackConfig is created on demand by this method using the source
+        * repository's settings.
+        *
+        * @return the pack configuration. Never null.
+        */
+       public PackConfig getPackConfig() {
+               if (packConfig == null)
+                       packConfig = new PackConfig(local);
+               return packConfig;
+       }
+
+       /**
+        * Set the configuration used by the pack generator.
+        *
+        * @param pc
+        *            configuration controlling packing parameters. If null the
+        *            source repository's settings will be used.
+        */
+       public void setPackConfig(PackConfig pc) {
+               packConfig = pc;
+       }
+
        /**
         * Fetch objects and refs from the remote repository to the local one.
         * <p>
index e7338598c54af4d812795b4c23bd59ba6dc16634..16d56df6664c0e6142f89ad110366214a1dea280 100644 (file)
@@ -69,6 +69,7 @@ import org.eclipse.jgit.revwalk.RevFlagSet;
 import org.eclipse.jgit.revwalk.RevObject;
 import org.eclipse.jgit.revwalk.RevTag;
 import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.storage.pack.PackConfig;
 import org.eclipse.jgit.storage.pack.PackWriter;
 import org.eclipse.jgit.transport.BasePackFetchConnection.MultiAck;
 import org.eclipse.jgit.transport.RefAdvertiser.PacketLineOutRefAdvertiser;
@@ -102,6 +103,9 @@ public class UploadPack {
        /** Revision traversal support over {@link #db}. */
        private final RevWalk walk;
 
+       /** Configuration to pass into the PackWriter. */
+       private PackConfig packConfig;
+
        /** Timeout in seconds to wait for client interaction. */
        private int timeout;
 
@@ -258,6 +262,17 @@ public class UploadPack {
                this.refFilter = refFilter != null ? refFilter : RefFilter.DEFAULT;
        }
 
+       /**
+        * Set the configuration used by the pack generator.
+        *
+        * @param pc
+        *            configuration controlling packing parameters. If null the
+        *            source repository's settings will be used.
+        */
+       public void setPackConfig(PackConfig pc) {
+               this.packConfig = pc;
+       }
+
        /**
         * Execute the upload task on the socket.
         *
@@ -566,7 +581,10 @@ public class UploadPack {
                                                SideBandOutputStream.CH_PROGRESS, bufsz, rawOut));
                }
 
-               final PackWriter pw = new PackWriter(db, walk.getObjectReader());
+               PackConfig cfg = packConfig;
+               if (cfg == null)
+                       cfg = new PackConfig(db);
+               final PackWriter pw = new PackWriter(cfg, walk.getObjectReader());
                try {
                        pw.setDeltaBaseAsOffset(options.contains(OPTION_OFS_DELTA));
                        pw.setThin(options.contains(OPTION_THIN_PACK));
index bbc918f25611bf4cb4cd5a7cdeaf248ec4c17576..9ce0ec1b3c18a3c95ac6a83db71aec11791394f3 100644 (file)
@@ -103,6 +103,9 @@ class WalkPushConnection extends BaseConnection implements PushConnection {
        /** Database connection to the remote repository. */
        private final WalkRemoteObjectDatabase dest;
 
+       /** The configured transport we were constructed by. */
+       private final Transport transport;
+
        /**
         * Packs already known to reside in the remote repository.
         * <p>
@@ -123,9 +126,9 @@ class WalkPushConnection extends BaseConnection implements PushConnection {
 
        WalkPushConnection(final WalkTransport walkTransport,
                        final WalkRemoteObjectDatabase w) {
-               Transport t = (Transport)walkTransport;
-               local = t.local;
-               uri = t.getURI();
+               transport = (Transport) walkTransport;
+               local = transport.local;
+               uri = transport.getURI();
                dest = w;
        }
 
@@ -209,7 +212,8 @@ class WalkPushConnection extends BaseConnection implements PushConnection {
                String pathPack = null;
                String pathIdx = null;
 
-               final PackWriter pw = new PackWriter(local);
+               final PackWriter writer = new PackWriter(transport.getPackConfig(),
+                               local.newObjectReader());
                try {
                        final List<ObjectId> need = new ArrayList<ObjectId>();
                        final List<ObjectId> have = new ArrayList<ObjectId>();
@@ -220,20 +224,20 @@ class WalkPushConnection extends BaseConnection implements PushConnection {
                                if (r.getPeeledObjectId() != null)
                                        have.add(r.getPeeledObjectId());
                        }
-                       pw.preparePack(monitor, need, have);
+                       writer.preparePack(monitor, need, have);
 
                        // We don't have to continue further if the pack will
                        // be an empty pack, as the remote has all objects it
                        // needs to complete this change.
                        //
-                       if (pw.getObjectsNumber() == 0)
+                       if (writer.getObjectsNumber() == 0)
                                return;
 
                        packNames = new LinkedHashMap<String, String>();
                        for (final String n : dest.getPackNames())
                                packNames.put(n, n);
 
-                       final String base = "pack-" + pw.computeName().name();
+                       final String base = "pack-" + writer.computeName().name();
                        final String packName = base + ".pack";
                        pathPack = "pack/" + packName;
                        pathIdx = "pack/" + base + ".idx";
@@ -254,7 +258,7 @@ class WalkPushConnection extends BaseConnection implements PushConnection {
                        OutputStream os = dest.writeFile(pathPack, monitor, wt + "..pack");
                        try {
                                os = new BufferedOutputStream(os);
-                               pw.writePack(monitor, monitor, os);
+                               writer.writePack(monitor, monitor, os);
                        } finally {
                                os.close();
                        }
@@ -262,7 +266,7 @@ class WalkPushConnection extends BaseConnection implements PushConnection {
                        os = dest.writeFile(pathIdx, monitor, wt + "..idx");
                        try {
                                os = new BufferedOutputStream(os);
-                               pw.writeIndex(os);
+                               writer.writeIndex(os);
                        } finally {
                                os.close();
                        }
@@ -282,7 +286,7 @@ class WalkPushConnection extends BaseConnection implements PushConnection {
 
                        throw new TransportException(uri, JGitText.get().cannotStoreObjects, err);
                } finally {
-                       pw.release();
+                       writer.release();
                }
        }