From 9b333e1c87722f1d580a7e1a9f5e72fad985cf60 Mon Sep 17 00:00:00 2001 From: Toshi MARUYAMA Date: Wed, 28 Mar 2012 17:18:04 +0000 Subject: [PATCH] scm: git: process new git revisions all at once rather than per branch (#10470) git-svn-id: svn+ssh://rubyforge.org/var/svn/redmine/trunk@9283 e93f8b46-1217-0410-a6f0-8f06a7374b81 --- app/models/repository/git.rb | 57 +++++++++++--------------- test/unit/repository_git_test.rb | 68 ++++++++++++++++++-------------- 2 files changed, 60 insertions(+), 65 deletions(-) diff --git a/app/models/repository/git.rb b/app/models/repository/git.rb index 933b5e3c1..6bd4a054a 100644 --- a/app/models/repository/git.rb +++ b/app/models/repository/git.rb @@ -107,8 +107,7 @@ class Repository::Git < Repository # However, Git does not have a sequential commit numbering. # # In order to fetch only new adding revisions, - # Redmine needs to parse revisions per branch. - # Branch "last_scmid" is for this requirement. + # Redmine needs to save "heads". # # In Git and Mercurial, revisions are not in date order. # Redmine Mercurial fixed issues. @@ -131,9 +130,17 @@ class Repository::Git < Repository def fetch_changesets scm_brs = branches return if scm_brs.nil? || scm_brs.empty? + h1 = extra_info || {} h = h1.dup - h["branches"] ||= {} + repo_heads = scm_brs.map{ |br| br.scmid } + h["heads"] ||= [] + prev_db_heads = h["heads"].dup + if prev_db_heads.empty? + prev_db_heads += heads_from_branches_hash + end + return if prev_db_heads.sort == repo_heads.sort + h["db_consistent"] ||= {} if changesets.count == 0 h["db_consistent"]["ordering"] = 1 @@ -144,31 +151,18 @@ class Repository::Git < Repository merge_extra_info(h) self.save end - save_revisions(h, scm_brs) + save_revisions(prev_db_heads, repo_heads) end - def save_revisions(h, scm_brs) - # Remember what revisions we already processed (in any branches) - all_revisions = [] - scm_brs.each do |br1| - br = br1.to_s - last_revision = nil - from_scmid = nil - from_scmid = h["branches"][br]["last_scmid"] if h["branches"][br] - h["branches"][br] ||= {} - - revisions = scm.revisions('', from_scmid, br, {:reverse => true}) - next if revisions.blank? - - # Remember the last commit id here, before we start removing revisions from the array. - # We'll do that for optimization, but it also means, that we may lose even all revisions. - last_revision = revisions.last + def save_revisions(prev_db_heads, repo_heads) + h = {} + opts = {} + opts[:reverse] = true + opts[:excludes] = prev_db_heads + opts[:includes] = repo_heads - # remove revisions that we have already processed (possibly in other branches) - revisions.reject!{|r| all_revisions.include?(r.scmid)} - # add revisions that we are to parse now to 'all processed revisions' - # (this equals to a union, because we executed diff above) - all_revisions += revisions.map{|r| r.scmid} + revisions = scm.revisions('', nil, nil, opts) + return if revisions.blank? # Make the search for existing revisions in the database in a more sufficient manner # This is replacing the one-after-one queries. @@ -206,18 +200,11 @@ class Repository::Git < Repository parents.each do |ch, chparents| ch.parents = chparents.collect{|rp| find_changeset_by_name(rp)}.compact end - # saving the last scmid was moved from here, because we won't come in here, - # if the revision was already added for another branch end end - - # save the data about the last revision for this branch - unless last_revision.nil? - h["branches"][br]["last_scmid"] = last_revision.scmid - merge_extra_info(h) - self.save - end - end + h["heads"] = repo_heads.dup + merge_extra_info(h) + self.save end private :save_revisions diff --git a/test/unit/repository_git_test.rb b/test/unit/repository_git_test.rb index bc1e3a632..91b41af94 100644 --- a/test/unit/repository_git_test.rb +++ b/test/unit/repository_git_test.rb @@ -116,7 +116,7 @@ class RepositoryGitTest < ActiveSupport::TestCase assert_equal "README", change.path assert_equal "A", change.action - assert_equal NUM_HEAD, @repository.extra_info["branches"].size + assert_equal NUM_HEAD, @repository.extra_info["heads"].size end def test_fetch_changesets_incremental @@ -124,11 +124,11 @@ class RepositoryGitTest < ActiveSupport::TestCase @repository.fetch_changesets @project.reload assert_equal NUM_REV, @repository.changesets.count - extra_info_db = @repository.extra_info["branches"] - assert_equal "1ca7f5ed374f3cb31a93ae5215c2e25cc6ec5127", - extra_info_db["latin-1-path-encoding"]["last_scmid"] - assert_equal "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c", - extra_info_db["master"]["last_scmid"] + extra_info_heads = @repository.extra_info["heads"].dup + assert_equal NUM_HEAD, extra_info_heads.size + extra_info_heads.delete_if { |x| x == "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c" } + assert_equal 4, extra_info_heads.size + del_revs = [ "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c", "ed5bb786bbda2dee66a2d50faf51429dbc043a7b", @@ -142,20 +142,19 @@ class RepositoryGitTest < ActiveSupport::TestCase end @project.reload cs1 = @repository.changesets - assert_equal 22, cs1.count - h = @repository.extra_info.dup - h["branches"]["master"]["last_scmid"] = - "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8" + assert_equal NUM_REV - 6, cs1.count + extra_info_heads << "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8" + h = {} + h["heads"] = extra_info_heads @repository.merge_extra_info(h) @repository.save @project.reload - extra_info_db_1 = @repository.extra_info["branches"] - assert_equal "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8", - extra_info_db_1["master"]["last_scmid"] - + assert @repository.extra_info["heads"].index("4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8") @repository.fetch_changesets @project.reload assert_equal NUM_REV, @repository.changesets.count + assert_equal NUM_HEAD, @repository.extra_info["heads"].size + assert @repository.extra_info["heads"].index("83ca5fd546063a3c7dc2e568ba3355661a9e2b2c") end def test_fetch_changesets_history_editing @@ -163,9 +162,11 @@ class RepositoryGitTest < ActiveSupport::TestCase @repository.fetch_changesets @project.reload assert_equal NUM_REV, @repository.changesets.count - assert_equal NUM_HEAD, @repository.extra_info["branches"].size - assert_equal "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c", - @repository.extra_info["branches"]["master"]["last_scmid"] + extra_info_heads = @repository.extra_info["heads"].dup + assert_equal NUM_HEAD, extra_info_heads.size + extra_info_heads.delete_if { |x| x == "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c" } + assert_equal 4, extra_info_heads.size + del_revs = [ "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c", "ed5bb786bbda2dee66a2d50faf51429dbc043a7b", @@ -189,17 +190,21 @@ class RepositoryGitTest < ActiveSupport::TestCase @project.reload assert_equal NUM_REV - 5, @repository.changesets.count - h = @repository.extra_info.dup - h["branches"]["master"]["last_scmid"] = "abcd1234efgh" + extra_info_heads << "abcd1234efgh" + h = {} + h["heads"] = extra_info_heads @repository.merge_extra_info(h) @repository.save @project.reload - assert_equal "abcd1234efgh", - @repository.extra_info["branches"]["master"]["last_scmid"] + h1 = @repository.extra_info["heads"].dup + assert h1.index("abcd1234efgh") + assert_equal 5, h1.size @repository.fetch_changesets @project.reload assert_equal NUM_REV - 5, @repository.changesets.count + h2 = @repository.extra_info["heads"].dup + assert_equal h1, h2 end def test_parents @@ -246,6 +251,8 @@ class RepositoryGitTest < ActiveSupport::TestCase @project.reload assert_equal 0, @repository.extra_info["db_consistent"]["ordering"] + extra_info_heads = @repository.extra_info["heads"].dup + extra_info_heads.delete_if { |x| x == "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c" } del_revs = [ "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c", "ed5bb786bbda2dee66a2d50faf51429dbc043a7b", @@ -261,18 +268,19 @@ class RepositoryGitTest < ActiveSupport::TestCase cs1 = @repository.changesets assert_equal NUM_REV - 6, cs1.count assert_equal 0, @repository.extra_info["db_consistent"]["ordering"] - h = @repository.extra_info.dup - h["branches"]["master"]["last_scmid"] = - "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8" + + extra_info_heads << "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8" + h = {} + h["heads"] = extra_info_heads @repository.merge_extra_info(h) @repository.save @project.reload - extra_info_db_1 = @repository.extra_info["branches"] - assert_equal "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8", - extra_info_db_1["master"]["last_scmid"] - + assert @repository.extra_info["heads"].index("4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8") @repository.fetch_changesets + @project.reload assert_equal NUM_REV, @repository.changesets.count + assert_equal NUM_HEAD, @repository.extra_info["heads"].size + assert_equal 0, @repository.extra_info["db_consistent"]["ordering"] end @@ -511,7 +519,7 @@ class RepositoryGitTest < ActiveSupport::TestCase @repository.fetch_changesets @project.reload assert_equal NUM_REV, @repository.changesets.count - %w|95488a44bc25f7d1f97d775a31359539ff333a63 95488a44b|.each do |r1| + %w|7234cb2750b63f47bff735edc50a1c0a433c2518 7234cb275|.each do |r1| changeset = @repository.find_changeset_by_name(r1) assert_nil changeset.previous end @@ -535,7 +543,7 @@ class RepositoryGitTest < ActiveSupport::TestCase @repository.fetch_changesets @project.reload assert_equal NUM_REV, @repository.changesets.count - %w|67e7792ce20ccae2e4bb73eed09bb397819c8834 67e7792ce20cca|.each do |r1| + %w|2a682156a3b6e77a8bf9cd4590e8db757f3c6c78 2a682156a3b6e77a|.each do |r1| changeset = @repository.find_changeset_by_name(r1) assert_nil changeset.next end -- 2.39.5