jro_apps/redmine Commit - r9149:9b333e1c8772

scm: git: process new git revisions all at once rather than per branch (#10470)...

Toshi MARUYAMA -

r9149:9b333e1c8772

parent child

Context file:

r9149:9b333e1c8772

Collapse all files

app/models/repository/git.rb +22 -35

               # However, Git does not have a sequential commit numbering.
               #
               # In order to fetch only new adding revisions,
-              # Redmine needs to parse revisions per branch.
+              # Redmine needs to save "heads".
-              # Branch "last_scmid" is for this requirement.
               #
               # In Git and Mercurial, revisions are not in date order.
               # Redmine Mercurial fixed issues.
               def fetch_changesets
                 scm_brs = branches
                 return if scm_brs.nil? || scm_brs.empty?
                 h1 = extra_info || {}
                 h  = h1.dup
-                h["branches"]       ||= {}
+                repo_heads = scm_brs.map{ |br| br.scmid }
+                h["heads"] ||= []
+                prev_db_heads = h["heads"].dup
+                if prev_db_heads.empty?
+                  prev_db_heads += heads_from_branches_hash
+                end
+                return if prev_db_heads.sort == repo_heads.sort
                 h["db_consistent"]  ||= {}
                 if changesets.count == 0
                   h["db_consistent"]["ordering"] = 1
                   merge_extra_info(h)
                   self.save
                 end
-                save_revisions(h, scm_brs)
+                save_revisions(prev_db_heads, repo_heads)
               end
-              def save_revisions(h, scm_brs)
+              def save_revisions(prev_db_heads, repo_heads)
-                # Remember what revisions we already processed (in any branches)
+                  h = {}
-                all_revisions = []
+                  opts = {}
-                scm_brs.each do |br1|
+                  opts[:reverse]  = true
-                  br = br1.to_s
+                  opts[:excludes] = prev_db_heads
-                  last_revision = nil
+                  opts[:includes] = repo_heads
-                  from_scmid = nil
-                  from_scmid = h["branches"][br]["last_scmid"] if h["branches"][br]
-                  h["branches"][br] ||= {}
-                  revisions = scm.revisions('', from_scmid, br, {:reverse => true})
-                  next if revisions.blank?
-                  # Remember the last commit id here, before we start removing revisions from the array.
-                  # We'll do that for optimization, but it also means, that we may lose even all revisions.
-                  last_revision  = revisions.last
-                  # remove revisions that we have already processed (possibly in other branches)
+                  revisions = scm.revisions('', nil, nil, opts)
-                  revisions.reject!{|r| all_revisions.include?(r.scmid)}
+                  return if revisions.blank?
-                  # add revisions that we are to parse now to 'all processed revisions'
-                  # (this equals to a union, because we executed diff above)
-                  all_revisions += revisions.map{|r| r.scmid}
                   # Make the search for existing revisions in the database in a more sufficient manner
                   # This is replacing the one-after-one queries.
                       parents.each do |ch, chparents|
                         ch.parents = chparents.collect{|rp| find_changeset_by_name(rp)}.compact
                       end
-                      # saving the last scmid was moved from here, because we won't come in here,
-                      # if the revision was already added for another branch
                     end
                   end
+                  h["heads"] = repo_heads.dup
-                  # save the data about the last revision for this branch
+                  merge_extra_info(h)
-                  unless last_revision.nil?
+                  self.save
-                    h["branches"][br]["last_scmid"] = last_revision.scmid
-                    merge_extra_info(h)
-                    self.save
-                  end
-                end
               end
               private :save_revisions

test/unit/repository_git_test.rb +38 -30

                   assert_equal "README", change.path
                   assert_equal "A", change.action
-                  assert_equal NUM_HEAD, @repository.extra_info["branches"].size
+                  assert_equal NUM_HEAD, @repository.extra_info["heads"].size
                 end
                 def test_fetch_changesets_incremental
                   @repository.fetch_changesets
                   @project.reload
                   assert_equal NUM_REV, @repository.changesets.count
-                  extra_info_db = @repository.extra_info["branches"]
+                  extra_info_heads = @repository.extra_info["heads"].dup
-                  assert_equal "1ca7f5ed374f3cb31a93ae5215c2e25cc6ec5127",
+                  assert_equal NUM_HEAD, extra_info_heads.size
-                                extra_info_db["latin-1-path-encoding"]["last_scmid"]
+                  extra_info_heads.delete_if { |x| x == "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c" }
-                  assert_equal "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c",
+                  assert_equal 4, extra_info_heads.size
-                                extra_info_db["master"]["last_scmid"]
                   del_revs = [
                       "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c",
                       "ed5bb786bbda2dee66a2d50faf51429dbc043a7b",
                   end
                   @project.reload
                   cs1 = @repository.changesets
-                  assert_equal 22, cs1.count
+                  assert_equal NUM_REV - 6, cs1.count
-                  h = @repository.extra_info.dup
+                  extra_info_heads << "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8"
-                  h["branches"]["master"]["last_scmid"] =
+                  h = {}
-                        "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8"
+                  h["heads"] = extra_info_heads
                   @repository.merge_extra_info(h)
                   @repository.save
                   @project.reload
-                  extra_info_db_1 = @repository.extra_info["branches"]
+                  assert @repository.extra_info["heads"].index("4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8")
-                  assert_equal "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8",
-                                extra_info_db_1["master"]["last_scmid"]
                   @repository.fetch_changesets
                   @project.reload
                   assert_equal NUM_REV, @repository.changesets.count
+                  assert_equal NUM_HEAD, @repository.extra_info["heads"].size
+                  assert @repository.extra_info["heads"].index("83ca5fd546063a3c7dc2e568ba3355661a9e2b2c")
                 end
                 def test_fetch_changesets_history_editing
                   @repository.fetch_changesets
                   @project.reload
                   assert_equal NUM_REV, @repository.changesets.count
-                  assert_equal NUM_HEAD, @repository.extra_info["branches"].size
+                  extra_info_heads = @repository.extra_info["heads"].dup
-                  assert_equal "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c",
+                  assert_equal NUM_HEAD, extra_info_heads.size
-                                 @repository.extra_info["branches"]["master"]["last_scmid"]
+                  extra_info_heads.delete_if { |x| x == "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c" }
+                  assert_equal 4, extra_info_heads.size
                   del_revs = [
                       "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c",
                       "ed5bb786bbda2dee66a2d50faf51429dbc043a7b",
                   @project.reload
                   assert_equal NUM_REV - 5, @repository.changesets.count
-                  h = @repository.extra_info.dup
+                  extra_info_heads << "abcd1234efgh"
-                  h["branches"]["master"]["last_scmid"] = "abcd1234efgh"
+                  h = {}
+                  h["heads"] = extra_info_heads
                   @repository.merge_extra_info(h)
                   @repository.save
                   @project.reload
-                  assert_equal "abcd1234efgh",
+                  h1 = @repository.extra_info["heads"].dup
-                                @repository.extra_info["branches"]["master"]["last_scmid"]
+                  assert h1.index("abcd1234efgh")
+                  assert_equal 5, h1.size
                   @repository.fetch_changesets
                   @project.reload
                   assert_equal NUM_REV - 5, @repository.changesets.count
+                  h2 = @repository.extra_info["heads"].dup
+                  assert_equal h1, h2
                 end
                 def test_parents
                   @project.reload
                   assert_equal 0, @repository.extra_info["db_consistent"]["ordering"]
+                  extra_info_heads = @repository.extra_info["heads"].dup
+                  extra_info_heads.delete_if { |x| x == "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c" }
                   del_revs = [
                       "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c",
                       "ed5bb786bbda2dee66a2d50faf51429dbc043a7b",
                   cs1 = @repository.changesets
                   assert_equal NUM_REV - 6, cs1.count
                   assert_equal 0, @repository.extra_info["db_consistent"]["ordering"]
-                  h = @repository.extra_info.dup
-                  h["branches"]["master"]["last_scmid"] =
+                  extra_info_heads << "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8"
-                        "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8"
+                  h = {}
+                  h["heads"] = extra_info_heads
                   @repository.merge_extra_info(h)
                   @repository.save
                   @project.reload
-                  extra_info_db_1 = @repository.extra_info["branches"]
+                  assert @repository.extra_info["heads"].index("4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8")
-                  assert_equal "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8",
-                                extra_info_db_1["master"]["last_scmid"]
                   @repository.fetch_changesets
+                  @project.reload
                   assert_equal NUM_REV, @repository.changesets.count
+                  assert_equal NUM_HEAD, @repository.extra_info["heads"].size
                   assert_equal 0, @repository.extra_info["db_consistent"]["ordering"]
                 end
                   @repository.fetch_changesets
                   @project.reload
                   assert_equal NUM_REV, @repository.changesets.count
-                  %w|95488a44bc25f7d1f97d775a31359539ff333a63 95488a44b|.each do |r1|
+                  %w|7234cb2750b63f47bff735edc50a1c0a433c2518 7234cb275|.each do |r1|
                     changeset = @repository.find_changeset_by_name(r1)
                     assert_nil changeset.previous
                   end
                   @repository.fetch_changesets
                   @project.reload
                   assert_equal NUM_REV, @repository.changesets.count
-                  %w|67e7792ce20ccae2e4bb73eed09bb397819c8834 67e7792ce20cca|.each do |r1|
+                  %w|2a682156a3b6e77a8bf9cd4590e8db757f3c6c78 2a682156a3b6e77a|.each do |r1|
                     changeset = @repository.find_changeset_by_name(r1)
                     assert_nil changeset.next
                   end

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages