##// END OF EJS Templates
scm: git: performance improvements in fetching revisions (#8857, #9472)...
Toshi MARUYAMA -
r9024:999a4ba30d7b
parent child
Show More
@@ -1,233 +1,265
1 1 # Redmine - project management software
2 2 # Copyright (C) 2006-2011 Jean-Philippe Lang
3 3 # Copyright (C) 2007 Patrick Aljord patcito@ŋmail.com
4 4 #
5 5 # This program is free software; you can redistribute it and/or
6 6 # modify it under the terms of the GNU General Public License
7 7 # as published by the Free Software Foundation; either version 2
8 8 # of the License, or (at your option) any later version.
9 9 #
10 10 # This program is distributed in the hope that it will be useful,
11 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 13 # GNU General Public License for more details.
14 14 #
15 15 # You should have received a copy of the GNU General Public License
16 16 # along with this program; if not, write to the Free Software
17 17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 18
19 19 require 'redmine/scm/adapters/git_adapter'
20 20
21 21 class Repository::Git < Repository
22 22 attr_protected :root_url
23 23 validates_presence_of :url
24 24
25 25 def self.human_attribute_name(attribute_key_name, *args)
26 26 attr_name = attribute_key_name.to_s
27 27 if attr_name == "url"
28 28 attr_name = "path_to_repository"
29 29 end
30 30 super(attr_name, *args)
31 31 end
32 32
33 33 def self.scm_adapter_class
34 34 Redmine::Scm::Adapters::GitAdapter
35 35 end
36 36
37 37 def self.scm_name
38 38 'Git'
39 39 end
40 40
41 41 def report_last_commit
42 42 extra_report_last_commit
43 43 end
44 44
45 45 def extra_report_last_commit
46 46 return false if extra_info.nil?
47 47 v = extra_info["extra_report_last_commit"]
48 48 return false if v.nil?
49 49 v.to_s != '0'
50 50 end
51 51
52 52 def supports_directory_revisions?
53 53 true
54 54 end
55 55
56 56 def supports_revision_graph?
57 57 true
58 58 end
59 59
60 60 def repo_log_encoding
61 61 'UTF-8'
62 62 end
63 63
64 64 # Returns the identifier for the given git changeset
65 65 def self.changeset_identifier(changeset)
66 66 changeset.scmid
67 67 end
68 68
69 69 # Returns the readable identifier for the given git changeset
70 70 def self.format_changeset_identifier(changeset)
71 71 changeset.revision[0, 8]
72 72 end
73 73
74 74 def branches
75 75 scm.branches
76 76 end
77 77
78 78 def tags
79 79 scm.tags
80 80 end
81 81
82 82 def default_branch
83 83 scm.default_branch
84 84 rescue Exception => e
85 85 logger.error "git: error during get default branch: #{e.message}"
86 86 nil
87 87 end
88 88
89 89 def find_changeset_by_name(name)
90 90 return nil if name.nil? || name.empty?
91 91 e = changesets.find(:first, :conditions => ['revision = ?', name.to_s])
92 92 return e if e
93 93 changesets.find(:first, :conditions => ['scmid LIKE ?', "#{name}%"])
94 94 end
95 95
96 96 def entries(path=nil, identifier=nil)
97 97 scm.entries(path,
98 98 identifier,
99 99 options = {:report_last_commit => extra_report_last_commit})
100 100 end
101 101
102 102 # With SCMs that have a sequential commit numbering,
103 103 # such as Subversion and Mercurial,
104 104 # Redmine is able to be clever and only fetch changesets
105 105 # going forward from the most recent one it knows about.
106 106 #
107 107 # However, Git does not have a sequential commit numbering.
108 108 #
109 109 # In order to fetch only new adding revisions,
110 110 # Redmine needs to parse revisions per branch.
111 111 # Branch "last_scmid" is for this requirement.
112 112 #
113 113 # In Git and Mercurial, revisions are not in date order.
114 114 # Redmine Mercurial fixed issues.
115 115 # * Redmine Takes Too Long On Large Mercurial Repository
116 116 # http://www.redmine.org/issues/3449
117 117 # * Sorting for changesets might go wrong on Mercurial repos
118 118 # http://www.redmine.org/issues/3567
119 119 #
120 120 # Database revision column is text, so Redmine can not sort by revision.
121 121 # Mercurial has revision number, and revision number guarantees revision order.
122 122 # Redmine Mercurial model stored revisions ordered by database id to database.
123 123 # So, Redmine Mercurial model can use correct ordering revisions.
124 124 #
125 125 # Redmine Mercurial adapter uses "hg log -r 0:tip --limit 10"
126 126 # to get limited revisions from old to new.
127 127 # But, Git 1.7.3.4 does not support --reverse with -n or --skip.
128 128 #
129 129 # The repository can still be fully reloaded by calling #clear_changesets
130 130 # before fetching changesets (eg. for offline resync)
131 131 def fetch_changesets
132 132 scm_brs = branches
133 133 return if scm_brs.nil? || scm_brs.empty?
134 134 h1 = extra_info || {}
135 135 h = h1.dup
136 136 h["branches"] ||= {}
137 137 h["db_consistent"] ||= {}
138 138 if changesets.count == 0
139 139 h["db_consistent"]["ordering"] = 1
140 140 merge_extra_info(h)
141 141 self.save
142 142 elsif ! h["db_consistent"].has_key?("ordering")
143 143 h["db_consistent"]["ordering"] = 0
144 144 merge_extra_info(h)
145 145 self.save
146 146 end
147 147 save_revisions(h, scm_brs)
148 148 end
149 149
150 150 def save_revisions(h, scm_brs)
151 # Remember what revisions we already processed (in any branches)
152 all_revisions = []
151 153 scm_brs.each do |br1|
152 154 br = br1.to_s
155 last_revision = nil
153 156 from_scmid = nil
154 157 from_scmid = h["branches"][br]["last_scmid"] if h["branches"][br]
155 158 h["branches"][br] ||= {}
156 begin
157 cnt = 0
158 last_rev_scmid = nil
159 scm.revisions('', from_scmid, br, {:reverse => true}) do |rev|
160 cnt += 1
161 db_rev = find_changeset_by_name(rev.revision)
162 if db_rev.nil?
163 transaction do
164 db_saved_rev = save_revision(rev)
165 parents = {}
166 parents[db_saved_rev] = rev.parents unless rev.parents.nil?
167 parents.each do |ch, chparents|
168 ch.parents = chparents.collect{|rp| find_changeset_by_name(rp)}.compact
169 end
170 end
171 end
172 last_rev_scmid = rev.scmid
173 if cnt > 100
174 cnt = 0
175 h["branches"][br]["last_scmid"] = last_rev_scmid
176 merge_extra_info(h)
177 self.save
159
160 revisions = scm.revisions('', from_scmid, br, {:reverse => true})
161 next if revisions.blank?
162
163 # Remember the last commit id here, before we start removing revisions from the array.
164 # We'll do that for optimization, but it also means, that we may lose even all revisions.
165 last_revision = revisions.last
166
167 # remove revisions that we have already processed (possibly in other branches)
168 revisions.reject!{|r| all_revisions.include?(r.scmid)}
169 # add revisions that we are to parse now to 'all processed revisions'
170 # (this equals to a union, because we executed diff above)
171 all_revisions += revisions.map{|r| r.scmid}
172
173 # Make the search for existing revisions in the database in a more sufficient manner
174 # This is replacing the one-after-one queries.
175 # Find all revisions, that are in the database, and then remove them from the revision array.
176 # Then later we won't need any conditions for db existence.
177 # Query for several revisions at once, and remove them from the revisions array, if they are there.
178 # Do this in chunks, to avoid eventual memory problems (in case of tens of thousands of commits).
179 # If there are no revisions (because the original code's algoritm filtered them),
180 # then this part will be stepped over.
181 # We make queries, just if there is any revision.
182 limit = 100
183 offset = 0
184 revisions_copy = revisions.clone # revisions will change
185 while offset < revisions_copy.size
186 recent_changesets_slice = changesets.find(
187 :all,
188 :conditions => [
189 'scmid IN (?)',
190 revisions_copy.slice(offset, limit).map{|x| x.scmid}
191 ]
192 )
193 # Subtract revisions that redmine already knows about
194 recent_revisions = recent_changesets_slice.map{|c| c.scmid}
195 revisions.reject!{|r| recent_revisions.include?(r.scmid)}
196 offset += limit
197 end
198
199 revisions.each do |rev|
200 transaction do
201 # There is no search in the db for this revision, because above we ensured,
202 # that it's not in the db.
203 db_saved_rev = save_revision(rev)
204 parents = {}
205 parents[db_saved_rev] = rev.parents unless rev.parents.nil?
206 parents.each do |ch, chparents|
207 ch.parents = chparents.collect{|rp| find_changeset_by_name(rp)}.compact
178 208 end
209 # saving the last scmid was moved from here, because we won't come in here,
210 # if the revision was already added for another branch
179 211 end
180 unless last_rev_scmid.nil?
181 h["branches"][br]["last_scmid"] = last_rev_scmid
182 merge_extra_info(h)
183 self.save
184 end
185 rescue Redmine::Scm::Adapters::CommandFailed => e
186 logger.error("save revisions error: #{e.message}")
212 end
213
214 # save the data about the last revision for this branch
215 unless last_revision.nil?
216 h["branches"][br]["last_scmid"] = last_revision.scmid
217 merge_extra_info(h)
218 self.save
187 219 end
188 220 end
189 221 end
190 222 private :save_revisions
191 223
192 224 def save_revision(rev)
193 225 changeset = Changeset.new(
194 226 :repository => self,
195 227 :revision => rev.identifier,
196 228 :scmid => rev.scmid,
197 229 :committer => rev.author,
198 230 :committed_on => rev.time,
199 231 :comments => rev.message
200 232 )
201 233 if changeset.save
202 234 rev.paths.each do |file|
203 235 Change.create(
204 236 :changeset => changeset,
205 237 :action => file[:action],
206 238 :path => file[:path])
207 239 end
208 240 end
209 241 changeset
210 242 end
211 243 private :save_revision
212 244
213 245 def heads_from_branches_hash
214 246 h1 = extra_info || {}
215 247 h = h1.dup
216 248 h["branches"] ||= {}
217 249 h['branches'].map{|br, hs| hs['last_scmid']}
218 250 end
219 251
220 252 def latest_changesets(path,rev,limit=10)
221 253 revisions = scm.revisions(path, nil, rev, :limit => limit, :all => false)
222 254 return [] if revisions.nil? || revisions.empty?
223 255
224 256 changesets.find(
225 257 :all,
226 258 :conditions => [
227 259 "scmid IN (?)",
228 260 revisions.map!{|c| c.scmid}
229 261 ],
230 262 :order => 'committed_on DESC'
231 263 )
232 264 end
233 265 end
General Comments 0
You need to be logged in to leave comments. Login now