##// END OF EJS Templates
Reduces memory usage when importing large git repositories (#1482)....
Jean-Philippe Lang -
r1585:00593f2f345f
parent child
Show More
@@ -1,70 +1,68
1 1 # redMine - project management software
2 2 # Copyright (C) 2006-2007 Jean-Philippe Lang
3 3 # Copyright (C) 2007 Patrick Aljord patcito@ŋmail.com
4 4 # This program is free software; you can redistribute it and/or
5 5 # modify it under the terms of the GNU General Public License
6 6 # as published by the Free Software Foundation; either version 2
7 7 # of the License, or (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software
16 16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 17
18 18 require 'redmine/scm/adapters/git_adapter'
19 19
20 20 class Repository::Git < Repository
21 21 attr_protected :root_url
22 22 validates_presence_of :url
23 23
24 24 def scm_adapter
25 25 Redmine::Scm::Adapters::GitAdapter
26 26 end
27 27
28 28 def self.scm_name
29 29 'Git'
30 30 end
31 31
32 32 def changesets_for_path(path)
33 33 Change.find(:all, :include => :changeset,
34 34 :conditions => ["repository_id = ? AND path = ?", id, path],
35 35 :order => "committed_on DESC, #{Changeset.table_name}.revision DESC").collect(&:changeset)
36 36 end
37 37
38 38 def fetch_changesets
39 39 scm_info = scm.info
40 40 if scm_info
41 41 # latest revision found in database
42 42 db_revision = latest_changeset ? latest_changeset.revision : nil
43 43 # latest revision in the repository
44 44 scm_revision = scm_info.lastrev.scmid
45 45
46 46 unless changesets.find_by_scmid(scm_revision)
47
48 revisions = scm.revisions('', db_revision, nil)
47 scm.revisions('', db_revision, nil, :reverse => true) do |revision|
49 48 transaction do
50 revisions.reverse_each do |revision|
51 49 changeset = Changeset.create(:repository => self,
52 50 :revision => revision.identifier,
53 51 :scmid => revision.scmid,
54 52 :committer => revision.author,
55 53 :committed_on => revision.time,
56 54 :comments => revision.message)
57 55
58 56 revision.paths.each do |change|
59 57 Change.create(:changeset => changeset,
60 58 :action => change[:action],
61 59 :path => change[:path],
62 60 :from_path => change[:from_path],
63 61 :from_revision => change[:from_revision])
64 62 end
65 63 end
66 64 end
67 65 end
68 66 end
69 67 end
70 68 end
@@ -1,260 +1,271
1 1 # redMine - project management software
2 2 # Copyright (C) 2006-2007 Jean-Philippe Lang
3 3 #
4 4 # This program is free software; you can redistribute it and/or
5 5 # modify it under the terms of the GNU General Public License
6 6 # as published by the Free Software Foundation; either version 2
7 7 # of the License, or (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software
16 16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 17
18 18 require 'redmine/scm/adapters/abstract_adapter'
19 19
20 20 module Redmine
21 21 module Scm
22 22 module Adapters
23 23 class GitAdapter < AbstractAdapter
24 24
25 25 # Git executable name
26 26 GIT_BIN = "git"
27 27
28 28 # Get the revision of a particuliar file
29 29 def get_rev (rev,path)
30 30
31 31 if rev != 'latest' && !rev.nil?
32 32 cmd="#{GIT_BIN} --git-dir #{target('')} show #{shell_quote rev} -- #{shell_quote path}"
33 33 else
34 34 branch = shellout("#{GIT_BIN} --git-dir #{target('')} branch") { |io| io.grep(/\*/)[0].strip.match(/\* (.*)/)[1] }
35 35 cmd="#{GIT_BIN} --git-dir #{target('')} log -1 #{branch} -- #{shell_quote path}"
36 36 end
37 37 rev=[]
38 38 i=0
39 39 shellout(cmd) do |io|
40 40 files=[]
41 41 changeset = {}
42 42 parsing_descr = 0 #0: not parsing desc or files, 1: parsing desc, 2: parsing files
43 43
44 44 io.each_line do |line|
45 45 if line =~ /^commit ([0-9a-f]{40})$/
46 46 key = "commit"
47 47 value = $1
48 48 if (parsing_descr == 1 || parsing_descr == 2)
49 49 parsing_descr = 0
50 50 rev = Revision.new({:identifier => changeset[:commit],
51 51 :scmid => changeset[:commit],
52 52 :author => changeset[:author],
53 53 :time => Time.parse(changeset[:date]),
54 54 :message => changeset[:description],
55 55 :paths => files
56 56 })
57 57 changeset = {}
58 58 files = []
59 59 end
60 60 changeset[:commit] = $1
61 61 elsif (parsing_descr == 0) && line =~ /^(\w+):\s*(.*)$/
62 62 key = $1
63 63 value = $2
64 64 if key == "Author"
65 65 changeset[:author] = value
66 66 elsif key == "Date"
67 67 changeset[:date] = value
68 68 end
69 69 elsif (parsing_descr == 0) && line.chomp.to_s == ""
70 70 parsing_descr = 1
71 71 changeset[:description] = ""
72 72 elsif (parsing_descr == 1 || parsing_descr == 2) && line =~ /^:\d+\s+\d+\s+[0-9a-f.]+\s+[0-9a-f.]+\s+(\w)\s+(.+)$/
73 73 parsing_descr = 2
74 74 fileaction = $1
75 75 filepath = $2
76 76 files << {:action => fileaction, :path => filepath}
77 77 elsif (parsing_descr == 1) && line.chomp.to_s == ""
78 78 parsing_descr = 2
79 79 elsif (parsing_descr == 1)
80 80 changeset[:description] << line
81 81 end
82 82 end
83 83 rev = Revision.new({:identifier => changeset[:commit],
84 84 :scmid => changeset[:commit],
85 85 :author => changeset[:author],
86 86 :time => (changeset[:date] ? Time.parse(changeset[:date]) : nil),
87 87 :message => changeset[:description],
88 88 :paths => files
89 89 })
90 90
91 91 end
92 92
93 93 get_rev('latest',path) if rev == []
94 94
95 95 return nil if $? && $?.exitstatus != 0
96 96 return rev
97 97 end
98 98
99 99
100 100 def info
101 101 revs = revisions(url,nil,nil,{:limit => 1})
102 102 if revs && revs.any?
103 103 Info.new(:root_url => url, :lastrev => revs.first)
104 104 else
105 105 nil
106 106 end
107 107 rescue Errno::ENOENT => e
108 108 return nil
109 109 end
110 110
111 111 def entries(path=nil, identifier=nil)
112 112 path ||= ''
113 113 entries = Entries.new
114 114 cmd = "#{GIT_BIN} --git-dir #{target('')} ls-tree -l "
115 115 cmd << shell_quote("HEAD:" + path) if identifier.nil?
116 116 cmd << shell_quote(identifier + ":" + path) if identifier
117 117 shellout(cmd) do |io|
118 118 io.each_line do |line|
119 119 e = line.chomp.to_s
120 120 if e =~ /^\d+\s+(\w+)\s+([0-9a-f]{40})\s+([0-9-]+)\s+(.+)$/
121 121 type = $1
122 122 sha = $2
123 123 size = $3
124 124 name = $4
125 125 entries << Entry.new({:name => name,
126 126 :path => (path.empty? ? name : "#{path}/#{name}"),
127 127 :kind => ((type == "tree") ? 'dir' : 'file'),
128 128 :size => ((type == "tree") ? nil : size),
129 129 :lastrev => get_rev(identifier,(path.empty? ? name : "#{path}/#{name}"))
130 130
131 131 }) unless entries.detect{|entry| entry.name == name}
132 132 end
133 133 end
134 134 end
135 135 return nil if $? && $?.exitstatus != 0
136 136 entries.sort_by_name
137 137 end
138 138
139 139 def revisions(path, identifier_from, identifier_to, options={})
140 140 revisions = Revisions.new
141 141 cmd = "#{GIT_BIN} --git-dir #{target('')} log --raw "
142 cmd << " --reverse" if options[:reverse]
142 143 cmd << " -n #{options[:limit].to_i} " if (!options.nil?) && options[:limit]
143 144 cmd << " #{shell_quote(identifier_from + '..')} " if identifier_from
144 145 cmd << " #{shell_quote identifier_to} " if identifier_to
145 #cmd << " HEAD " if !identifier_to
146 146 shellout(cmd) do |io|
147 147 files=[]
148 148 changeset = {}
149 149 parsing_descr = 0 #0: not parsing desc or files, 1: parsing desc, 2: parsing files
150 150 revno = 1
151 151
152 152 io.each_line do |line|
153 153 if line =~ /^commit ([0-9a-f]{40})$/
154 154 key = "commit"
155 155 value = $1
156 156 if (parsing_descr == 1 || parsing_descr == 2)
157 157 parsing_descr = 0
158 revisions << Revision.new({:identifier => changeset[:commit],
158 revision = Revision.new({:identifier => changeset[:commit],
159 159 :scmid => changeset[:commit],
160 160 :author => changeset[:author],
161 161 :time => Time.parse(changeset[:date]),
162 162 :message => changeset[:description],
163 163 :paths => files
164 164 })
165 if block_given?
166 yield revision
167 else
168 revisions << revision
169 end
165 170 changeset = {}
166 171 files = []
167 172 revno = revno + 1
168 173 end
169 174 changeset[:commit] = $1
170 175 elsif (parsing_descr == 0) && line =~ /^(\w+):\s*(.*)$/
171 176 key = $1
172 177 value = $2
173 178 if key == "Author"
174 179 changeset[:author] = value
175 180 elsif key == "Date"
176 181 changeset[:date] = value
177 182 end
178 183 elsif (parsing_descr == 0) && line.chomp.to_s == ""
179 184 parsing_descr = 1
180 185 changeset[:description] = ""
181 186 elsif (parsing_descr == 1 || parsing_descr == 2) && line =~ /^:\d+\s+\d+\s+[0-9a-f.]+\s+[0-9a-f.]+\s+(\w)\s+(.+)$/
182 187 parsing_descr = 2
183 188 fileaction = $1
184 189 filepath = $2
185 190 files << {:action => fileaction, :path => filepath}
186 191 elsif (parsing_descr == 1) && line.chomp.to_s == ""
187 192 parsing_descr = 2
188 193 elsif (parsing_descr == 1)
189 194 changeset[:description] << line[4..-1]
190 195 end
191 196 end
192 197
193 revisions << Revision.new({:identifier => changeset[:commit],
198 if changeset[:commit]
199 revision = Revision.new({:identifier => changeset[:commit],
194 200 :scmid => changeset[:commit],
195 201 :author => changeset[:author],
196 202 :time => Time.parse(changeset[:date]),
197 203 :message => changeset[:description],
198 204 :paths => files
199 }) if changeset[:commit]
200
205 })
206 if block_given?
207 yield revision
208 else
209 revisions << revision
210 end
211 end
201 212 end
202 213
203 214 return nil if $? && $?.exitstatus != 0
204 215 revisions
205 216 end
206 217
207 218 def diff(path, identifier_from, identifier_to=nil)
208 219 path ||= ''
209 220 if !identifier_to
210 221 identifier_to = nil
211 222 end
212 223
213 224 cmd = "#{GIT_BIN} --git-dir #{target('')} show #{shell_quote identifier_from}" if identifier_to.nil?
214 225 cmd = "#{GIT_BIN} --git-dir #{target('')} diff #{shell_quote identifier_to} #{shell_quote identifier_from}" if !identifier_to.nil?
215 226 cmd << " -- #{shell_quote path}" unless path.empty?
216 227 diff = []
217 228 shellout(cmd) do |io|
218 229 io.each_line do |line|
219 230 diff << line
220 231 end
221 232 end
222 233 return nil if $? && $?.exitstatus != 0
223 234 diff
224 235 end
225 236
226 237 def annotate(path, identifier=nil)
227 238 identifier = 'HEAD' if identifier.blank?
228 239 cmd = "#{GIT_BIN} --git-dir #{target('')} blame -l #{shell_quote identifier} -- #{shell_quote path}"
229 240 blame = Annotate.new
230 241 content = nil
231 242 shellout(cmd) { |io| io.binmode; content = io.read }
232 243 return nil if $? && $?.exitstatus != 0
233 244 # git annotates binary files
234 245 return nil if content.is_binary_data?
235 246 content.split("\n").each do |line|
236 247 next unless line =~ /([0-9a-f]{39,40})\s\((\w*)[^\)]*\)(.*)/
237 248 blame.add_line($3.rstrip, Revision.new(:identifier => $1, :author => $2.strip))
238 249 end
239 250 blame
240 251 end
241 252
242 253 def cat(path, identifier=nil)
243 254 if identifier.nil?
244 255 identifier = 'HEAD'
245 256 end
246 257 cmd = "#{GIT_BIN} --git-dir #{target('')} show #{shell_quote(identifier + ':' + path)}"
247 258 cat = nil
248 259 shellout(cmd) do |io|
249 260 io.binmode
250 261 cat = io.read
251 262 end
252 263 return nil if $? && $?.exitstatus != 0
253 264 cat
254 265 end
255 266 end
256 267 end
257 268 end
258 269
259 270 end
260 271
General Comments 0
You need to be logged in to leave comments. Login now