jro_apps/redmine Commit - r3280:3c20a9b0acd5

Do not parse the entire git log to fetch new commits (takes several minutes for a few thousands commits), but only 1 week before the last known commit (#4547, #4716)....

Jean-Philippe Lang -

r3280:3c20a9b0acd5

parent child

Context file:

r3280:3c20a9b0acd5

Collapse all files

app/models/repository/git.rb +12 -9

             # redMine - project management software
             # Copyright (C) 2006-2007  Jean-Philippe Lang
             # Copyright (C) 2007  Patrick Aljord patcito@ŋmail.com
             # This program is free software; you can redistribute it and/or
             # modify it under the terms of the GNU General Public License
             # as published by the Free Software Foundation; either version 2
             # of the License, or (at your option) any later version.
             #
             # This program is distributed in the hope that it will be useful,
             # but WITHOUT ANY WARRANTY; without even the implied warranty of
             # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
             # GNU General Public License for more details.
             #
             # You should have received a copy of the GNU General Public License
             # along with this program; if not, write to the Free Software
             # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
             require 'redmine/scm/adapters/git_adapter'
             class Repository::Git < Repository
               attr_protected :root_url
               validates_presence_of :url
               def scm_adapter
                 Redmine::Scm::Adapters::GitAdapter
               end
               def self.scm_name
                 'Git'
               end
               def branches
                 scm.branches
               end
               def tags
                 scm.tags
               end
               # With SCM's that have a sequential commit numbering, redmine is able to be
               # clever and only fetch changesets going forward from the most recent one
               # it knows about.  However, with git, you never know if people have merged
-              # commits into the middle of the repository history, so we always have to
+              # commits into the middle of the repository history, so we should parse
-              # parse the entire log.
+              # the entire log. Since it's way too slow for large repositories, we only
+              # parse 1 week before the last known commit.
+              # The repository can still be fully reloaded by calling #clear_changesets
+              # before fetching changesets (eg. for offline resync)
               def fetch_changesets
-                # Save ourselves an expensive operation if we're already up to date
+                c = changesets.find(:first, :order => 'committed_on DESC')
-                return if scm.num_revisions == changesets.count
+                since = (c ? c.committed_on - 7.days : nil)
-                revisions = scm.revisions('', nil, nil, :all => true)
+                revisions = scm.revisions('', nil, nil, :all => true, :since => since)
                 return if revisions.nil? || revisions.empty?
-                # Find revisions that redmine knows about already
+                recent_changesets = changesets.find(:all, :conditions => ['committed_on >= ?', since])
-                existing_revisions = changesets.find(:all).map!{|c| c.scmid}
                 # Clean out revisions that are no longer in git
-                Changeset.delete_all(["scmid NOT IN (?) AND repository_id = (?)", revisions.map{|r| r.scmid}, self.id])
+                recent_changesets.each {|c| c.destroy unless revisions.detect {|r| r.scmid.to_s == c.scmid.to_s }}
                 # Subtract revisions that redmine already knows about
-                revisions.reject!{|r| existing_revisions.include?(r.scmid)}
+                recent_revisions = recent_changesets.map{|c| c.scmid}
+                revisions.reject!{|r| recent_revisions.include?(r.scmid)}
                 # Save the remaining ones to the database
                 revisions.each{|r| r.save(self)} unless revisions.nil?
               end
               def latest_changesets(path,rev,limit=10)
                 revisions = scm.revisions(path, nil, rev, :limit => limit, :all => false)
                 return [] if revisions.nil? || revisions.empty?
                 changesets.find(
                   :all,
                   :conditions => [
                     "scmid IN (?)",
                     revisions.map!{|c| c.scmid}
                   ],
                   :order => 'committed_on DESC'
                 )
               end
             end

lib/redmine/scm/adapters/git_adapter.rb +1 -5

             # redMine - project management software
             # Copyright (C) 2006-2007  Jean-Philippe Lang
             #
             # This program is free software; you can redistribute it and/or
             # modify it under the terms of the GNU General Public License
             # as published by the Free Software Foundation; either version 2
             # of the License, or (at your option) any later version.
             #
             # This program is distributed in the hope that it will be useful,
             # but WITHOUT ANY WARRANTY; without even the implied warranty of
             # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
             # GNU General Public License for more details.
             #
             # You should have received a copy of the GNU General Public License
             # along with this program; if not, write to the Free Software
             # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
             require 'redmine/scm/adapters/abstract_adapter'
             module Redmine
               module Scm
                 module Adapters
                   class GitAdapter < AbstractAdapter
                     # Git executable name
                     GIT_BIN = "git"
                     def info
                       begin
                         Info.new(:root_url => url, :lastrev => lastrev('',nil))
                       rescue
                         nil
                       end
                     end
                     def branches
                       branches = []
                       cmd = "#{GIT_BIN} --git-dir #{target('')} branch"
                       shellout(cmd) do |io|
                         io.each_line do |line|
                           branches << line.match('\s*\*?\s*(.*)$')[1]
                         end
                       end
                       branches.sort!
                     end
                     def tags
                       tags = []
                       cmd = "#{GIT_BIN} --git-dir #{target('')} tag"
                       shellout(cmd) do |io|
                         io.readlines.sort!.map{|t| t.strip}
                       end
                     end
                     def default_branch
                       branches.include?('master') ? 'master' : branches.first
                     end
                     def entries(path=nil, identifier=nil)
                       path ||= ''
                       entries = Entries.new
                       cmd = "#{GIT_BIN} --git-dir #{target('')} ls-tree -l "
                       cmd << shell_quote("HEAD:" + path) if identifier.nil?
                       cmd << shell_quote(identifier + ":" + path) if identifier
                       shellout(cmd)  do |io|
                         io.each_line do |line|
                           e = line.chomp.to_s
                           if e =~ /^\d+\s+(\w+)\s+([0-9a-f]{40})\s+([0-9-]+)\s+(.+)$/
                             type = $1
                             sha = $2
                             size = $3
                             name = $4
                             full_path = path.empty? ? name : "#{path}/#{name}"
                             entries << Entry.new({:name => name,
                              :path => full_path,
                              :kind => (type == "tree") ? 'dir' : 'file',
                              :size => (type == "tree") ? nil : size,
                              :lastrev => lastrev(full_path,identifier)
                             }) unless entries.detect{|entry| entry.name == name}
                           end
                         end
                       end
                       return nil if $? && $?.exitstatus != 0
                       entries.sort_by_name
                     end
                     def lastrev(path,rev)
                       return nil if path.nil?
                       cmd = "#{GIT_BIN} --git-dir #{target('')} log --pretty=fuller --no-merges -n 1 "
                       cmd << " #{shell_quote rev} " if rev
                       cmd <<  "-- #{path} " unless path.empty?
                       shellout(cmd) do |io|
                         begin
                           id = io.gets.split[1]
                           author = io.gets.match('Author:\s+(.*)$')[1]
 .times { io.gets }
                           time = io.gets.match('CommitDate:\s+(.*)$')[1]
                           Revision.new({
                             :identifier => id,
                             :scmid => id,
                             :author => author,
                             :time => time,
                             :message => nil,
                             :paths => nil
                           })
                         rescue NoMethodError => e
                           logger.error("The revision '#{path}' has a wrong format")
                           return nil
                         end
                       end
                     end
-                    def num_revisions
-                      cmd = "#{GIT_BIN} --git-dir #{target('')} log --all --pretty=format:'' | wc -l"
-                      shellout(cmd) {|io| io.gets.chomp.to_i + 1}
-                    end
                     def revisions(path, identifier_from, identifier_to, options={})
                       revisions = Revisions.new
                       cmd = "#{GIT_BIN} --git-dir #{target('')} log --find-copies-harder --raw --date=iso --pretty=fuller"
                       cmd << " --reverse" if options[:reverse]
                       cmd << " --all" if options[:all]
                       cmd << " -n #{options[:limit]} " if options[:limit]
                       cmd << " #{shell_quote(identifier_from + '..')} " if identifier_from
                       cmd << " #{shell_quote identifier_to} " if identifier_to
+                      cmd << " --since=#{shell_quote(options[:since].strftime("%Y-%m-%d %H:%M:%S"))}" if options[:since]
                       cmd << " -- #{path}" if path && !path.empty?
                       shellout(cmd) do |io|
                         files=[]
                         changeset = {}
                         parsing_descr = 0  #0: not parsing desc or files, 1: parsing desc, 2: parsing files
                         revno = 1
                         io.each_line do |line|
                           if line =~ /^commit ([0-9a-f]{40})$/
                             key = "commit"
                             value = $1
                             if (parsing_descr == 1 || parsing_descr == 2)
                               parsing_descr = 0
                               revision = Revision.new({
                                 :identifier => changeset[:commit],
                                 :scmid => changeset[:commit],
                                 :author => changeset[:author],
                                 :time => Time.parse(changeset[:date]),
                                 :message => changeset[:description],
                                 :paths => files
                               })
                               if block_given?
                                 yield revision
                               else
                                 revisions << revision
                               end
                               changeset = {}
                               files = []
                               revno = revno + 1
                             end
                             changeset[:commit] = $1
                           elsif (parsing_descr == 0) && line =~ /^(\w+):\s*(.*)$/
                             key = $1
                             value = $2
                             if key == "Author"
                               changeset[:author] = value
                             elsif key == "CommitDate"
                               changeset[:date] = value
                             end
                           elsif (parsing_descr == 0) && line.chomp.to_s == ""
                             parsing_descr = 1
                             changeset[:description] = ""
                           elsif (parsing_descr == 1 || parsing_descr == 2) \
                           && line =~ /^:\d+\s+\d+\s+[0-9a-f.]+\s+[0-9a-f.]+\s+(\w)\s+(.+)$/
                             parsing_descr = 2
                             fileaction = $1
                             filepath = $2
                             files << {:action => fileaction, :path => filepath}
                           elsif (parsing_descr == 1 || parsing_descr == 2) \
                           && line =~ /^:\d+\s+\d+\s+[0-9a-f.]+\s+[0-9a-f.]+\s+(\w)\d+\s+(\S+)\s+(.+)$/
                             parsing_descr = 2
                             fileaction = $1
                             filepath = $3
                             files << {:action => fileaction, :path => filepath}
                           elsif (parsing_descr == 1) && line.chomp.to_s == ""
                             parsing_descr = 2
                           elsif (parsing_descr == 1)
                             changeset[:description] << line[4..-1]
                           end
                         end
                         if changeset[:commit]
                           revision = Revision.new({
                             :identifier => changeset[:commit],
                             :scmid => changeset[:commit],
                             :author => changeset[:author],
                             :time => Time.parse(changeset[:date]),
                             :message => changeset[:description],
                             :paths => files
                           })
                           if block_given?
                             yield revision
                           else
                             revisions << revision
                           end
                         end
                       end
                       return nil if $? && $?.exitstatus != 0
                       revisions
                     end
                     def diff(path, identifier_from, identifier_to=nil)
                       path ||= ''
                       if identifier_to
                         cmd = "#{GIT_BIN} --git-dir #{target('')} diff #{shell_quote identifier_to} #{shell_quote identifier_from}"
                       else
                         cmd = "#{GIT_BIN} --git-dir #{target('')} show #{shell_quote identifier_from}"
                       end
                       cmd << " -- #{shell_quote path}" unless path.empty?
                       diff = []
                       shellout(cmd) do |io|
                         io.each_line do |line|
                           diff << line
                         end
                       end
                       return nil if $? && $?.exitstatus != 0
                       diff
                     end
                     def annotate(path, identifier=nil)
                       identifier = 'HEAD' if identifier.blank?
                       cmd = "#{GIT_BIN} --git-dir #{target('')} blame -l #{shell_quote identifier} -- #{shell_quote path}"
                       blame = Annotate.new
                       content = nil
                       shellout(cmd) { |io| io.binmode; content = io.read }
                       return nil if $? && $?.exitstatus != 0
                       # git annotates binary files
                       return nil if content.is_binary_data?
                       content.split("\n").each do |line|
                         next unless line =~ /([0-9a-f]{39,40})\s\((\w*)[^\)]*\)(.*)/
                         blame.add_line($3.rstrip, Revision.new(:identifier => $1, :author => $2.strip))
                       end
                       blame
                     end
                     def cat(path, identifier=nil)
                       if identifier.nil?
                         identifier = 'HEAD'
                       end
                       cmd = "#{GIT_BIN} --git-dir #{target('')} show #{shell_quote(identifier + ':' + path)}"
                       cat = nil
                       shellout(cmd) do |io|
                         io.binmode
                         cat = io.read
                       end
                       return nil if $? && $?.exitstatus != 0
                       cat
                     end
                   end
                 end
               end
             end

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages