jro_apps/redmine Commit - r3352:46aa855572a2

Remove invalid utf8 sequences from commit comments and author name (#4773)....

Jean-Philippe Lang -

r3352:46aa855572a2

parent child

Context file:

r3352:46aa855572a2

Collapse all files

test/fixtures/encoding/iso-8859-1.txt created 644 +1 0

	@@ -0,0 +1,1
				1	Texte encod� en ISO-8859-1. No newline at end of file

app/models/changeset.rb +8 -3

             # Redmine - project management software
-            # Copyright (C) 2006-2008  Jean-Philippe Lang
+            # Copyright (C) 2006-2010  Jean-Philippe Lang
             #
             # This program is free software; you can redistribute it and/or
             # modify it under the terms of the GNU General Public License
             # as published by the Free Software Foundation; either version 2
             # of the License, or (at your option) any later version.
             #
             # This program is distributed in the hope that it will be useful,
             # but WITHOUT ANY WARRANTY; without even the implied warranty of
             # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
             # GNU General Public License for more details.
             #
             # You should have received a copy of the GNU General Public License
             # along with this program; if not, write to the Free Software
             # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
             require 'iconv'
             class Changeset < ActiveRecord::Base
               belongs_to :repository
               belongs_to :user
               has_many :changes, :dependent => :delete_all
               has_and_belongs_to_many :issues
               acts_as_event :title => Proc.new {|o| "#{l(:label_revision)} #{o.revision}" + (o.short_comments.blank? ? '' : (': ' + o.short_comments))},
                             :description => :long_comments,
                             :datetime => :committed_on,
                             :url => Proc.new {|o| {:controller => 'repositories', :action => 'revision', :id => o.repository.project, :rev => o.revision}}
               acts_as_searchable :columns => 'comments',
                                  :include => {:repository => :project},
                                  :project_key => "#{Repository.table_name}.project_id",
                                  :date_column => 'committed_on'
               acts_as_activity_provider :timestamp => "#{table_name}.committed_on",
                                         :author_key => :user_id,
                                         :find_options => {:include => [:user, {:repository => :project}]}
               validates_presence_of :repository_id, :revision, :committed_on, :commit_date
               validates_uniqueness_of :revision, :scope => :repository_id
               validates_uniqueness_of :scmid, :scope => :repository_id, :allow_nil => true
               named_scope :visible, lambda {|*args| { :include => {:repository => :project},
                                                       :conditions => Project.allowed_to_condition(args.first || User.current, :view_changesets) } }
               def revision=(r)
                 write_attribute :revision, (r.nil? ? nil : r.to_s)
               end
               def comments=(comment)
                 write_attribute(:comments, Changeset.normalize_comments(comment))
               end
               def committed_on=(date)
                 self.commit_date = date
                 super
               end
+              def committer=(arg)
+                write_attribute(:committer, self.class.to_utf8(arg.to_s))
+              end
               def project
                 repository.project
               end
               def author
                 user || committer.to_s.split('<').first
               end
               def before_create
                 self.user = repository.find_committer_user(committer)
               end
               def after_create
                 scan_comment_for_issue_ids
               end
               require 'pp'
               def scan_comment_for_issue_ids
                 return if comments.blank?
                 # keywords used to reference issues
                 ref_keywords = Setting.commit_ref_keywords.downcase.split(",").collect(&:strip)
                 # keywords used to fix issues
                 fix_keywords = Setting.commit_fix_keywords.downcase.split(",").collect(&:strip)
                 # status and optional done ratio applied
                 fix_status = IssueStatus.find_by_id(Setting.commit_fix_status_id)
                 done_ratio = Setting.commit_fix_done_ratio.blank? ? nil : Setting.commit_fix_done_ratio.to_i
                 kw_regexp = (ref_keywords + fix_keywords).collect{|kw| Regexp.escape(kw)}.join("|")
                 return if kw_regexp.blank?
                 referenced_issues = []
                 if ref_keywords.delete('*')
                   # find any issue ID in the comments
                   target_issue_ids = []
                   comments.scan(%r{([\s\(\[,-]|^)#(\d+)(?=[[:punct:]]|\s|<|$)}).each { |m| target_issue_ids << m[1] }
                   referenced_issues += find_referenced_issues_by_id(target_issue_ids)
                 end
                 comments.scan(Regexp.new("(#{kw_regexp})[\s:]+(([\s,;&]*#?\\d+)+)", Regexp::IGNORECASE)).each do |match|
                   action = match[0]
                   target_issue_ids = match[1].scan(/\d+/)
                   target_issues = find_referenced_issues_by_id(target_issue_ids)
                   if fix_status && fix_keywords.include?(action.downcase)
                     # update status of issues
                     logger.debug "Issues fixed by changeset #{self.revision}: #{issue_ids.join(', ')}." if logger && logger.debug?
                     target_issues.each do |issue|
                       # the issue may have been updated by the closure of another one (eg. duplicate)
                       issue.reload
                       # don't change the status is the issue is closed
                       next if issue.status.is_closed?
                       csettext = "r#{self.revision}"
                       if self.scmid && (! (csettext =~ /^r[0-9]+$/))
                         csettext = "commit:\"#{self.scmid}\""
                       end
                       journal = issue.init_journal(user || User.anonymous, ll(Setting.default_language, :text_status_changed_by_changeset, csettext))
                       issue.status = fix_status
                       issue.done_ratio = done_ratio if done_ratio
                       Redmine::Hook.call_hook(:model_changeset_scan_commit_for_issue_ids_pre_issue_update,
                                               { :changeset => self, :issue => issue })
                       issue.save
                     end
                   end
                   referenced_issues += target_issues
                 end
                 self.issues = referenced_issues.uniq
               end
               def short_comments
                 @short_comments || split_comments.first
               end
               def long_comments
                 @long_comments || split_comments.last
               end
               # Returns the previous changeset
               def previous
                 @previous ||= Changeset.find(:first, :conditions => ['id < ? AND repository_id = ?', self.id, self.repository_id], :order => 'id DESC')
               end
               # Returns the next changeset
               def next
                 @next ||= Changeset.find(:first, :conditions => ['id > ? AND repository_id = ?', self.id, self.repository_id], :order => 'id ASC')
               end
               # Strips and reencodes a commit log before insertion into the database
               def self.normalize_comments(str)
                 to_utf8(str.to_s.strip)
               end
               # Creates a new Change from it's common parameters
               def create_change(change)
                 Change.create(:changeset => self,
                               :action => change[:action],
                               :path => change[:path],
                               :from_path => change[:from_path],
                               :from_revision => change[:from_revision])
               end
               private
               # Finds issues that can be referenced by the commit message
               # i.e. issues that belong to the repository project, a subproject or a parent project
               def find_referenced_issues_by_id(ids)
                 Issue.find_all_by_id(ids, :include => :project).select {|issue|
                   project == issue.project || project.is_ancestor_of?(issue.project) || project.is_descendant_of?(issue.project)
                 }
               end
               def split_comments
                 comments =~ /\A(.+?)\r?\n(.*)$/m
                 @short_comments = $1 || comments
                 @long_comments = $2.to_s.strip
                 return @short_comments, @long_comments
               end
               def self.to_utf8(str)
                 return str if /\A[\r\n\t\x20-\x7e]*\Z/n.match(str) # for us-ascii
                 encoding = Setting.commit_logs_encoding.to_s.strip
                 unless encoding.blank? || encoding == 'UTF-8'
                   begin
-                    return Iconv.conv('UTF-8', encoding, str)
+                    str = Iconv.conv('UTF-8', encoding, str)
                   rescue Iconv::Failure
                     # do nothing here
                   end
                 end
-                str
+                # removes invalid UTF8 sequences
+                Iconv.conv('UTF-8//IGNORE', 'UTF-8', str + '  ')[0..-3]
               end
             end

test/unit/changeset_test.rb +18 -2

-            # redMine - project management software
+            # encoding: utf-8
-            # Copyright (C) 2006-2007  Jean-Philippe Lang
+            # Redmine - project management software
+            # Copyright (C) 2006-2010  Jean-Philippe Lang
             #
             # This program is free software; you can redistribute it and/or
             # modify it under the terms of the GNU General Public License
             # as published by the Free Software Foundation; either version 2
             # of the License, or (at your option) any later version.
             #
             # This program is distributed in the hope that it will be useful,
             # but WITHOUT ANY WARRANTY; without even the implied warranty of
             # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
             # GNU General Public License for more details.
             #
             # You should have received a copy of the GNU General Public License
             # along with this program; if not, write to the Free Software
             # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
             require File.dirname(__FILE__) + '/../test_helper'
             class ChangesetTest < ActiveSupport::TestCase
               fixtures :projects, :repositories, :issues, :issue_statuses, :changesets, :changes, :issue_categories, :enumerations, :custom_fields, :custom_values, :users, :members, :member_roles, :trackers
               def setup
               end
               def test_ref_keywords_any
                 ActionMailer::Base.deliveries.clear
                 Setting.commit_fix_status_id = IssueStatus.find(:first, :conditions => ["is_closed = ?", true]).id
                 Setting.commit_fix_done_ratio = '90'
                 Setting.commit_ref_keywords = '*'
                 Setting.commit_fix_keywords = 'fixes , closes'
                 c = Changeset.new(:repository => Project.find(1).repository,
                                   :committed_on => Time.now,
                                   :comments => 'New commit (#2). Fixes #1')
                 c.scan_comment_for_issue_ids
                 assert_equal [1, 2], c.issue_ids.sort
                 fixed = Issue.find(1)
                 assert fixed.closed?
                 assert_equal 90, fixed.done_ratio
                 assert_equal 1, ActionMailer::Base.deliveries.size
               end
               def test_ref_keywords_any_line_start
                 Setting.commit_ref_keywords = '*'
                 c = Changeset.new(:repository => Project.find(1).repository,
                                   :committed_on => Time.now,
                                   :comments => '#1 is the reason of this commit')
                 c.scan_comment_for_issue_ids
                 assert_equal [1], c.issue_ids.sort
               end
               def test_ref_keywords_allow_brackets_around_a_issue_number
                 Setting.commit_ref_keywords = '*'
                 c = Changeset.new(:repository => Project.find(1).repository,
                                   :committed_on => Time.now,
                                   :comments => '[#1] Worked on this issue')
                 c.scan_comment_for_issue_ids
                 assert_equal [1], c.issue_ids.sort
               end
               def test_ref_keywords_allow_brackets_around_multiple_issue_numbers
                 Setting.commit_ref_keywords = '*'
                 c = Changeset.new(:repository => Project.find(1).repository,
                                   :committed_on => Time.now,
                                   :comments => '[#1 #2, #3] Worked on these')
                 c.scan_comment_for_issue_ids
                 assert_equal [1,2,3], c.issue_ids.sort
               end
               def test_commit_referencing_a_subproject_issue
                 c = Changeset.new(:repository => Project.find(1).repository,
                                   :committed_on => Time.now,
                                   :comments => 'refs #5, a subproject issue')
                 c.scan_comment_for_issue_ids
                 assert_equal [5], c.issue_ids.sort
                 assert c.issues.first.project != c.project
               end
               def test_commit_referencing_a_parent_project_issue
                 # repository of child project
                 r = Repository::Subversion.create!(:project => Project.find(3), :url => 'svn://localhost/test')
                 c = Changeset.new(:repository => r,
                                   :committed_on => Time.now,
                                   :comments => 'refs #2, an issue of a parent project')
                 c.scan_comment_for_issue_ids
                 assert_equal [2], c.issue_ids.sort
                 assert c.issues.first.project != c.project
               end
               def test_previous
                 changeset = Changeset.find_by_revision('3')
                 assert_equal Changeset.find_by_revision('2'), changeset.previous
               end
               def test_previous_nil
                 changeset = Changeset.find_by_revision('1')
                 assert_nil changeset.previous
               end
               def test_next
                 changeset = Changeset.find_by_revision('2')
                 assert_equal Changeset.find_by_revision('3'), changeset.next
               end
               def test_next_nil
                 changeset = Changeset.find_by_revision('10')
                 assert_nil changeset.next
               end
+              def test_comments_should_be_converted_to_utf8
+                with_settings :commit_logs_encoding => 'ISO-8859-1' do
+                  c = Changeset.new
+                  c.comments = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
+                  assert_equal "Texte encodé en ISO-8859-1.", c.comments
+                end
+              end
+              def test_invalid_utf8_sequences_in_comments_should_be_stripped
+                c = Changeset.new
+                c.comments = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
+                assert_equal "Texte encod en ISO-8859-1.", c.comments
+              end
             end

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages