jro_apps/redmine Commit - r5253:6536c53e0973

scm: replace invalid utf-8 sequences in comments instead of stripping on Ruby 1.8....

Toshi MARUYAMA -

r5253:6536c53e0973

parent child

Context file:

r5253:6536c53e0973

Collapse all files

app/models/changeset.rb +12 -13

             # Redmine - project management software
             # Copyright (C) 2006-2010  Jean-Philippe Lang
             #
             # This program is free software; you can redistribute it and/or
             # modify it under the terms of the GNU General Public License
             # as published by the Free Software Foundation; either version 2
             # of the License, or (at your option) any later version.
             #
             # This program is distributed in the hope that it will be useful,
             # but WITHOUT ANY WARRANTY; without even the implied warranty of
             # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
             # GNU General Public License for more details.
             #
             # You should have received a copy of the GNU General Public License
             # along with this program; if not, write to the Free Software
             # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
             require 'iconv'
             class Changeset < ActiveRecord::Base
               belongs_to :repository
               belongs_to :user
               has_many :changes, :dependent => :delete_all
               has_and_belongs_to_many :issues
               acts_as_event :title => Proc.new {|o| "#{l(:label_revision)} #{o.format_identifier}" + (o.short_comments.blank? ? '' : (': ' + o.short_comments))},
                             :description => :long_comments,
                             :datetime => :committed_on,
                             :url => Proc.new {|o| {:controller => 'repositories', :action => 'revision', :id => o.repository.project, :rev => o.identifier}}
               acts_as_searchable :columns => 'comments',
                                  :include => {:repository => :project},
                                  :project_key => "#{Repository.table_name}.project_id",
                                  :date_column => 'committed_on'
               acts_as_activity_provider :timestamp => "#{table_name}.committed_on",
                                         :author_key => :user_id,
                                         :find_options => {:include => [:user, {:repository => :project}]}
               validates_presence_of :repository_id, :revision, :committed_on, :commit_date
               validates_uniqueness_of :revision, :scope => :repository_id
               validates_uniqueness_of :scmid, :scope => :repository_id, :allow_nil => true
               named_scope :visible, lambda {|*args| { :include => {:repository => :project},
                                                       :conditions => Project.allowed_to_condition(args.shift || User.current, :view_changesets, *args) } }
               def revision=(r)
                 write_attribute :revision, (r.nil? ? nil : r.to_s)
               end
               # Returns the identifier of this changeset; depending on repository backends
               def identifier
                 if repository.class.respond_to? :changeset_identifier
                   repository.class.changeset_identifier self
                 else
                   revision.to_s
                 end
               end
               def committed_on=(date)
                 self.commit_date = date
                 super
               end
               # Returns the readable identifier
               def format_identifier
                 if repository.class.respond_to? :format_changeset_identifier
                   repository.class.format_changeset_identifier self
                 else
                   identifier
                 end
               end
               def project
                 repository.project
               end
               def author
                 user || committer.to_s.split('<').first
               end
               def before_create
                 self.committer = self.class.to_utf8(self.committer, repository.repo_log_encoding)
                 self.comments  = self.class.normalize_comments(
                                    self.comments, repository.repo_log_encoding)
                 self.user = repository.find_committer_user(self.committer)
               end
               def after_create
                 scan_comment_for_issue_ids
               end
               TIMELOG_RE = /
                 (
                 ((\d+)(h|hours?))((\d+)(m|min)?)?
                 |
                 ((\d+)(h|hours?|m|min))
                 |
                 (\d+):(\d+)
                 |
                 (\d+([\.,]\d+)?)h?
                 )
                 /x
               def scan_comment_for_issue_ids
                 return if comments.blank?
                 # keywords used to reference issues
                 ref_keywords = Setting.commit_ref_keywords.downcase.split(",").collect(&:strip)
                 ref_keywords_any = ref_keywords.delete('*')
                 # keywords used to fix issues
                 fix_keywords = Setting.commit_fix_keywords.downcase.split(",").collect(&:strip)
                 kw_regexp = (ref_keywords + fix_keywords).collect{|kw| Regexp.escape(kw)}.join("|")
                 referenced_issues = []
                 comments.scan(/([\s\(\[,-]|^)((#{kw_regexp})[\s:]+)?(#\d+(\s+@#{TIMELOG_RE})?([\s,;&]+#\d+(\s+@#{TIMELOG_RE})?)*)(?=[[:punct:]]|\s|<|$)/i) do |match|
                   action, refs = match[2], match[3]
                   next unless action.present? || ref_keywords_any
                   refs.scan(/#(\d+)(\s+@#{TIMELOG_RE})?/).each do |m|
                     issue, hours = find_referenced_issue_by_id(m[0].to_i), m[2]
                     if issue
                       referenced_issues << issue
                       fix_issue(issue) if fix_keywords.include?(action.to_s.downcase)
                       log_time(issue, hours) if hours && Setting.commit_logtime_enabled?
                     end
                   end
                 end
                 referenced_issues.uniq!
                 self.issues = referenced_issues unless referenced_issues.empty?
               end
               def short_comments
                 @short_comments || split_comments.first
               end
               def long_comments
                 @long_comments || split_comments.last
               end
               def text_tag
                 if scmid?
                   "commit:#{scmid}"
                 else
                   "r#{revision}"
                 end
               end
               # Returns the previous changeset
               def previous
                 @previous ||= Changeset.find(:first, :conditions => ['id < ? AND repository_id = ?', self.id, self.repository_id], :order => 'id DESC')
               end
               # Returns the next changeset
               def next
                 @next ||= Changeset.find(:first, :conditions => ['id > ? AND repository_id = ?', self.id, self.repository_id], :order => 'id ASC')
               end
               # Creates a new Change from it's common parameters
               def create_change(change)
                 Change.create(:changeset => self,
                               :action => change[:action],
                               :path => change[:path],
                               :from_path => change[:from_path],
                               :from_revision => change[:from_revision])
               end
               private
               # Finds an issue that can be referenced by the commit message
               # i.e. an issue that belong to the repository project, a subproject or a parent project
               def find_referenced_issue_by_id(id)
                 return nil if id.blank?
                 issue = Issue.find_by_id(id.to_i, :include => :project)
                 if issue
                   unless issue.project &&
                             (project == issue.project || project.is_ancestor_of?(issue.project) ||
                              project.is_descendant_of?(issue.project))
                     issue = nil
                   end
                 end
                 issue
               end
               def fix_issue(issue)
                 status = IssueStatus.find_by_id(Setting.commit_fix_status_id.to_i)
                 if status.nil?
                   logger.warn("No status macthes commit_fix_status_id setting (#{Setting.commit_fix_status_id})") if logger
                   return issue
                 end
                 # the issue may have been updated by the closure of another one (eg. duplicate)
                 issue.reload
                 # don't change the status is the issue is closed
                 return if issue.status && issue.status.is_closed?
                 journal = issue.init_journal(user || User.anonymous, ll(Setting.default_language, :text_status_changed_by_changeset, text_tag))
                 issue.status = status
                 unless Setting.commit_fix_done_ratio.blank?
                   issue.done_ratio = Setting.commit_fix_done_ratio.to_i
                 end
                 Redmine::Hook.call_hook(:model_changeset_scan_commit_for_issue_ids_pre_issue_update,
                                         { :changeset => self, :issue => issue })
                 unless issue.save
                   logger.warn("Issue ##{issue.id} could not be saved by changeset #{id}: #{issue.errors.full_messages}") if logger
                 end
                 issue
               end
               def log_time(issue, hours)
                 time_entry = TimeEntry.new(
                   :user => user,
                   :hours => hours,
                   :issue => issue,
                   :spent_on => commit_date,
                   :comments => l(:text_time_logged_by_changeset, :value => text_tag,
                                  :locale => Setting.default_language)
                   )
                 time_entry.activity = log_time_activity unless log_time_activity.nil?
                 unless time_entry.save
                   logger.warn("TimeEntry could not be created by changeset #{id}: #{time_entry.errors.full_messages}") if logger
                 end
                 time_entry
               end
               def log_time_activity
                 if Setting.commit_logtime_activity_id.to_i > 0
                   TimeEntryActivity.find_by_id(Setting.commit_logtime_activity_id.to_i)
                 end
               end
               def split_comments
                 comments =~ /\A(.+?)\r?\n(.*)$/m
                 @short_comments = $1 || comments
                 @long_comments = $2.to_s.strip
                 return @short_comments, @long_comments
               end
               public
               # Strips and reencodes a commit log before insertion into the database
               def self.normalize_comments(str, encoding)
                 Changeset.to_utf8(str.to_s.strip, encoding)
               end
               private
               def self.to_utf8(str, encoding)
                 return str if str.nil?
                 str.force_encoding("ASCII-8BIT") if str.respond_to?(:force_encoding)
                 if str.empty?
                   str.force_encoding("UTF-8") if str.respond_to?(:force_encoding)
                   return str
                 end
-                if str.respond_to?(:force_encoding)
                 enc = encoding.blank? ? "UTF-8" : encoding
+                if str.respond_to?(:force_encoding)
                   if enc != "UTF-8"
                     str.force_encoding(enc)
                     str = str.encode("UTF-8", :invalid => :replace,
                           :undef => :replace, :replace => '?')
                   else
                     str.force_encoding("UTF-8")
                     if ! str.valid_encoding?
                       str = str.encode("US-ASCII", :invalid => :replace,
                             :undef => :replace, :replace => '?').encode("UTF-8")
                     end
                   end
                 else
-                  unless encoding.blank? || encoding == 'UTF-8'
+                  ic = Iconv.new('UTF-8', enc)
-                    begin
+                  txtar = ""
-                      str = Iconv.conv('UTF-8', encoding, str)
-                    rescue Iconv::Failure
-                      # do nothing here
-                    end
-                  end
-                  # removes invalid UTF8 sequences
                   begin
-                    str = Iconv.conv('UTF-8//IGNORE', 'UTF-8', str + '  ')[0..-3]
+                    txtar += ic.iconv(str)
-                  rescue Iconv::InvalidEncoding
+                  rescue Iconv::IllegalSequence
-                    # "UTF-8//IGNORE" is not supported on some OS
+                    txtar += $!.success
-                  end
+                    str = '?' + $!.failed[1,$!.failed.length]
+                    retry
+                  rescue
+                    txtar += $!.success
+                  end
+                  str = txtar
                 end
                 str
               end
             end

test/unit/changeset_test.rb +6 -12

             # encoding: utf-8
             #
             # Redmine - project management software
             # Copyright (C) 2006-2010  Jean-Philippe Lang
             #
             # This program is free software; you can redistribute it and/or
             # modify it under the terms of the GNU General Public License
             # as published by the Free Software Foundation; either version 2
             # of the License, or (at your option) any later version.
             #
             # This program is distributed in the hope that it will be useful,
             # but WITHOUT ANY WARRANTY; without even the implied warranty of
             # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
             # GNU General Public License for more details.
             #
             # You should have received a copy of the GNU General Public License
             # along with this program; if not, write to the Free Software
             # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
             require File.expand_path('../../test_helper', __FILE__)
             class ChangesetTest < ActiveSupport::TestCase
               fixtures :projects, :repositories, :issues, :issue_statuses,
-                       :changesets, :changes, :issue_categories, :enumerations, :custom_fields, :custom_values, :users, :members, :member_roles, :trackers
+                       :changesets, :changes, :issue_categories, :enumerations,
+                       :custom_fields, :custom_values, :users, :members, :member_roles, :trackers
               def setup
               end
               def test_ref_keywords_any
                 ActionMailer::Base.deliveries.clear
                 Setting.commit_fix_status_id = IssueStatus.find(
                                                :first, :conditions => ["is_closed = ?", true]).id
                 Setting.commit_fix_done_ratio = '90'
                 Setting.commit_ref_keywords = '*'
                 Setting.commit_fix_keywords = 'fixes , closes'
                 c = Changeset.new(:repository => Project.find(1).repository,
                                   :committed_on => Time.now,
                                   :comments => 'New commit (#2). Fixes #1')
                 c.scan_comment_for_issue_ids
                 assert_equal [1, 2], c.issue_ids.sort
                 fixed = Issue.find(1)
                 assert fixed.closed?
                 assert_equal 90, fixed.done_ratio
                 assert_equal 1, ActionMailer::Base.deliveries.size
               end
               def test_ref_keywords
                 Setting.commit_ref_keywords = 'refs'
                 Setting.commit_fix_keywords = ''
                 c = Changeset.new(:repository => Project.find(1).repository,
                                   :committed_on => Time.now,
                                   :comments => 'Ignores #2. Refs #1')
                 c.scan_comment_for_issue_ids
                 assert_equal [1], c.issue_ids.sort
               end
               def test_ref_keywords_any_only
                 Setting.commit_ref_keywords = '*'
                 Setting.commit_fix_keywords = ''
                 c = Changeset.new(:repository => Project.find(1).repository,
                                   :committed_on => Time.now,
                                   :comments => 'Ignores #2. Refs #1')
                 c.scan_comment_for_issue_ids
                 assert_equal [1, 2], c.issue_ids.sort
               end
               def test_ref_keywords_any_with_timelog
                 Setting.commit_ref_keywords = '*'
                 Setting.commit_logtime_enabled = '1'
                 {
                   '2' => 2.0,
                   '2h' => 2.0,
                   '2hours' => 2.0,
                   '15m' => 0.25,
                   '15min' => 0.25,
                   '3h15' => 3.25,
                   '3h15m' => 3.25,
                   '3h15min' => 3.25,
                   '3:15' => 3.25,
                   '3.25' => 3.25,
                   '3.25h' => 3.25,
                   '3,25' => 3.25,
                   '3,25h' => 3.25,
                 }.each do |syntax, expected_hours|
                   c = Changeset.new(:repository => Project.find(1).repository,
                                     :committed_on => 24.hours.ago,
                                     :comments => "Worked on this issue #1 @#{syntax}",
                                     :revision => '520',
                                     :user => User.find(2))
                   assert_difference 'TimeEntry.count' do
                     c.scan_comment_for_issue_ids
                   end
                   assert_equal [1], c.issue_ids.sort
                   time = TimeEntry.first(:order => 'id desc')
                   assert_equal 1, time.issue_id
                   assert_equal 1, time.project_id
                   assert_equal 2, time.user_id
                   assert_equal expected_hours, time.hours,
                       "@#{syntax} should be logged as #{expected_hours} hours but was #{time.hours}"
                   assert_equal Date.yesterday, time.spent_on
                   assert time.activity.is_default?
                   assert time.comments.include?('r520'),
                         "r520 was expected in time_entry comments: #{time.comments}"
                 end
               end
               def test_ref_keywords_closing_with_timelog
                 Setting.commit_fix_status_id = IssueStatus.find(
                                                 :first, :conditions => ["is_closed = ?", true]).id
                 Setting.commit_ref_keywords = '*'
                 Setting.commit_fix_keywords = 'fixes , closes'
                 Setting.commit_logtime_enabled = '1'
                 c = Changeset.new(:repository => Project.find(1).repository,
                                   :committed_on => Time.now,
                                   :comments => 'This is a comment. Fixes #1 @4.5, #2 @1',
                                   :user => User.find(2))
                 assert_difference 'TimeEntry.count', 2 do
                   c.scan_comment_for_issue_ids
                 end
                 assert_equal [1, 2], c.issue_ids.sort
                 assert Issue.find(1).closed?
                 assert Issue.find(2).closed?
                 times = TimeEntry.all(:order => 'id desc', :limit => 2)
                 assert_equal [1, 2], times.collect(&:issue_id).sort
               end
               def test_ref_keywords_any_line_start
                 Setting.commit_ref_keywords = '*'
                 c = Changeset.new(:repository => Project.find(1).repository,
                                   :committed_on => Time.now,
                                   :comments => '#1 is the reason of this commit')
                 c.scan_comment_for_issue_ids
                 assert_equal [1], c.issue_ids.sort
               end
               def test_ref_keywords_allow_brackets_around_a_issue_number
                 Setting.commit_ref_keywords = '*'
                 c = Changeset.new(:repository => Project.find(1).repository,
                                   :committed_on => Time.now,
                                   :comments => '[#1] Worked on this issue')
                 c.scan_comment_for_issue_ids
                 assert_equal [1], c.issue_ids.sort
               end
               def test_ref_keywords_allow_brackets_around_multiple_issue_numbers
                 Setting.commit_ref_keywords = '*'
                 c = Changeset.new(:repository => Project.find(1).repository,
                                   :committed_on => Time.now,
                                   :comments => '[#1 #2, #3] Worked on these')
                 c.scan_comment_for_issue_ids
                 assert_equal [1,2,3], c.issue_ids.sort
               end
               def test_commit_referencing_a_subproject_issue
                 c = Changeset.new(:repository => Project.find(1).repository,
                                   :committed_on => Time.now,
                                   :comments => 'refs #5, a subproject issue')
                 c.scan_comment_for_issue_ids
                 assert_equal [5], c.issue_ids.sort
                 assert c.issues.first.project != c.project
               end
               def test_commit_referencing_a_parent_project_issue
                 # repository of child project
                 r = Repository::Subversion.create!(
                       :project => Project.find(3),
                       :url     => 'svn://localhost/test')
                 c = Changeset.new(:repository => r,
                                   :committed_on => Time.now,
                                   :comments => 'refs #2, an issue of a parent project')
                 c.scan_comment_for_issue_ids
                 assert_equal [2], c.issue_ids.sort
                 assert c.issues.first.project != c.project
               end
               def test_text_tag_revision
                 c = Changeset.new(:revision => '520')
                 assert_equal 'r520', c.text_tag
               end
               def test_text_tag_hash
                 c = Changeset.new(
                       :scmid    => '7234cb2750b63f47bff735edc50a1c0a433c2518',
                       :revision => '7234cb2750b63f47bff735edc50a1c0a433c2518')
                 assert_equal 'commit:7234cb2750b63f47bff735edc50a1c0a433c2518', c.text_tag
               end
               def test_text_tag_hash_all_number
                 c = Changeset.new(:scmid => '0123456789', :revision => '0123456789')
                 assert_equal 'commit:0123456789', c.text_tag
               end
               def test_previous
                 changeset = Changeset.find_by_revision('3')
                 assert_equal Changeset.find_by_revision('2'), changeset.previous
               end
               def test_previous_nil
                 changeset = Changeset.find_by_revision('1')
                 assert_nil changeset.previous
               end
               def test_next
                 changeset = Changeset.find_by_revision('2')
                 assert_equal Changeset.find_by_revision('3'), changeset.next
               end
               def test_next_nil
                 changeset = Changeset.find_by_revision('10')
                 assert_nil changeset.next
               end
               def test_comments_should_be_converted_to_utf8
                   proj = Project.find(3)
                   # str = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
                   str = "Texte encod\xe9 en ISO-8859-1."
                   str.force_encoding("ASCII-8BIT") if str.respond_to?(:force_encoding)
                   r = Repository::Bazaar.create!(
                         :project => proj, :url => '/tmp/test/bazaar',
                         :log_encoding => 'ISO-8859-1' )
                   assert r
                   c = Changeset.new(:repository => r,
                                     :committed_on => Time.now,
                                     :revision => '123',
                                     :scmid => '12345',
                                     :comments => str)
                   assert( c.save )
                   str_utf8 = "Texte encod\xc3\xa9 en ISO-8859-1."
                   str_utf8.force_encoding("UTF-8") if str_utf8.respond_to?(:force_encoding)
                   assert_equal str_utf8, c.comments
               end
-              def test_invalid_utf8_sequences_in_comments_should_be_stripped
+              def test_invalid_utf8_sequences_in_comments_should_be_replaced_latin1
                   proj = Project.find(3)
                   # str = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
                   str = "Texte encod\xe9 en ISO-8859-1."
                   str.force_encoding("ASCII-8BIT") if str.respond_to?(:force_encoding)
                   r = Repository::Bazaar.create!(
-                        :project => proj, :url => '/tmp/test/bazaar',
+                        :project => proj,
+                        :url => '/tmp/test/bazaar',
                         :log_encoding => 'UTF-8' )
                   assert r
                   c = Changeset.new(:repository   => r,
                                     :committed_on => Time.now,
                                     :revision     => '123',
                                     :scmid        => '12345',
                                     :comments     => str)
                   assert( c.save )
-                  if str.respond_to?(:force_encoding)
                   assert_equal "Texte encod? en ISO-8859-1.", c.comments
-                  else
-                    assert_equal "Texte encod en ISO-8859-1.", c.comments
-                  end
               end
-              def test_invalid_utf8_sequences_in_comments_should_be_stripped_ja_jis
+              def test_invalid_utf8_sequences_in_comments_should_be_replaced_ja_jis
                   proj = Project.find(3)
                   str = "test\xb5\xfetest\xb5\xfe"
                   if str.respond_to?(:force_encoding)
                     str.force_encoding('ASCII-8BIT')
                   end
                   r = Repository::Bazaar.create!(
                         :project => proj,
                         :url     => '/tmp/test/bazaar',
                         :log_encoding => 'ISO-2022-JP' )
                   assert r
                   c = Changeset.new(:repository   => r,
                                     :committed_on => Time.now,
                                     :revision     => '123',
                                     :scmid        => '12345',
                                     :comments     => str)
                   assert( c.save )
-                  if str.respond_to?(:force_encoding)
                   assert_equal "test??test??", c.comments
-                  else
-                    assert_equal "testtest", c.comments
-                  end
               end
               def test_comments_should_be_converted_all_latin1_to_utf8
                   s1 = "\xC2\x80"
                   s2 = "\xc3\x82\xc2\x80"
                   s4 = s2.dup
                   if s1.respond_to?(:force_encoding)
                     s3 = s1.dup
                     s1.force_encoding('ASCII-8BIT')
                     s2.force_encoding('ASCII-8BIT')
                     s3.force_encoding('ISO-8859-1')
                     s4.force_encoding('UTF-8')
                     assert_equal s3.encode('UTF-8'), s4
                   end
                   proj = Project.find(3)
                   r = Repository::Bazaar.create!(
                         :project => proj, :url => '/tmp/test/bazaar',
                         :log_encoding => 'ISO-8859-1' )
                   assert r
                   c = Changeset.new(:repository => r,
                                     :committed_on => Time.now,
                                     :revision => '123',
                                     :scmid => '12345',
                                     :comments => s1)
                   assert( c.save )
                   assert_equal s4, c.comments
               end
               def test_comments_nil
                   proj = Project.find(3)
                   r = Repository::Bazaar.create!(
                         :project => proj, :url => '/tmp/test/bazaar',
                         :log_encoding => 'ISO-8859-1' )
                   assert r
                   c = Changeset.new(:repository => r,
                                     :committed_on => Time.now,
                                     :revision => '123',
                                     :scmid => '12345',
                                     :comments => nil)
                   assert( c.save )
                   assert_equal "", c.comments
                   if c.comments.respond_to?(:force_encoding)
                     assert_equal "UTF-8", c.comments.encoding.to_s
                   end
               end
               def test_comments_empty
                   proj = Project.find(3)
                   r = Repository::Bazaar.create!(
                         :project => proj, :url => '/tmp/test/bazaar',
                         :log_encoding => 'ISO-8859-1' )
                   assert r
                   c = Changeset.new(:repository => r,
                                     :committed_on => Time.now,
                                     :revision => '123',
                                     :scmid => '12345',
                                     :comments => "")
                   assert( c.save )
                   assert_equal "", c.comments
                   if c.comments.respond_to?(:force_encoding)
                     assert_equal "UTF-8", c.comments.encoding.to_s
                   end
               end
               def test_identifier
                 c = Changeset.find_by_revision('1')
                 assert_equal c.revision, c.identifier
               end
             end

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages