jro_apps/redmine Commit - r5307:bb48d96d3037

scm: replace invalid utf-8 sequences instead of stripping in displaying repository contents on Ruby 1.8....

Toshi MARUYAMA -

r5307:bb48d96d3037

parent child

Context file:

r5307:bb48d96d3037

Collapse all files

app/helpers/repositories_helper.rb +10 -4

             # redMine - project management software
             # Copyright (C) 2006  Jean-Philippe Lang
             #
             # This program is free software; you can redistribute it and/or
             # modify it under the terms of the GNU General Public License
             # as published by the Free Software Foundation; either version 2
             # of the License, or (at your option) any later version.
             #
             # This program is distributed in the hope that it will be useful,
             # but WITHOUT ANY WARRANTY; without even the implied warranty of
             # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
             # GNU General Public License for more details.
             #
             # You should have received a copy of the GNU General Public License
             # along with this program; if not, write to the Free Software
             # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
             require 'iconv'
             module RepositoriesHelper
               def format_revision(revision)
                 if revision.respond_to? :format_identifier
                   revision.format_identifier
                 else
                   revision.to_s
                 end
               end
               def truncate_at_line_break(text, length = 255)
                 if text
                   text.gsub(%r{^(.{#{length}}[^\n]*)\n.+$}m, '\\1...')
                 end
               end
               def render_properties(properties)
                 unless properties.nil? || properties.empty?
                   content = ''
                   properties.keys.sort.each do |property|
                     content << content_tag('li', "<b>#{h property}</b>: <span>#{h properties[property]}</span>")
                   end
                   content_tag('ul', content, :class => 'properties')
                 end
               end
               def render_changeset_changes
                 changes = @changeset.changes.find(:all, :limit => 1000, :order => 'path').collect do |change|
                   case change.action
                   when 'A'
                     # Detects moved/copied files
                     if !change.from_path.blank?
                       change.action = @changeset.changes.detect {|c| c.action == 'D' && c.path == change.from_path} ? 'R' : 'C'
                     end
                     change
                   when 'D'
                     @changeset.changes.detect {|c| c.from_path == change.path} ? nil : change
                   else
                     change
                   end
                end.compact
                 tree = { }
                 changes.each do |change|
                   p = tree
                   dirs = change.path.to_s.split('/').select {|d| !d.blank?}
                   path = ''
                   dirs.each do |dir|
                     path += '/' + dir
                     p[:s] ||= {}
                     p = p[:s]
                     p[path] ||= {}
                     p = p[path]
                   end
                   p[:c] = change
                 end
                 render_changes_tree(tree[:s])
               end
               def render_changes_tree(tree)
                 return '' if tree.nil?
                 output = ''
                 output << '<ul>'
                 tree.keys.sort.each do |file|
                   style = 'change'
                   text = File.basename(h(file))
                   if s = tree[file][:s]
                     style << ' folder'
                     path_param = to_path_param(@repository.relative_path(file))
                     text = link_to(text, :controller => 'repositories',
                                          :action => 'show',
                                          :id => @project,
                                          :path => path_param,
                                          :rev => @changeset.identifier)
                     output << "<li class='#{style}'>#{text}</li>"
                     output << render_changes_tree(s)
                   elsif c = tree[file][:c]
                     style << " change-#{c.action}"
                     path_param = to_path_param(@repository.relative_path(c.path))
                     text = link_to(text, :controller => 'repositories',
                                          :action => 'entry',
                                          :id => @project,
                                          :path => path_param,
                                          :rev => @changeset.identifier) unless c.action == 'D'
                     text << " - #{c.revision}" unless c.revision.blank?
                     text << ' (' + link_to('diff', :controller => 'repositories',
                                                    :action => 'diff',
                                                    :id => @project,
                                                    :path => path_param,
                                                    :rev => @changeset.identifier) + ') ' if c.action == 'M'
                     text << ' ' + content_tag('span', c.from_path, :class => 'copied-from') unless c.from_path.blank?
                     output << "<li class='#{style}'>#{text}</li>"
                   end
                 end
                 output << '</ul>'
                 output
               end
               def to_utf8(str)
                 return str if str.nil?
                 str = to_utf8_internal(str)
                 if str.respond_to?(:force_encoding)
                   str.force_encoding('UTF-8')
                 end
                 str
               end
               def to_utf8_internal(str)
                 return str if str.nil?
                 if str.respond_to?(:force_encoding)
                   str.force_encoding('ASCII-8BIT')
                 end
                 return str if str.empty?
                 return str if /\A[\r\n\t\x20-\x7e]*\Z/n.match(str) # for us-ascii
                 if str.respond_to?(:force_encoding)
                   str.force_encoding('UTF-8')
                 end
                 @encodings ||= Setting.repositories_encodings.split(',').collect(&:strip)
                 @encodings.each do |encoding|
                   begin
                     return Iconv.conv('UTF-8', encoding, str)
                   rescue Iconv::Failure
                     # do nothing here and try the next encoding
                   end
                 end
                 str = replace_invalid_utf8(str)
               end
               private :to_utf8_internal
               def replace_invalid_utf8(str)
                 return str if str.nil?
                 if str.respond_to?(:force_encoding)
                   str.force_encoding('UTF-8')
                   if ! str.valid_encoding?
                     str = str.encode("US-ASCII", :invalid => :replace,
                           :undef => :replace, :replace => '?').encode("UTF-8")
                   end
                 else
-                  # removes invalid UTF8 sequences
+                  ic = Iconv.new('UTF-8', 'UTF-8')
+                  txtar = ""
                   begin
-                    str = Iconv.conv('UTF-8//IGNORE', 'UTF-8', str + '  ')[0..-3]
+                    txtar += ic.iconv(str)
-                  rescue Iconv::InvalidEncoding
+                  rescue Iconv::IllegalSequence
-                    # "UTF-8//IGNORE" is not supported on some OS
+                    txtar += $!.success
+                    str = '?' + $!.failed[1,$!.failed.length]
+                    retry
+                  rescue
+                    txtar += $!.success
                   end
+                  str = txtar
                 end
                 str
               end
               def repository_field_tags(form, repository)
                 method = repository.class.name.demodulize.underscore + "_field_tags"
                 if repository.is_a?(Repository) &&
                     respond_to?(method) && method != 'repository_field_tags'
                   send(method, form, repository)
                 end
               end
               def scm_select_tag(repository)
                 scm_options = [["--- #{l(:actionview_instancetag_blank_option)} ---", '']]
                 Redmine::Scm::Base.all.each do |scm|
                 if Setting.enabled_scm.include?(scm) ||
                       (repository && repository.class.name.demodulize == scm)
                     scm_options << ["Repository::#{scm}".constantize.scm_name, scm]
                   end
                 end
                 select_tag('repository_scm',
                            options_for_select(scm_options, repository.class.name.demodulize),
                            :disabled => (repository && !repository.new_record?),
                            :onchange => remote_function(
                               :url => {
                                   :controller => 'repositories',
                                   :action => 'edit',
                                   :id => @project
                                     },
                            :method => :get,
                            :with => "Form.serialize(this.form)")
                            )
               end
               def with_leading_slash(path)
                 path.to_s.starts_with?('/') ? path : "/#{path}"
               end
               def without_leading_slash(path)
                 path.gsub(%r{^/+}, '')
               end
               def subversion_field_tags(form, repository)
                   content_tag('p', form.text_field(:url, :size => 60, :required => true,
                                    :disabled => (repository && !repository.root_url.blank?)) +
                                    '<br />(file:///, http://, https://, svn://, svn+[tunnelscheme]://)') +
                   content_tag('p', form.text_field(:login, :size => 30)) +
                   content_tag('p', form.password_field(
                                         :password, :size => 30, :name => 'ignore',
                                         :value => ((repository.new_record? || repository.password.blank?) ? '' : ('x'*15)),
                                         :onfocus => "this.value=''; this.name='repository[password]';",
                                         :onchange => "this.name='repository[password]';"))
               end
               def darcs_field_tags(form, repository)
                   content_tag('p', form.text_field(:url, :label => 'Root directory',
                                    :size => 60, :required => true,
                                    :disabled => (repository && !repository.new_record?))) +
                   content_tag('p', form.select(:log_encoding, [nil] + Setting::ENCODINGS,
                                                :label => 'Commit messages encoding', :required => true))
               end
               def mercurial_field_tags(form, repository)
                 content_tag('p', form.text_field(:url, :label => 'Root directory',
                                    :size => 60, :required => true,
                                    :disabled => (repository && !repository.root_url.blank?)) +
                                    '<br />Local repository (e.g. /hgrepo, c:\hgrepo)' ) +
                 content_tag('p', form.select(
                                     :path_encoding, [nil] + Setting::ENCODINGS,
                                     :label => 'Path encoding') +
                                     '<br />Default: UTF-8')
               end
               def git_field_tags(form, repository)
                 content_tag('p', form.text_field(:url, :label => 'Path to repository',
                                    :size => 60, :required => true,
                                    :disabled => (repository && !repository.root_url.blank?)) +
                                    '<br />Bare and local repository (e.g. /gitrepo, c:\gitrepo)') +
                 content_tag('p', form.select(
                                     :path_encoding, [nil] + Setting::ENCODINGS,
                                     :label => 'Path encoding') +
                                     '<br />Default: UTF-8')
               end
               def cvs_field_tags(form, repository)
                 content_tag('p', form.text_field(:root_url,
                                  :label => 'CVSROOT', :size => 60, :required => true,
                                  :disabled => !repository.new_record?)) +
                 content_tag('p', form.text_field(:url, :label => 'Module',
                                  :size => 30, :required => true,
                                  :disabled => !repository.new_record?)) +
                 content_tag('p', form.select(:log_encoding, [nil] + Setting::ENCODINGS,
                                              :label => 'Commit messages encoding', :required => true))
               end
               def bazaar_field_tags(form, repository)
                 content_tag('p', form.text_field(:url, :label => 'Root directory',
                                  :size => 60, :required => true,
                                  :disabled => (repository && !repository.new_record?))) +
                 content_tag('p', form.select(:log_encoding, [nil] + Setting::ENCODINGS,
                                              :label => 'Commit messages encoding', :required => true))
               end
               def filesystem_field_tags(form, repository)
                 content_tag('p', form.text_field(:url, :label => 'Root directory',
                                  :size => 60, :required => true,
                                  :disabled => (repository && !repository.root_url.blank?))) +
                 content_tag('p', form.select(
                                     :path_encoding, [nil] + Setting::ENCODINGS,
                                     :label => 'Path encoding') +
                                     '<br />Default: UTF-8')
               end
             end

test/unit/helpers/repository_helper_test.rb +1 -3

             # Redmine - project management software
             # Copyright (C) 2006-2011  Jean-Philippe Lang
             #
             # This program is free software; you can redistribute it and/or
             # modify it under the terms of the GNU General Public License
             # as published by the Free Software Foundation; either version 2
             # of the License, or (at your option) any later version.
             #
             # This program is distributed in the hope that it will be useful,
             # but WITHOUT ANY WARRANTY; without even the implied warranty of
             # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
             # GNU General Public License for more details.
             #
             # You should have received a copy of the GNU General Public License
             # along with this program; if not, write to the Free Software
             # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
             require File.expand_path('../../../test_helper', __FILE__)
             class RepositoryHelperTest < HelperTestCase
               include RepositoriesHelper
               def test_from_latin1_to_utf8
                 with_settings :repositories_encodings => 'UTF-8,ISO-8859-1' do
                   s1 = "Texte encod\xc3\xa9"
                   s2 = "Texte encod\xe9"
                   s3 = s2.dup
                   if s1.respond_to?(:force_encoding)
                     s1.force_encoding("UTF-8")
                     s2.force_encoding("ASCII-8BIT")
                     s3.force_encoding("UTF-8")
                   end
                   assert_equal s1, to_utf8(s2)
                   assert_equal s1, to_utf8(s3)
                 end
               end
               def test_from_euc_jp_to_utf8
                 with_settings :repositories_encodings => 'UTF-8,EUC-JP' do
                   s1 = "\xe3\x83\xac\xe3\x83\x83\xe3\x83\x89\xe3\x83\x9e\xe3\x82\xa4\xe3\x83\xb3"
                   s2 = "\xa5\xec\xa5\xc3\xa5\xc9\xa5\xde\xa5\xa4\xa5\xf3"
                   s3 = s2.dup
                   if s1.respond_to?(:force_encoding)
                     s1.force_encoding("UTF-8")
                     s2.force_encoding("ASCII-8BIT")
                     s3.force_encoding("UTF-8")
                   end
                   assert_equal s1, to_utf8(s2)
                   assert_equal s1, to_utf8(s3)
                 end
               end
               def test_to_utf8_should_be_converted_all_latin1_to_utf8
                 with_settings :repositories_encodings => 'ISO-8859-1' do
                   s1 = "\xc3\x82\xc2\x80"
                   s2 = "\xC2\x80"
                   s3 = s2.dup
                   if s1.respond_to?(:force_encoding)
                     s1.force_encoding("UTF-8")
                     s2.force_encoding("ASCII-8BIT")
                     s3.force_encoding("UTF-8")
                   end
                   assert_equal s1, to_utf8(s2)
                   assert_equal s1, to_utf8(s3)
                 end
               end
               def test_to_utf8_blank_string
                 assert_equal "",  to_utf8("")
                 assert_equal nil, to_utf8(nil)
               end
               def test_to_utf8_returns_ascii_as_utf8
                 s1 = "ASCII"
                 s2 = s1.dup
                 if s1.respond_to?(:force_encoding)
                   s1.force_encoding("UTF-8")
                   s2.force_encoding("ISO-8859-1")
                 end
                 str1 = to_utf8(s1)
                 str2 = to_utf8(s2)
                 assert_equal s1, str1
                 assert_equal s1, str2
                 if s1.respond_to?(:force_encoding)
                   assert_equal "UTF-8", str1.encoding.to_s
                   assert_equal "UTF-8", str2.encoding.to_s
                 end
               end
               def test_to_utf8_invalid_utf8_sequences_should_be_stripped
                 with_settings :repositories_encodings => '' do
                   # s1 = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
                   s1 = "Texte encod\xe9 en ISO-8859-1."
                   s1.force_encoding("ASCII-8BIT") if s1.respond_to?(:force_encoding)
                   str = to_utf8(s1)
                   if str.respond_to?(:force_encoding)
-                    assert_equal "Texte encod? en ISO-8859-1.", str
                     assert str.valid_encoding?
                     assert_equal "UTF-8", str.encoding.to_s
-                  else
-                    assert_equal "Texte encod en ISO-8859-1.", str
                   end
+                  assert_equal "Texte encod? en ISO-8859-1.", str
                 end
               end
             end

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages