jro_apps/redmine Commit - r15683:f30462595e83

Optional Regex delimiters to truncate incoming emails (#5864)....

Jean-Philippe Lang -

r15683:f30462595e83

parent child

Context file:

r15683:f30462595e83

Collapse all files

test/fixtures/mail_handler/ticket_reply_from_mail.eml created 644 +35 0

@@ -0,0 +1,35
	1	Return-Path: <jsmith@somenet.foo>
	2	Received: from osiris ([127.0.0.1])
	3	by OSIRIS
	4	with hMailServer; Wed, 12 Oct 2016 03:05:50 -0700
	5	Message-ID: <000501c8d452$a95cd7e0$0a00a8c0@osiris>
	6	From: "John Smith" <JSmith@somenet.foo>
	7	To: <redmine@somenet.foo>
	8	Subject: New ticket on a given project
	9	Date: Wed, 12 Oct 2016 13:05:38 +0300
	10	MIME-Version: 1.0
	11	Content-Type: text/plain;
	12	format=flowed;
	13	charset="iso-8859-1";
	14	reply-type=original
	15	Content-Transfer-Encoding: 7bit
	16	X-Priority: 3
	17	X-MSMail-Priority: Normal
	18	X-Mailer: Microsoft Outlook Express 6.00.2900.2869
	19	X-MimeOLE: Produced By Microsoft MimeOLE V6.00.2900.2869
	20
	21	Project: onlinestore
	22	Status: Resolved
	23	due date: 2010-12-31
	24	Start Date:2010-01-01
	25	Assigned to: John Smith
	26	fixed version: alpha
	27	estimated hours: 2.5
	28	remaining hours: 1
	29	done ratio: 30
	30
	31	This paragraph is before delimiter
	32
	33	On Wed, 11 Oct at 1:05 PM, Jon Smith <jsmith@somenet.foo<mailto:jsmith@somenet.foo>> wrote:
	34
	35	This paragraph is after the delimiter

app/controllers/settings_controller.rb +18 -13

@@ -34,24 +34,29 class SettingsController < ApplicationController
34	def edit	34	def edit
35	@notifiables = Redmine::Notifiable.all	35	@notifiables = Redmine::Notifiable.all
36	if request.post?	36	if request.post?
37	if Setting.set_all_from_params(params[:settings])	37	errors = Setting.set_all_from_params(params[:settings])
		38	if errors.blank?
38	flash[:notice] = l(:notice_successful_update)	39	flash[:notice] = l(:notice_successful_update)
		40	redirect_to settings_path(:tab => params[:tab])
		41	return
		42	else
		43	@setting_errors = errors
		44	# render the edit form with error messages
39	end	45	end
40	redirect_to settings_path(:tab => params[:tab])	46	end
41	else
42	@options = {}
43	user_format = User::USER_FORMATS.collect{\|key, value\| [key, value[:setting_order]]}.sort{\|a, b\| a[1] <=> b[1]}
44	@options[:user_format] = user_format.collect{\|f\| [User.current.name(f[0]), f[0].to_s]}
45	@deliveries = ActionMailer::Base.perform_deliveries
46		47
47	@guessed_host_and_path = request.host_with_port.dup	48	@options = {}
48	@guessed_host_and_path << ('/'+ Redmine::Utils.relative_url_root.gsub(%r{^\/}, '')) unless Redmine::Utils.relative_url_root.blank?	49	user_format = User::USER_FORMATS.collect{\|key, value\| [key, value[:setting_order]]}.sort{\|a, b\| a[1] <=> b[1]}
		50	@options[:user_format] = user_format.collect{\|f\| [User.current.name(f[0]), f[0].to_s]}
		51	@deliveries = ActionMailer::Base.perform_deliveries
49		52
50	@commit_update_keywords = Setting.commit_update_keywords.dup	53	@guessed_host_and_path = request.host_with_port.dup
51	@commit_update_keywords = [{}] unless @commit_update_keywords.is_a?(Array) && @commit_update_keywords.any?	54	@guessed_host_and_path << ('/'+ Redmine::Utils.relative_url_root.gsub(%r{^\/}, '')) unless Redmine::Utils.relative_url_root.blank?
52		55
53	Redmine::Themes.rescan	56	@commit_update_keywords = Setting.commit_update_keywords.dup
54	end	57	@commit_update_keywords = [{}] unless @commit_update_keywords.is_a?(Array) && @commit_update_keywords.any?
		58
		59	Redmine::Themes.rescan
55	end	60	end
56		61
57	def plugin	62	def plugin

app/helpers/settings_helper.rb +23 -6

                         ]
               end
+              def render_settings_error(errors)
+                return if errors.blank?
+                s = ''.html_safe
+                errors.each do |name, message|
+                  s << content_tag('li', content_tag('b', l("setting_#{name}")) + " " + message)
+                end
+                content_tag('div', content_tag('ul', s), :id => 'errorExplanation')
+              end
+              def setting_value(setting)
+                value = nil
+                if params[:settings]
+                  value = params[:settings][setting]
+                end
+                value || Setting.send(setting)
+              end
               def setting_select(setting, choices, options={})
                 if blank_text = options.delete(:blank)
                   choices = [[blank_text.is_a?(Symbol) ? l(blank_text) : blank_text, '']] + choices
                 end
                 setting_label(setting, options).html_safe +
                   select_tag("settings[#{setting}]",
-                             options_for_select(choices, Setting.send(setting).to_s),
+                             options_for_select(choices, setting_value(setting).to_s),
                              options).html_safe
               end
               def setting_multiselect(setting, choices, options={})
-                setting_values = Setting.send(setting)
+                setting_values = setting_value(setting)
                 setting_values = [] unless setting_values.is_a?(Array)
                 content_tag("label", l(options[:label] || "setting_#{setting}")) +
               def setting_text_field(setting, options={})
                 setting_label(setting, options).html_safe +
-                  text_field_tag("settings[#{setting}]", Setting.send(setting), options).html_safe
+                  text_field_tag("settings[#{setting}]", setting_value(setting), options).html_safe
               end
               def setting_text_area(setting, options={})
                 setting_label(setting, options).html_safe +
-                  text_area_tag("settings[#{setting}]", Setting.send(setting), options).html_safe
+                  text_area_tag("settings[#{setting}]", setting_value(setting), options).html_safe
               end
               def setting_check_box(setting, options={})
                 setting_label(setting, options).html_safe +
                   hidden_field_tag("settings[#{setting}]", 0, :id => nil).html_safe +
-                    check_box_tag("settings[#{setting}]", 1, Setting.send("#{setting}?"), options).html_safe
+                    check_box_tag("settings[#{setting}]", 1, setting_value(setting).to_s != '0', options).html_safe
               end
               def setting_label(setting, options={})
                 tag = check_box_tag('settings[notified_events][]',
                   notifiable.name,
-                  Setting.notified_events.include?(notifiable.name),
+                  setting_value('notified_events').include?(notifiable.name),
                   :id => nil,
                   :data => tag_data)

app/models/mail_handler.rb +11 -2

@@ -561,9 +561,18 class MailHandler < ActionMailer::Base
561		561
562	# Removes the email body of text after the truncation configurations.	562	# Removes the email body of text after the truncation configurations.
563	def cleanup_body(body)	563	def cleanup_body(body)
564	delimiters = Setting.mail_handler_body_delimiters.to_s.split(/[\r\n]+/).reject(&:blank?).~~map~~ {\|s\| ~~Regexp~~.~~escape~~(s)}	564	delimiters = Setting.mail_handler_body_delimiters.to_s.split(/[\r\n]+/).reject(&:blank?)
		565
		566	if Setting.mail_handler_enable_regex_delimiters?
		567	begin
		568	delimiters = delimiters.map {\|s\| Regexp.new(s)}
		569	rescue RegexpError => e
		570	logger.error "MailHandler: invalid regexp delimiter found in mail_handler_body_delimiters setting (#{e.message})" if logger
		571	end
		572	end
		573
565	unless delimiters.empty?	574	unless delimiters.empty?
566	regex = Regexp.new("^[> ](#{ delimiters.~~join~~(~~'\|'~~) })\s[\r\n].*", Regexp::MULTILINE)	575	regex = Regexp.new("^[> ](#{ Regexp.union(delimiters) })\s[\r\n].*", Regexp::MULTILINE)
567	body = body.gsub(regex, '')	576	body = body.gsub(regex, '')
568	end	577	end
569	body.strip	578	body.strip

app/models/setting.rb +28 -2

@@ -120,8 +120,12 class Setting < ActiveRecord::Base
120		120
121	# Updates multiple settings from params and sends a security notification if needed	121	# Updates multiple settings from params and sends a security notification if needed
122	def self.set_all_from_params(settings)	122	def self.set_all_from_params(settings)
123	return ~~false~~ unless settings.is_a?(Hash)	123	return nil unless settings.is_a?(Hash)
124	settings = settings.dup.symbolize_keys	124	settings = settings.dup.symbolize_keys
		125
		126	errors = validate_all_from_params(settings)
		127	return errors if errors.present?
		128
125	changes = []	129	changes = []
126	settings.each do \|name, value\|	130	settings.each do \|name, value\|
127	next unless available_settings[name.to_s]	131	next unless available_settings[name.to_s]
@@ -134,7 +138,29 class Setting < ActiveRecord::Base
134	if changes.any?	138	if changes.any?
135	Mailer.security_settings_updated(changes)	139	Mailer.security_settings_updated(changes)
136	end	140	end
137	true	141	nil
		142	end
		143
		144	def self.validate_all_from_params(settings)
		145	messages = []
		146
		147	if settings.key?(:mail_handler_body_delimiters) \|\| settings.key?(:mail_handler_enable_regex_delimiters)
		148	regexp = Setting.mail_handler_enable_regex_delimiters?
		149	if settings.key?(:mail_handler_enable_regex_delimiters)
		150	regexp = settings[:mail_handler_enable_regex_delimiters].to_s != '0'
		151	end
		152	if regexp
		153	settings[:mail_handler_body_delimiters].to_s.split(/[\r\n]+/).each do \|delimiter\|
		154	begin
		155	Regexp.new(delimiter)
		156	rescue RegexpError => e
		157	messages << [:mail_handler_body_delimiters, "#{l('activerecord.errors.messages.not_a_regexp')} (#{e.message})"]
		158	end
		159	end
		160	end
		161	end
		162
		163	messages
138	end	164	end
139		165
140	# Sets a setting value from params	166	# Sets a setting value from params

app/views/settings/_mail_handler.html.erb +1 0

             <div class="box tabular settings">
               <p>
                 <%= setting_text_area :mail_handler_body_delimiters, :rows => 5 %>
+                <%= setting_check_box :mail_handler_enable_regex_delimiters, :label => false %> <%= l(:setting_mail_handler_enable_regex_delimiters) %>
                 <em class="info"><%= l(:text_line_separated) %></em>
               </p>
               <p>

app/views/settings/edit.html.erb +2 0

             <h2><%= l(:label_settings) %></h2>
+            <%= render_settings_error @setting_errors %>
             <%= render_tabs administration_settings_tabs %>
             <% html_title(l(:label_settings), l(:label_administration)) -%>

config/locales/en.yml +2 0

                     circular_dependency: "This relation would create a circular dependency"
                     cant_link_an_issue_with_a_descendant: "An issue cannot be linked to one of its subtasks"
                     earlier_than_minimum_start_date: "cannot be earlier than %{date} because of preceding issues"
+                    not_a_regexp: "is not a valid regular expression"
               actionview_instancetag_blank_option: Please select
               setting_display_subprojects_issues: Display subprojects issues on main projects by default
               setting_enabled_scm: Enabled SCM
               setting_mail_handler_body_delimiters: "Truncate emails after one of these lines"
+              setting_mail_handler_enable_regex_delimiters: "Enable regular expressions"
               setting_mail_handler_api_enabled: Enable WS for incoming emails
               setting_mail_handler_api_key: Incoming email WS API key
               setting_sys_api_key: Repository management WS API key

config/locales/fr.yml +2 0

                     circular_dependency: "Cette relation créerait une dépendance circulaire"
                     cant_link_an_issue_with_a_descendant: "Une demande ne peut pas être liée à l'une de ses sous-tâches"
                     earlier_than_minimum_start_date: "ne peut pas être antérieure au %{date} à cause des demandes qui précèdent"
+                    not_a_regexp: "n'est pas une expression regulière valide"
               actionview_instancetag_blank_option: Choisir
               setting_display_subprojects_issues: Afficher par défaut les demandes des sous-projets sur les projets principaux
               setting_enabled_scm: SCM activés
               setting_mail_handler_body_delimiters: "Tronquer les emails après l'une de ces lignes"
+              setting_mail_handler_enable_regex_delimiters: "Utiliser les expressions regulières"
               setting_mail_handler_api_enabled: "Activer le WS pour la réception d'emails"
               setting_mail_handler_api_key: Clé de protection de l'API
               setting_sequential_project_identifiers: Générer des identifiants de projet séquentiels

config/settings.yml +2 0

               - issue_updated
             mail_handler_body_delimiters:
               default: ''
+            mail_handler_enable_regex_delimiters:
+              default: 0
             mail_handler_excluded_filenames:
               default: ''
             mail_handler_api_enabled:

test/functional/settings_controller_test.rb +29 0

@@ -254,4 +254,33 class SettingsControllerTest < Redmine::ControllerTest
254	ensure	254	ensure
255	Redmine::Plugin.unregister(:foo)	255	Redmine::Plugin.unregister(:foo)
256	end	256	end
		257
		258	def test_post_mail_handler_delimiters_should_not_save_invalid_regex_delimiters
		259	post :edit, :params => {
		260	:settings => {
		261	:mail_handler_enable_regex_delimiters => '1',
		262	:mail_handler_body_delimiters => 'Abc[',
		263	}
		264	}
		265
		266	assert_response :success
		267	assert_equal '0', Setting.mail_handler_enable_regex_delimiters
		268	assert_equal '', Setting.mail_handler_body_delimiters
		269
		270	assert_select_error /is not a valid regular expression/
		271	assert_select 'textarea[name=?]', 'settings[mail_handler_body_delimiters]', :text => 'Abc['
		272	end
		273
		274	def test_post_mail_handler_delimiters_should_save_valid_regex_delimiters
		275	post :edit, :params => {
		276	:settings => {
		277	:mail_handler_enable_regex_delimiters => '1',
		278	:mail_handler_body_delimiters => 'On ., . at ., . <.<mailto:.>> wrote:',
		279	}
		280	}
		281
		282	assert_redirected_to '/settings'
		283	assert_equal '1', Setting.mail_handler_enable_regex_delimiters
		284	assert_equal 'On ., . at ., . <.<mailto:.>> wrote:', Setting.mail_handler_body_delimiters
		285	end
257	end	286	end

test/unit/mail_handler_test.rb +19 0

@@ -977,6 +977,25 class MailHandlerTest < ActiveSupport::TestCase
977	end	977	end
978	end	978	end
979		979
		980	test "truncate emails using a regex delimiter" do
		981	delimiter = "On ., . at ., . <.<mailto:.>> wrote:"
		982	with_settings :mail_handler_enable_regex_delimiters => '1', :mail_handler_body_delimiters => delimiter do
		983	issue = submit_email('ticket_reply_from_mail.eml')
		984	assert_issue_created(issue)
		985	assert issue.description.include?('This paragraph is before delimiter')
		986	assert !issue.description.include?('On Wed, 11 Oct at 1:05 PM, Jon Smith <jsmith@somenet.foo<mailto:jsmith@somenet.foo>> wrote:')
		987	assert !issue.description.include?('This paragraph is after the delimiter')
		988	end
		989
		990	with_settings :mail_handler_enable_regex_delimiters => '0', :mail_handler_body_delimiters => delimiter do
		991	issue = submit_email('ticket_reply_from_mail.eml')
		992	assert_issue_created(issue)
		993	assert issue.description.include?('This paragraph is before delimiter')
		994	assert issue.description.include?('On Wed, 11 Oct at 1:05 PM, Jon Smith <jsmith@somenet.foo<mailto:jsmith@somenet.foo>> wrote:')
		995	assert issue.description.include?('This paragraph is after the delimiter')
		996	end
		997	end
		998
980	def test_attachments_that_match_mail_handler_excluded_filenames_should_be_ignored	999	def test_attachments_that_match_mail_handler_excluded_filenames_should_be_ignored
981	with_settings :mail_handler_excluded_filenames => '.vcf, .jpg' do	1000	with_settings :mail_handler_excluded_filenames => '.vcf, .jpg' do
982	issue = submit_email('ticket_with_attachment.eml', :issue => {:project => 'onlinestore'})	1001	issue = submit_email('ticket_with_attachment.eml', :issue => {:project => 'onlinestore'})

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages