Merge branch 'dd/sequencer-utf8'

Handling of commit objects that use non UTF-8 encoding during
"rebase -i" has been improved.

* dd/sequencer-utf8:
  sequencer: reencode commit message for am/rebase --show-current-patch
  sequencer: reencode old merge-commit message
  sequencer: reencode squashing commit's message
  sequencer: reencode revert/cherry-pick's todo list
  sequencer: reencode to utf-8 before arrange rebase's todo list
  t3900: demonstrate git-rebase problem with multi encoding
  configure.ac: define ICONV_OMITS_BOM if necessary
  t0028: eliminate non-standard usage of printf

5 files changed, 130 insertions, 2 deletions

diff --git a/t/t0028-working-tree-encoding.sh b/t/t0028-working-tree-encoding.sh
index 7aa0945d8d..bfc4fb9af5 100755
--- a/t/t0028-working-tree-encoding.sh
+++ b/t/t0028-working-tree-encoding.sh
@@ -17,7 +17,7 @@ test_lazy_prereq NO_UTF32_BOM '
 write_utf16 () {
 	if test_have_prereq NO_UTF16_BOM
 	then
-		printf '\xfe\xff'
+		printf '\376\377'
 	fi &&
 	iconv -f UTF-8 -t UTF-16
 }
@@ -25,7 +25,7 @@ write_utf16 () {
 write_utf32 () {
 	if test_have_prereq NO_UTF32_BOM
 	then
-		printf '\x00\x00\xfe\xff'
+		printf '\0\0\376\377'
 	fi &&
 	iconv -f UTF-8 -t UTF-32
 }
diff --git a/t/t3434-rebase-i18n.sh b/t/t3434-rebase-i18n.sh
new file mode 100755
index 0000000000..4b5b128cd6
--- /dev/null
+++ b/t/t3434-rebase-i18n.sh
@@ -0,0 +1,84 @@
+#!/bin/sh
+#
+# Copyright (c) 2019 Doan Tran Cong Danh
+#
+
+test_description='rebase with changing encoding
+
+Initial setup:
+
+1 - 2              master
+ \
+  3 - 4            first
+   \
+    5 - 6          second
+'
+
+. ./test-lib.sh
+
+compare_msg () {
+	iconv -f "$2" -t "$3" "$TEST_DIRECTORY/t3434/$1" >expect &&
+	git cat-file commit HEAD >raw &&
+	sed "1,/^$/d" raw >actual &&
+	test_cmp expect actual
+}
+
+test_expect_success setup '
+	test_commit one &&
+	git branch first &&
+	test_commit two &&
+	git switch first &&
+	test_commit three &&
+	git branch second &&
+	test_commit four &&
+	git switch second &&
+	test_commit five &&
+	test_commit six
+'
+
+test_expect_success 'rebase --rebase-merges update encoding eucJP to UTF-8' '
+	git switch -c merge-eucJP-UTF-8 first &&
+	git config i18n.commitencoding eucJP &&
+	git merge -F "$TEST_DIRECTORY/t3434/eucJP.txt" second &&
+	git config i18n.commitencoding UTF-8 &&
+	git rebase --rebase-merges master &&
+	compare_msg eucJP.txt eucJP UTF-8
+'
+
+test_expect_failure 'rebase --rebase-merges update encoding eucJP to ISO-2022-JP' '
+	git switch -c merge-eucJP-ISO-2022-JP first &&
+	git config i18n.commitencoding eucJP &&
+	git merge -F "$TEST_DIRECTORY/t3434/eucJP.txt" second &&
+	git config i18n.commitencoding ISO-2022-JP &&
+	git rebase --rebase-merges master &&
+	compare_msg eucJP.txt eucJP ISO-2022-JP
+'
+
+test_rebase_continue_update_encode () {
+	old=$1
+	new=$2
+	msgfile=$3
+	test_expect_success "rebase --continue update from $old to $new" '
+		(git rebase --abort || : abort current git-rebase failure) &&
+		git switch -c conflict-$old-$new one &&
+		echo for-conflict >two.t &&
+		git add two.t &&
+		git config i18n.commitencoding $old &&
+		git commit -F "$TEST_DIRECTORY/t3434/$msgfile" &&
+		git config i18n.commitencoding $new &&
+		test_must_fail git rebase -m master &&
+		test -f .git/rebase-merge/message &&
+		git stripspace <.git/rebase-merge/message >two.t &&
+		git add two.t &&
+		git rebase --continue &&
+		compare_msg $msgfile $old $new &&
+		: git-commit assume invalid utf-8 is latin1 &&
+		test_cmp expect two.t
+	'
+}
+
+test_rebase_continue_update_encode ISO-8859-1 UTF-8 ISO8859-1.txt
+test_rebase_continue_update_encode eucJP UTF-8 eucJP.txt
+test_rebase_continue_update_encode eucJP ISO-2022-JP eucJP.txt
+
+test_done
diff --git a/t/t3434/ISO8859-1.txt b/t/t3434/ISO8859-1.txt
new file mode 100644
index 0000000000..7cbef0ee6f
--- /dev/null
+++ b/t/t3434/ISO8859-1.txt
@@ -0,0 +1,3 @@
+ÄËÑÏÖ
+
+Ábçdèfg
diff --git a/t/t3434/eucJP.txt b/t/t3434/eucJP.txt
new file mode 100644
index 0000000000..546f2aac01
--- /dev/null
+++ b/t/t3434/eucJP.txt
@@ -0,0 +1,4 @@
+¤Ï¤ì¤Ò¤Û¤Õ
+
+¤·¤Æ¤¤¤ë¤Î¤¬¡¢¤¤¤ë¤Î¤Ç¡£
+ßÀÉͤۤì¤×¤ê¤Ý¤ì¤Þ¤Ó¤°¤ê¤í¤Ø¡£
diff --git a/t/t3900-i18n-commit.sh b/t/t3900-i18n-commit.sh
index b92ff95977..d277a9f4b7 100755
--- a/t/t3900-i18n-commit.sh
+++ b/t/t3900-i18n-commit.sh
@@ -204,4 +204,41 @@ test_commit_autosquash_flags eucJP fixup
 
 test_commit_autosquash_flags ISO-2022-JP squash
 
+test_commit_autosquash_multi_encoding () {
+	flag=$1
+	old=$2
+	new=$3
+	msg=$4
+	test_expect_success "commit --$flag into $old from $new" '
+		git checkout -b $flag-$old-$new C0 &&
+		git config i18n.commitencoding $old &&
+		echo $old >>F &&
+		git commit -a -F "$TEST_DIRECTORY"/t3900/$msg &&
+		test_tick &&
+		echo intermediate stuff >>G &&
+		git add G &&
+		git commit -a -m "intermediate commit" &&
+		test_tick &&
+		git config i18n.commitencoding $new &&
+		echo $new-$flag >>F &&
+		git commit -a --$flag HEAD^ &&
+		git rebase --autosquash -i HEAD^^^ &&
+		git rev-list HEAD >actual &&
+		test_line_count = 3 actual &&
+		iconv -f $old -t UTF-8 "$TEST_DIRECTORY"/t3900/$msg >expect &&
+		if test $flag = squash; then
+			subject="$(head -1 expect)" &&
+			printf "\nsquash! %s\n" "$subject" >>expect
+		fi &&
+		git cat-file commit HEAD^ >raw &&
+		(sed "1,/^$/d" raw | iconv -f $new -t utf-8) >actual &&
+		test_cmp expect actual
+	'
+}
+
+test_commit_autosquash_multi_encoding fixup UTF-8 ISO-8859-1 1-UTF-8.txt
+test_commit_autosquash_multi_encoding squash ISO-8859-1 UTF-8 ISO8859-1.txt
+test_commit_autosquash_multi_encoding squash eucJP ISO-2022-JP eucJP.txt
+test_commit_autosquash_multi_encoding fixup ISO-2022-JP UTF-8 ISO-2022-JP.txt
+
 test_done