Add IDN support

.github/workflows/almalinux-8-build.yml

@@ -48,7 +48,8 @@ jobs:
             attr libattr-devel acl libacl-devel \
             zstd libzstd-devel \
             lz4 lz4-devel \
-            xxhash xxhash-devel
+            xxhash xxhash-devel \
+            libidn2 libidn2-devel
         alternatives --set python3 /usr/bin/python3.9
         pip3 install commonmark
     - name: configure

.github/workflows/android-static-build.yml

@@ -76,7 +76,7 @@ jobs:
           # checksums and its bundled zlib.
           ./configure --host=${{ matrix.triple }} --build=x86_64-pc-linux-gnu \
             --enable-ipv6 \
-            --disable-zstd --disable-lz4 --disable-xxhash --disable-openssl \
+            --disable-zstd --disable-lz4 --disable-xxhash --disable-openssl --disable-idn \
             --disable-iconv --disable-iconv-open \
             --disable-acl-support --disable-xattr-support \
             --disable-md2man --disable-roll-simd \

.github/workflows/asan-build.yml

@@ -41,7 +41,7 @@ jobs:
     - name: prep
       run: |
         sudo apt-get update
-        sudo apt-get install -y clang acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev openssl
+        sudo apt-get install -y clang acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev openssl
         echo "/usr/local/bin" >>"$GITHUB_PATH"
     - name: configure
       # -DNDEBUG builds as a shipped release does (assert() compiled out), so

.github/workflows/coverage.yml

@@ -26,7 +26,7 @@ jobs:
     - name: prep
       run: |
         sudo apt-get update
-        sudo apt-get install -y acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl gcovr
+        sudo apt-get install -y acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl gcovr
         echo "/usr/local/bin" >>"$GITHUB_PATH"
     - name: configure
       run: ./configure --enable-coverage --with-rrsync

.github/workflows/cygwin-build.yml

@@ -26,7 +26,7 @@ jobs:
       run: choco install -y --no-progress cygwin cyg-get
     - name: prep
       run: |
-        cyg-get make autoconf automake gcc-core attr libattr-devel python39 python39-pip libzstd-devel liblz4-devel libssl-devel libxxhash0 libxxhash-devel
+        cyg-get make autoconf automake gcc-core attr libattr-devel python39 python39-pip libzstd-devel liblz4-devel libssl-devel libxxhash0 libxxhash-devel libidn2-devel
         echo "C:/tools/cygwin/bin" >>$Env:GITHUB_PATH
     - name: commonmark
       run: bash -c 'python3 -mpip install --user commonmark'

.github/workflows/fleettest.yml

@@ -33,7 +33,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y gcc g++ gawk autoconf automake \
-          acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev \
+          acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev \
           python3-cmarkgfm openssl rsync openssh-server
     - name: set up ssh to localhost
       run: |

.github/workflows/freebsd-build.yml

@@ -31,7 +31,7 @@ jobs:
           pkg install -y bash autotools m4 devel/xxhash zstd liblz4 python3 archivers/liblz4 git
         run: |
           freebsd-version
-          ./configure --with-rrsync -disable-zstd --disable-md2man --disable-xxhash --disable-lz4
+          ./configure --with-rrsync -disable-zstd --disable-md2man --disable-xxhash --disable-lz4 --disable-idn
           make
           ./rsync --version
           make check

.github/workflows/macos-build.yml

@@ -24,7 +24,7 @@ jobs:
         fetch-depth: 0
     - name: prep
       run: |
-        brew install automake openssl xxhash zstd lz4
+        brew install automake openssl xxhash zstd lz4 libidn2
         pip3 install --user --break-system-packages commonmark
         echo "$(brew --prefix)/bin" >>"$GITHUB_PATH"
     - name: configure

.github/workflows/netbsd-build.yml

@@ -32,7 +32,7 @@ jobs:
           ln -sf /usr/pkg/bin/python3.12 /usr/pkg/bin/python3
         run: |
           uname -a
-          ./configure --with-rrsync --disable-zstd --disable-md2man --disable-xxhash --disable-lz4
+          ./configure --with-rrsync --disable-zstd --disable-md2man --disable-xxhash --disable-lz4 --disable-idn
           make
           ./rsync --version
           make check

.github/workflows/openbsd-build.yml

@@ -33,7 +33,7 @@ jobs:
           uname -a
           export AUTOCONF_VERSION=2.71
           export AUTOMAKE_VERSION=1.16
-          ./configure --with-rrsync --disable-zstd --disable-md2man --disable-xxhash --disable-lz4
+          ./configure --with-rrsync --disable-zstd --disable-md2man --disable-xxhash --disable-lz4 --disable-idn
           make
           ./rsync --version
           make check

.github/workflows/scan-build.yml

@@ -30,7 +30,7 @@ jobs:
     - name: prep
       run: |
         sudo apt-get update
-        sudo apt-get install -y clang-18 clang-tools-18 acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libpopt-dev openssl
+        sudo apt-get install -y clang-18 clang-tools-18 acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev libpopt-dev openssl
     - name: configure (under scan-build)
       # Run configure under scan-build so its analyzer compiler-wrapper is baked
       # into the Makefile's $(CC); --disable-md2man avoids the doc toolchain.

.github/workflows/solaris-build.yml

@@ -31,7 +31,7 @@ jobs:
           pkg install bash automake gnu-m4 pkg://solaris/runtime/python-35 autoconf gcc git
         run: |
           uname -a
-          ./configure --with-rrsync -disable-zstd --disable-md2man --disable-xxhash --disable-lz4
+          ./configure --with-rrsync -disable-zstd --disable-md2man --disable-xxhash --disable-lz4 --disable-idn
           make
           ./rsync --version
           make check

.github/workflows/ubuntu-22.04-build.yml

@@ -28,7 +28,7 @@ jobs:
         fetch-depth: 0
     - name: prep
       run: |
-        sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl
+        sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl
         echo "/usr/local/bin" >>"$GITHUB_PATH"
     - name: configure
       run: ./configure --with-rrsync

.github/workflows/ubuntu-build.yml

@@ -24,7 +24,7 @@ jobs:
         fetch-depth: 0
     - name: prep
       run: |
-        sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl
+        sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl
         echo "/usr/local/bin" >>"$GITHUB_PATH"
     - name: configure
       run: ./configure --with-rrsync

.github/workflows/ubuntu-version-mix.yml

@@ -45,7 +45,7 @@ jobs:
         fetch-depth: 0
     - name: prep
       run: |
-        sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl
+        sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl
         echo "/usr/local/bin" >>"$GITHUB_PATH"
     - name: configure
       run: ./configure --with-rrsync

.github/workflows/valgrind.yml

@@ -33,7 +33,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y valgrind acl libacl1-dev attr libattr1-dev \
-          liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl
+          liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl
         echo "/usr/local/bin" >>"$GITHUB_PATH"
     - name: configure
       run: ./configure --with-rrsync --enable-debug

INSTALL.md

@@ -114,6 +114,16 @@ checksums.
 
 [4]: https://www.openssl.org/docs/man1.0.2/man3/crypto.html
 
+## libidn2
+
+The [libidn2 library][5] converts an internationalized domain name into the
+IDNA A-label ("Punycode") form that a resolver understands.  Installing this
+development library lets rsync connect to a daemon whose name has non-ASCII
+characters in it, and lets a daemon's "hosts allow" & "hosts deny" settings be
+written the same way.
+
+[5]: https://www.gnu.org/software/libidn/#libidn2
+
 ## Package summary
 
 To help you get the libraries installed, here are some package install commands
@@ -131,6 +141,7 @@ like.
     >     sudo apt install -y libzstd-dev
     >     sudo apt install -y liblz4-dev
     >     sudo apt install -y libssl-dev
+    >     sudo apt install -y libidn2-dev
 
 Or run support/install_deps_ubuntu.sh
 
@@ -144,6 +155,7 @@ Or run support/install_deps_ubuntu.sh
     >     sudo yum -y install libzstd-devel
     >     sudo yum -y install lz4-devel
     >     sudo yum -y install openssl-devel
+    >     sudo yum -y install libidn2-devel
     >     python3 -mpip install --user commonmark
 
  -  For Fedora 33:
@@ -154,13 +166,15 @@ Or run support/install_deps_ubuntu.sh
     >     sudo dnf -y install libzstd-devel
     >     sudo dnf -y install lz4-devel
     >     sudo dnf -y install openssl-devel
+    >     sudo dnf -y install libidn2-devel
 
  -  For FreeBSD (this assumes that the python3 version is 3.7):
 
     >     sudo pkg install -y autotools python3 py37-CommonMark
     >     sudo pkg install -y xxhash
     >     sudo pkg install -y zstd
     >     sudo pkg install -y liblz4
+    >     sudo pkg install -y libidn2
 
  -  For macOS:
 
@@ -169,6 +183,7 @@ Or run support/install_deps_ubuntu.sh
     >     brew install zstd
     >     brew install lz4
     >     brew install openssl
+    >     brew install libidn2
 
  -  For Cygwin (with all cygwin programs stopped, run the appropriate setup program from a cmd shell):
 
@@ -177,6 +192,7 @@ Or run support/install_deps_ubuntu.sh
     >     setup-x86_64 --quiet-mode -P libzstd-devel
     >     setup-x86_64 --quiet-mode -P liblz4-devel
     >     setup-x86_64 --quiet-mode -P libssl-devel
+    >     setup-x86_64 --quiet-mode -P libidn2-devel
 
     Sometimes cygwin has commonmark packaged and sometimes it doesn't. Now that
     its python38 has stabilized, you could install python38-commonmark. Or just

access.c

@@ -33,6 +33,9 @@ static int match_hostname(const char **host_ptr, const char *addr, const char *t
 	struct hostent *hp;
 	unsigned int i;
 	const char *host = *host_ptr;
+#ifdef SUPPORT_IDN
+	char idn_tok[1024];
+#endif
 
 	if (!host || !*host)
 		return 0;
@@ -42,6 +45,14 @@ static int match_hostname(const char **host_ptr, const char *addr, const char *t
 		return innetgr(tok + 1, host, NULL, NULL);
 #endif
 
+#ifdef SUPPORT_IDN
+	/* A hostname reaches us from DNS as ASCII, so fold an IDN token to its
+	 * A-label form before comparing.  An all-ASCII token, and a token we
+	 * can't fold, are both left as they are. */
+	if (idn_to_ascii(tok, 0, idn_tok, sizeof idn_tok))
+		tok = idn_tok;
+#endif
+
 	/* First check if the reverse-DNS-determined hostname matches. */
 	if (iwildmatch(tok, host))
 		return 1;

configure.ac

@@ -13,7 +13,7 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \
     sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h dl.h \
     popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netgroup.h \
     zlib.h xxhash.h openssl/md4.h openssl/md5.h zstd.h lz4.h sys/file.h \
-    bsd/string.h)
+    bsd/string.h idn2.h)
 AC_CHECK_HEADERS([netinet/ip.h], [], [], [[#include <netinet/in.h>]])
 AC_HEADER_MAJOR_FIXED
 
@@ -626,6 +626,27 @@ else
     AC_MSG_RESULT(no)
 fi
 
+AC_MSG_CHECKING([whether to enable IDN support])
+AC_ARG_ENABLE([idn],
+        AS_HELP_STRING([--disable-idn], [disable to omit IDN (Internationalized Domain Name) support]))
+AH_TEMPLATE([SUPPORT_IDN],
+[Undefine if you do not want IDN support.  By default this is defined.])
+if test x"$enable_idn" != x"no"; then
+    if test x"$ac_cv_header_idn2_h" = x"yes"; then
+	AC_MSG_RESULT(yes)
+	AC_SEARCH_LIBS(idn2_lookup_ul, idn2,
+	    [AC_DEFINE(SUPPORT_IDN)],
+	    [err_msg="$err_msg$nl- Failed to find idn2_lookup_ul function in idn2 lib.";
+	     no_lib="$no_lib idn"])
+    else
+	AC_MSG_RESULT(no)
+	err_msg="$err_msg$nl- Failed to find idn2.h for IDN support."
+	no_lib="$no_lib idn"
+    fi
+else
+    AC_MSG_RESULT(no)
+fi
+
 if test x"$no_lib" != x; then
     echo ""
     echo "Configure found the following issues:"

main.c

@@ -517,6 +517,16 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in
 	char *args[MAX_ARGS], *need_to_free = NULL;
 	pid_t pid;
 	int dash_l_set = 0;
+#ifdef SUPPORT_IDN
+	char idn_machine[1024];
+
+	/* A daemon-over-remote-shell host is ours to resolve, so give the helper
+	 * the A-label form.  A "host:path" transfer is left alone because that
+	 * name belongs to the user's ssh, which may be matching it against an
+	 * ssh_config Host pattern. */
+	if (machine && daemon_connection > 0 && idn_to_ascii(machine, 1, idn_machine, sizeof idn_machine))
+		machine = idn_machine;
+#endif
 
 	if (!read_batch && !local_server) {
 		char *t, *f, in_quote = '\0';

rsync.1.md

@@ -321,6 +321,19 @@ when scripting rsync.
 WARNING: On some systems environment variables are visible to all users.  On
 those systems using [`--password-file`](#opt) is recommended.
 
+If rsync was built with IDN support (look for "IDN" in `rsync --version`), the
+daemon host may contain non-ASCII characters: those labels are converted to
+their IDNA A-label ("Punycode") form before the name is looked up.  The name is
+read using your locale's character encoding, so be sure your locale is set
+correctly.  Only the non-ASCII labels change, so an address literal, a name you
+punycoded yourself, and a name that is not a valid IDN are all looked up just
+as you typed them.
+
+This applies to the host of a daemon connection only.  The host of a plain
+remote-shell transfer (the single-colon syntax) is passed to your remote-shell
+program unchanged, since that name may well be an ssh_config "Host" alias
+rather than a DNS name.
+
 You may establish the connection via a web proxy by setting the environment
 variable [`RSYNC_PROXY`](#) to a hostname:port pair pointing to your web proxy.
 Note that your web proxy's configuration must support proxy connections to port

rsyncd.conf.5.md

@@ -764,6 +764,18 @@ in the values of parameters.  See that section for details.
     - an '@' followed by a netgroup name, which will match if the reverse DNS
       of the connecting IP is in the specified netgroup.
 
+    If rsync was built with IDN support (look for "IDN" in `rsync --version`),
+    a hostname pattern may contain non-ASCII characters: this file is read as
+    UTF-8, and each non-ASCII label is converted to its IDNA A-label
+    ("Punycode") form before matching, since the name the daemon has for a
+    client always reaches it from DNS as ASCII.  Only those labels change, so
+    an address, a mask, an already-punycoded name, and the wildcard characters
+    are all matched exactly as written.  A pattern that cannot be converted
+    that way is matched as it stands, and thus matches nothing.  That includes
+    a pattern whose conversion would have introduced a character it was not
+    written with, such as the U+FF0A FULLWIDTH ASTERISK that the IDNA mapping
+    turns into a "*".
+
     Note IPv6 link-local addresses can have a scope in the address
     specification:
 

socket.c

@@ -196,6 +196,14 @@ int open_socket_out(char *host, int port, const char *bind_addr, int af_hint)
 	int proxied = 0;
 	char buffer[1024];
 	char *proxy_user = NULL, *proxy_pass = NULL;
+#ifdef SUPPORT_IDN
+	char idn_host[1024];
+
+	/* The resolver only speaks ASCII, so an IDN host goes out as A-labels.
+	 * An all-ASCII host is passed along untouched. */
+	if (idn_to_ascii(host, 1, idn_host, sizeof idn_host))
+		host = idn_host;
+#endif
 
 	/* if we have a RSYNC_PROXY env variable then redirect our
 	 * connection via a web proxy at the given address. */

support/install_deps_ubuntu.sh

@@ -9,3 +9,4 @@ sudo apt install -y libxxhash-dev
 sudo apt install -y libzstd-dev
 sudo apt install -y liblz4-dev
 sudo apt install -y libssl-dev
+sudo apt install -y libidn2-dev