Merge pull request #375 from zuisong/utf8-hader-value

decode header value with utf-8

src/download.rs

@@ -14,7 +14,7 @@ use reqwest::{
 };
 
 use crate::decoder::{decompress, get_compression_type};
-use crate::utils::{copy_largebuf, test_pretend_term};
+use crate::utils::{copy_largebuf, test_pretend_term, HeaderValueExt};
 
 fn get_content_length(headers: &HeaderMap) -> Option<u64> {
     headers
@@ -31,7 +31,11 @@ fn get_file_name(response: &Response, orig_url: &reqwest::Url) -> String {
         // Against the spec, but used by e.g. Github's zip downloads
         let unquoted = Regex::new("filename=([^;=\"]*)").unwrap();
 
-        let header = response.headers().get(CONTENT_DISPOSITION)?.to_str().ok()?;
+        let header = response
+            .headers()
+            .get(CONTENT_DISPOSITION)?
+            .to_utf8_str()
+            .ok()?;
         let caps = quoted
             .captures(header)
             .or_else(|| unquoted.captures(header))?;

src/main.rs

@@ -19,7 +19,7 @@ mod vendored;
 use std::env;
 use std::fs::File;
 use std::io::{self, IsTerminal, Read};
-use std::net::{IpAddr, SocketAddr};
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
 use std::path::PathBuf;
 use std::process;
 use std::str::FromStr;
@@ -265,23 +265,19 @@ fn run(args: Cli) -> Result<i32> {
         }?);
     }
 
-    if matches!(
-        args.http_version,
-        Some(HttpVersion::Http10) | Some(HttpVersion::Http11)
-    ) {
-        client = client.http1_only();
-    }
-
-    if matches!(args.http_version, Some(HttpVersion::Http2PriorKnowledge)) {
-        client = client.http2_prior_knowledge();
-    }
+    client = match args.http_version {
+        Some(HttpVersion::Http10 | HttpVersion::Http11) => client.http1_only(),
+        Some(HttpVersion::Http2PriorKnowledge) => client.http2_prior_knowledge(),
+        Some(HttpVersion::Http2) => client,
+        None => client,
+    };
 
     let cookie_jar = Arc::new(reqwest_cookie_store::CookieStoreMutex::default());
     client = client.cookie_provider(cookie_jar.clone());
 
     client = match (args.ipv4, args.ipv6) {
-        (true, false) => client.local_address(IpAddr::from_str("0.0.0.0")?),
-        (false, true) => client.local_address(IpAddr::from_str("::")?),
+        (true, false) => client.local_address(IpAddr::from(Ipv4Addr::UNSPECIFIED)),
+        (false, true) => client.local_address(IpAddr::from(Ipv6Addr::UNSPECIFIED)),
         _ => client,
     };
 

src/redirect.rs

@@ -7,7 +7,7 @@ use reqwest::header::{
 use reqwest::{Method, StatusCode, Url};
 
 use crate::middleware::{Context, Middleware};
-use crate::utils::clone_request;
+use crate::utils::{clone_request, HeaderValueExt};
 
 pub struct RedirectFollower {
     max_redirects: usize,
@@ -51,7 +51,7 @@ fn get_next_request(mut request: Request, response: &Response) -> Option<Request
         response
             .headers()
             .get(LOCATION)
-            .and_then(|location| location.to_str().ok())
+            .and_then(|location| location.to_utf8_str().ok())
             .and_then(|location| request.url().join(location).ok())
     };
 

src/to_curl.rs

@@ -7,7 +7,7 @@ use std::ffi::OsString;
 
 use crate::cli::{AuthType, Cli, HttpVersion, Verify};
 use crate::request_items::{Body, RequestItem, FORM_CONTENT_TYPE, JSON_ACCEPT, JSON_CONTENT_TYPE};
-use crate::utils::url_with_query;
+use crate::utils::{url_with_query, HeaderValueExt};
 
 pub fn print_curl_translation(args: Cli) -> Result<()> {
     let cmd = translate(args)?;
@@ -308,7 +308,7 @@ pub fn translate(args: Cli) -> Result<Command> {
         if value.is_empty() {
             cmd.arg(format!("{};", header));
         } else {
-            cmd.arg(format!("{}: {}", header, value.to_str()?));
+            cmd.arg(format!("{}: {}", header, value.to_utf8_str()?));
         }
     }
     for header in headers_to_unset {
@@ -543,6 +543,10 @@ mod tests {
             (
                 "xh http://example.com/[1-100].png?q={80,90}",
                 "curl -g 'http://example.com/[1-100].png?q={80,90}'",
+            ),
+            (
+                "xh https://exmaple.com/ hello:你好",
+                "curl https://exmaple.com/ -H 'hello: 你好'"
             )
         ];
         for (input, output) in expected {

src/utils.rs

@@ -2,9 +2,11 @@ use std::borrow::Cow;
 use std::env::var_os;
 use std::io::{self, Write};
 use std::path::{Path, PathBuf};
+use std::str::Utf8Error;
 
 use anyhow::Result;
 use reqwest::blocking::Request;
+use reqwest::header::HeaderValue;
 use url::Url;
 
 pub fn unescape(text: &str, special_chars: &'static str) -> String {
@@ -175,3 +177,13 @@ pub fn copy_largebuf(
         }
     }
 }
+
+pub(crate) trait HeaderValueExt {
+    fn to_utf8_str(&self) -> Result<&str, Utf8Error>;
+}
+
+impl HeaderValueExt for HeaderValue {
+    fn to_utf8_str(&self) -> Result<&str, Utf8Error> {
+        std::str::from_utf8(self.as_bytes())
+    }
+}

tests/cli.rs

@@ -530,6 +530,27 @@ fn download_supplied_filename() {
     );
 }
 
+#[test]
+fn download_supplied_unicode_filename() {
+    let dir = tempdir().unwrap();
+    let server = server::http(|_req| async move {
+        hyper::Response::builder()
+            .header("Content-Disposition", r#"attachment; filename="😀.bar""#)
+            .body("file".into())
+            .unwrap()
+    });
+
+    get_command()
+        .args(["--download", &server.base_url()])
+        .current_dir(&dir)
+        .assert()
+        .success();
+    assert_eq!(
+        fs::read_to_string(dir.path().join("😀.bar")).unwrap(),
+        "file"
+    );
+}
+
 #[test]
 fn download_supplied_unquoted_filename() {
     let dir = tempdir().unwrap();
@@ -1657,6 +1678,82 @@ fn body_from_raw() {
         .success();
 }
 
+#[test]
+fn support_utf8_header_value() {
+    let server = server::http(|req| async move {
+        assert_eq!(req.headers()["hello"].as_bytes(), "你好".as_bytes());
+        hyper::Response::builder()
+            .header("hello", "你好呀")
+            .header("Date", "N/A")
+            .body("".into())
+            .unwrap()
+    });
+
+    get_command()
+        .args([&server.base_url(), "hello:你好"])
+        .assert()
+        .stdout(indoc! {r#"
+        HTTP/1.1 200 OK
+        Content-Length: 0
+        Date: N/A
+        Hello: "\xe4\xbd\xa0\xe5\xa5\xbd\xe5\x91\x80"
+
+
+        "#})
+        .success();
+}
+
+#[test]
+fn redirect_support_utf8_location() {
+    let server = server::http(|req| async move {
+        match req.uri().path() {
+            "/first_page" => hyper::Response::builder()
+                .status(302)
+                .header("Date", "N/A")
+                .header("Location", "/page二")
+                .body("redirecting...".into())
+                .unwrap(),
+            "/page%E4%BA%8C" => hyper::Response::builder()
+                .header("Date", "N/A")
+                .body("final destination".into())
+                .unwrap(),
+            _ => panic!("unknown path"),
+        }
+    });
+
+    get_command()
+        .args([&server.url("/first_page"), "--follow", "--verbose", "--all"])
+        .assert()
+        .stdout(indoc! {r#"
+            GET /first_page HTTP/1.1
+            Accept: */*
+            Accept-Encoding: gzip, deflate, br, zstd
+            Connection: keep-alive
+            Host: http.mock
+            User-Agent: xh/0.0.0 (test mode)
+
+            HTTP/1.1 302 Found
+            Content-Length: 14
+            Date: N/A
+            Location: "/page\xe4\xba\x8c"
+
+            redirecting...
+
+            GET /page%E4%BA%8C HTTP/1.1
+            Accept: */*
+            Accept-Encoding: gzip, deflate, br, zstd
+            Connection: keep-alive
+            Host: http.mock
+            User-Agent: xh/0.0.0 (test mode)
+
+            HTTP/1.1 200 OK
+            Content-Length: 17
+            Date: N/A
+
+            final destination
+        "#});
+}
+
 #[test]
 fn mixed_stdin_request_items() {
     redirecting_command()
@@ -3594,7 +3691,7 @@ fn multiple_format_options_are_merged() {
     get_command()
         .arg("--format-options=json.indent:2,json.indent:8")
         .arg("--format-options=headers.sort:false")
-        .arg(&server.base_url())
+        .arg(server.base_url())
         .assert()
         .stdout(indoc! {r#"
             HTTP/1.1 200 OK