From d8e80be88ff2e90b32017cc88576f93aac552f8d Mon Sep 17 00:00:00 2001 From: zuisong Date: Thu, 25 Apr 2024 10:15:27 +0800 Subject: [PATCH 1/6] improve code style --- src/main.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/main.rs b/src/main.rs index fbbed2d3..24351d6e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,7 +19,7 @@ mod vendored; use std::env; use std::fs::File; use std::io::{self, IsTerminal, Read}; -use std::net::{IpAddr, SocketAddr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::path::PathBuf; use std::process; use std::str::FromStr; @@ -265,23 +265,19 @@ fn run(args: Cli) -> Result { }?); } - if matches!( - args.http_version, - Some(HttpVersion::Http10) | Some(HttpVersion::Http11) - ) { - client = client.http1_only(); - } - - if matches!(args.http_version, Some(HttpVersion::Http2PriorKnowledge)) { - client = client.http2_prior_knowledge(); - } + client = match args.http_version { + Some(HttpVersion::Http10 | HttpVersion::Http11) => client.http1_only(), + Some(HttpVersion::Http2PriorKnowledge) => client.http2_prior_knowledge(), + Some(HttpVersion::Http2) => client, + None => client, + }; let cookie_jar = Arc::new(reqwest_cookie_store::CookieStoreMutex::default()); client = client.cookie_provider(cookie_jar.clone()); client = match (args.ipv4, args.ipv6) { - (true, false) => client.local_address(IpAddr::from_str("0.0.0.0")?), - (false, true) => client.local_address(IpAddr::from_str("::")?), + (true, false) => client.local_address(IpAddr::from(Ipv4Addr::UNSPECIFIED)), + (false, true) => client.local_address(IpAddr::from(Ipv6Addr::UNSPECIFIED)), _ => client, }; From dec58124c70660a1a2ae8401d5c29eb01ce2cc2c Mon Sep 17 00:00:00 2001 From: zuisong Date: Fri, 21 Jun 2024 17:44:05 +0800 Subject: [PATCH 2/6] decode header value with utf-8 --- src/printer.rs | 4 ++-- src/redirect.rs | 4 ++-- src/to_curl.rs | 6 +++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/printer.rs b/src/printer.rs index 515ef950..c83776a4 100644 --- a/src/printer.rs +++ b/src/printer.rs @@ -336,8 +336,8 @@ impl Printer { header_string.push_str(key.as_str()); } header_string.push_str(": "); - match value.to_str() { - Ok(value) => header_string.push_str(value), + match String::from_utf8(value.as_bytes().to_vec()) { + Ok(value) => header_string.push_str(&value), #[allow(clippy::format_push_string)] Err(_) => header_string.push_str(&format!("{:?}", value)), } diff --git a/src/redirect.rs b/src/redirect.rs index 202fef3a..1d0ed924 100644 --- a/src/redirect.rs +++ b/src/redirect.rs @@ -51,8 +51,8 @@ fn get_next_request(mut request: Request, response: &Response) -> Option Result { if value.is_empty() { cmd.arg(format!("{};", header)); } else { - cmd.arg(format!("{}: {}", header, value.to_str()?)); + cmd.arg(format!( + "{}: {}", + header, + String::from_utf8(value.as_bytes().to_vec())? + )); } } for header in headers_to_unset { From 697422a58c24eb296e050e4bfd8e2f0db6c5d680 Mon Sep 17 00:00:00 2001 From: zuisong Date: Fri, 21 Jun 2024 22:06:36 +0800 Subject: [PATCH 3/6] apply suggestion --- src/printer.rs | 5 +++-- src/redirect.rs | 4 ++-- src/to_curl.rs | 8 ++------ src/utils.rs | 12 ++++++++++++ 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/printer.rs b/src/printer.rs index c83776a4..a139449e 100644 --- a/src/printer.rs +++ b/src/printer.rs @@ -13,6 +13,7 @@ use reqwest::header::{ use reqwest::Version; use url::Url; +use crate::utils::HeaderValueExt; use crate::{ buffer::Buffer, cli::FormatOptions, @@ -336,8 +337,8 @@ impl Printer { header_string.push_str(key.as_str()); } header_string.push_str(": "); - match String::from_utf8(value.as_bytes().to_vec()) { - Ok(value) => header_string.push_str(&value), + match value.to_utf8_str() { + Ok(value) => header_string.push_str(value), #[allow(clippy::format_push_string)] Err(_) => header_string.push_str(&format!("{:?}", value)), } diff --git a/src/redirect.rs b/src/redirect.rs index 1d0ed924..cb63c707 100644 --- a/src/redirect.rs +++ b/src/redirect.rs @@ -7,7 +7,7 @@ use reqwest::header::{ use reqwest::{Method, StatusCode, Url}; use crate::middleware::{Context, Middleware}; -use crate::utils::clone_request; +use crate::utils::{clone_request, HeaderValueExt}; pub struct RedirectFollower { max_redirects: usize, @@ -51,7 +51,7 @@ fn get_next_request(mut request: Request, response: &Response) -> Option Result<()> { let cmd = translate(args)?; @@ -308,11 +308,7 @@ pub fn translate(args: Cli) -> Result { if value.is_empty() { cmd.arg(format!("{};", header)); } else { - cmd.arg(format!( - "{}: {}", - header, - String::from_utf8(value.as_bytes().to_vec())? - )); + cmd.arg(format!("{}: {}", header, value.to_utf8_str()?)); } } for header in headers_to_unset { diff --git a/src/utils.rs b/src/utils.rs index 137fb3af..5ebd3f25 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -2,8 +2,10 @@ use std::borrow::Cow; use std::env::var_os; use std::io::{self, Write}; use std::path::{Path, PathBuf}; +use std::str::Utf8Error; use anyhow::Result; +use reqwest::header::HeaderValue; use reqwest::blocking::Request; use url::Url; @@ -175,3 +177,13 @@ pub fn copy_largebuf( } } } + +pub(crate) trait HeaderValueExt { + fn to_utf8_str(&self) -> Result<&str, Utf8Error>; +} + +impl HeaderValueExt for HeaderValue { + fn to_utf8_str(&self) -> Result<&str, Utf8Error> { + std::str::from_utf8(self.as_bytes()) + } +} From cb44ff5e8a922a81cbf8bf3c90b29e9f00e30182 Mon Sep 17 00:00:00 2001 From: zuisong Date: Fri, 21 Jun 2024 22:54:53 +0800 Subject: [PATCH 4/6] add test case --- src/redirect.rs | 2 +- src/utils.rs | 2 +- tests/cli.rs | 84 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/src/redirect.rs b/src/redirect.rs index cb63c707..1e983594 100644 --- a/src/redirect.rs +++ b/src/redirect.rs @@ -52,7 +52,7 @@ fn get_next_request(mut request: Request, response: &Response) -> Option String { diff --git a/tests/cli.rs b/tests/cli.rs index 113c213e..3414eae0 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1657,6 +1657,90 @@ fn body_from_raw() { .success(); } +#[test] +fn support_utf8_header_value() { + let server = server::http(|req| async move { + assert_eq!(req.headers()["hello"].as_bytes(), "你好".as_bytes()); + hyper::Response::builder() + // Valid JSON, but not declared as text + .header("hello", "你好呀") + .body("".into()) + .unwrap() + }); + + get_command() + .args([&server.base_url(), "hello:你好"]) + .assert() + .stdout(contains("Hello: 你好呀")) + .success(); +} + +#[test] +fn redirect_support_utf8_location() { + let server = server::http(|req| async move { + match req.uri().path() { + "/first_page" => hyper::Response::builder() + .status(302) + .header("Date", "N/A") + .header("Location", "/page二") + .body("redirecting...".into()) + .unwrap(), + "/page%E4%BA%8C" => hyper::Response::builder() + .header("Date", "N/A") + .body("final destination".into()) + .unwrap(), + _ => panic!("unknown path"), + } + }); + + get_command() + .args([&server.url("/first_page"), "--follow", "--verbose", "--all"]) + .assert() + .stdout(indoc! {r#" + GET /first_page HTTP/1.1 + Accept: */* + Accept-Encoding: gzip, deflate, br, zstd + Connection: keep-alive + Host: http.mock + User-Agent: xh/0.0.0 (test mode) + + HTTP/1.1 302 Found + Content-Length: 14 + Date: N/A + Location: /page二 + + redirecting... + + GET /page%E4%BA%8C HTTP/1.1 + Accept: */* + Accept-Encoding: gzip, deflate, br, zstd + Connection: keep-alive + Host: http.mock + User-Agent: xh/0.0.0 (test mode) + + HTTP/1.1 200 OK + Content-Length: 17 + Date: N/A + + final destination + "#}); +} + +#[test] +fn to_curl_support_utf8_header_value() { + get_command() + .args(["https://exmaple.com/", "hello:你好", "--curl"]) + .assert() + .stdout(contains("curl https://exmaple.com/ -H 'hello: 你好'")) + .success(); + + get_command() + .args(["https://exmaple.com/", "hello:你好", "--curl-long"]) + .assert() + .stdout(contains("curl https://exmaple.com/ --header 'hello: 你好'")) + .success(); +} + #[test] fn mixed_stdin_request_items() { redirecting_command() From 3f24597737f25e7e1b444209851b1fa8d8e913dd Mon Sep 17 00:00:00 2001 From: zuisong Date: Sat, 22 Jun 2024 00:09:35 +0800 Subject: [PATCH 5/6] decode CONTENT_DISPOSITION header value with utf-8 download file support unicode file name --- src/download.rs | 8 ++++++-- tests/cli.rs | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/download.rs b/src/download.rs index 251c6852..bb2ede54 100644 --- a/src/download.rs +++ b/src/download.rs @@ -14,7 +14,7 @@ use reqwest::{ }; use crate::decoder::{decompress, get_compression_type}; -use crate::utils::{copy_largebuf, test_pretend_term}; +use crate::utils::{copy_largebuf, test_pretend_term, HeaderValueExt}; fn get_content_length(headers: &HeaderMap) -> Option { headers @@ -31,7 +31,11 @@ fn get_file_name(response: &Response, orig_url: &reqwest::Url) -> String { // Against the spec, but used by e.g. Github's zip downloads let unquoted = Regex::new("filename=([^;=\"]*)").unwrap(); - let header = response.headers().get(CONTENT_DISPOSITION)?.to_str().ok()?; + let header = response + .headers() + .get(CONTENT_DISPOSITION)? + .to_utf8_str() + .ok()?; let caps = quoted .captures(header) .or_else(|| unquoted.captures(header))?; diff --git a/tests/cli.rs b/tests/cli.rs index 3414eae0..2507fcfe 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -530,6 +530,27 @@ fn download_supplied_filename() { ); } +#[test] +fn download_supplied_unicode_filename() { + let dir = tempdir().unwrap(); + let server = server::http(|_req| async move { + hyper::Response::builder() + .header("Content-Disposition", r#"attachment; filename="😀.bar""#) + .body("file".into()) + .unwrap() + }); + + get_command() + .args(["--download", &server.base_url()]) + .current_dir(&dir) + .assert() + .success(); + assert_eq!( + fs::read_to_string(dir.path().join("😀.bar")).unwrap(), + "file" + ); +} + #[test] fn download_supplied_unquoted_filename() { let dir = tempdir().unwrap(); From 6c80a493538c57f35b00746072b785ba8cc290b0 Mon Sep 17 00:00:00 2001 From: zuisong Date: Sat, 22 Jun 2024 09:17:48 +0800 Subject: [PATCH 6/6] apply suggestion --- src/printer.rs | 3 +-- src/to_curl.rs | 4 ++++ tests/cli.rs | 30 +++++++++++------------------- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/printer.rs b/src/printer.rs index a139449e..515ef950 100644 --- a/src/printer.rs +++ b/src/printer.rs @@ -13,7 +13,6 @@ use reqwest::header::{ use reqwest::Version; use url::Url; -use crate::utils::HeaderValueExt; use crate::{ buffer::Buffer, cli::FormatOptions, @@ -337,7 +336,7 @@ impl Printer { header_string.push_str(key.as_str()); } header_string.push_str(": "); - match value.to_utf8_str() { + match value.to_str() { Ok(value) => header_string.push_str(value), #[allow(clippy::format_push_string)] Err(_) => header_string.push_str(&format!("{:?}", value)), diff --git a/src/to_curl.rs b/src/to_curl.rs index a06aecad..99c52265 100644 --- a/src/to_curl.rs +++ b/src/to_curl.rs @@ -544,6 +544,10 @@ mod tests { ( "xh http://example.com/[1-100].png?q={80,90}", "curl -g 'http://example.com/[1-100].png?q={80,90}'", + ), + ( + "xh https://exmaple.com/ hello:你好", + "curl https://exmaple.com/ -H 'hello: 你好'" ) ]; for (input, output) in expected { diff --git a/tests/cli.rs b/tests/cli.rs index 2507fcfe..f7bda253 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1683,8 +1683,8 @@ fn support_utf8_header_value() { let server = server::http(|req| async move { assert_eq!(req.headers()["hello"].as_bytes(), "你好".as_bytes()); hyper::Response::builder() - // Valid JSON, but not declared as text .header("hello", "你好呀") + .header("Date", "N/A") .body("".into()) .unwrap() }); @@ -1692,7 +1692,14 @@ fn support_utf8_header_value() { get_command() .args([&server.base_url(), "hello:你好"]) .assert() - .stdout(contains("Hello: 你好呀")) + .stdout(indoc! {r#" + HTTP/1.1 200 OK + Content-Length: 0 + Date: N/A + Hello: "\xe4\xbd\xa0\xe5\xa5\xbd\xe5\x91\x80" + + + "#}) .success(); } @@ -1728,7 +1735,7 @@ fn redirect_support_utf8_location() { HTTP/1.1 302 Found Content-Length: 14 Date: N/A - Location: /page二 + Location: "/page\xe4\xba\x8c" redirecting... @@ -1747,21 +1754,6 @@ fn redirect_support_utf8_location() { "#}); } -#[test] -fn to_curl_support_utf8_header_value() { - get_command() - .args(["https://exmaple.com/", "hello:你好", "--curl"]) - .assert() - .stdout(contains("curl https://exmaple.com/ -H 'hello: 你好'")) - .success(); - - get_command() - .args(["https://exmaple.com/", "hello:你好", "--curl-long"]) - .assert() - .stdout(contains("curl https://exmaple.com/ --header 'hello: 你好'")) - .success(); -} - #[test] fn mixed_stdin_request_items() { redirecting_command() @@ -3699,7 +3691,7 @@ fn multiple_format_options_are_merged() { get_command() .arg("--format-options=json.indent:2,json.indent:8") .arg("--format-options=headers.sort:false") - .arg(&server.base_url()) + .arg(server.base_url()) .assert() .stdout(indoc! {r#" HTTP/1.1 200 OK