feat: adding unicode support RFC6532

Sayan751 · Sep 4, 2020 · 5d1f60e · 5d1f60e
1 parent 897b78d
commit 5d1f60e
Show file tree

Hide file tree

Showing 6 changed files with 52 additions and 62 deletions.
diff --git a/rust-lib/build.rs b/rust-lib/build.rs
@@ -377,7 +377,7 @@ macro_rules! generate_is_valid_test {
         #[test]
         #[wasm_bindgen_test]
         fn $case() {
-          assert_eq!(EmailAddress::is_valid(&$address, None), $is_valid, \"expected {} not to be valid: {}\", $address, $is_valid);
+          assert_eq!(EmailAddress::is_valid(&$address, None), $is_valid, \"expected {} to be valid: {}\", $address, $is_valid);
         }
       )*
     }

diff --git a/rust-lib/resources/.test_data/idns.txt b/rust-lib/resources/.test_data/idns.txt
diff --git a/rust-lib/resources/.test_data/valid_domains.txt b/rust-lib/resources/.test_data/valid_domains.txt
@@ -5,4 +5,34 @@ xn--masekowski-d0b.pl
 [192.168.0.1]
 [1.2.3.4]
 [0.0.0.1]
-[255.255.255.254]
+[255.255.255.254]
+masełkowski.pl.com
+中国互联网络信息中心.中国
+中国互联网络信息中心.xn--masekowski-d0b
+bücher.com
+nächste.de
+löwen.de
+افغانستا.com
+বাংলাদেশ.com
+беларусь.com
+belgië.com
+českárepublika.com
+مصر.com
+ελλάδα.com
+ísland.com
+भारत.com
+איקו״ם.ישראל.com
+қазақстан.com
+한국.com
+кыргызстан.com
+ລາວ.com
+македонија.com
+монголулс.com
+россия.иком.com
+இலங்கை.com
+españa.com
+ไทย.com
+việtnam.com
+📻.fm
+🎁.de
+âêîôû.Çéàèù.ëïü
diff --git a/rust-lib/resources/rfc6532.pdf b/rust-lib/resources/rfc6532.pdf
diff --git a/rust-lib/src/email_address.rs b/rust-lib/src/email_address.rs
@@ -15,7 +15,7 @@ impl Default for ParsingOptions {
   fn default() -> Self {
     ParsingOptions {
       is_lax: false,
-      supports_unicode: false,
+      supports_unicode: true,
     }
   }
 }
@@ -364,4 +364,11 @@ mod tests {
     println!("{:#?}", actual);
     assert_eq!(actual.is_err(), false);
   }
+
+  #[test]
+  fn can_parse_idn() {
+    let actual = RFC5322::parse(Rule::domain_complete, "bücher.com");
+    println!("{:#?}", actual);
+    assert_eq!(actual.is_err(), false);
+  }
 }
diff --git a/rust-lib/src/rfc5322.pest b/rust-lib/src/rfc5322.pest
@@ -20,14 +20,23 @@ FWS = _{ ((WSP* ~ CRLF)? ~ WSP+) | obs_FWS }
 DQUOTE = { "\"" }
 quoted_pair = { ("\\" ~ (printable_us_ascii | WSP)) | obs_qp }
 // Printable US-ASCII characters not including "(", ")", or "\".
-ctext = { !("(" | ")" | "\\") ~ (printable_us_ascii | obs_ctext) }
+ctext = { !("(" | ")" | "\\") ~ (printable_us_ascii | UTF8_non_ascii | obs_ctext) }
 ccontent = { ctext | quoted_pair | comment }
 comment = { "(" ~ (FWS? ~ ccontent)* ~ FWS? ~ ")" }
 CFWS = _{ ((FWS? ~ comment)+ ~ FWS?) | FWS }
 
 atext = { ASCII_ALPHANUMERIC 
   | "!" | "#" |"$" | "%" | "&" | "'" |"*" | "+" |"-" | "/" |"=" | "?" |"^" | "_" |"`" | "{" |"|" | "}" |"~"
+  | UTF8_non_ascii
 }
+
+/*------------ UTF8 START -------------*/
+UTF8_2 = { '\u{80}'..'\u{7FF}' }
+UTF8_3 = { '\u{800}'..'\u{FFFF}' }
+UTF8_4 = { '\u{10000}'..'\u{10FFFF}' }
+UTF8_non_ascii = { UTF8_2 | UTF8_3 | UTF8_4 } 
+/*------------ UTF8 END -------------*/
+
 /**
  * The construction dot_atom_text rule is an intentional deviation from the RFC5322.
  * The original rule is `{ atext+ ("." ~ atext+)* }`.
@@ -43,11 +52,11 @@ dot_atom_text = {
 dot_atom = { WSP? ~ dot_atom_text ~ WSP? }
 
 // Printable US-ASCII characters not including "[", "]", or "\".
-dtext = { !("[" | "]" | "\\") ~ (printable_us_ascii | obs_dtext) }
+dtext = { !("[" | "]" | "\\") ~ (printable_us_ascii | UTF8_non_ascii | obs_dtext) }
 domain_literal = { CFWS? ~ "[" ~ (FWS? ~ dtext)* ~ FWS? ~ "]" ~ CFWS? }
 
 // Printable US-ASCII characters not including "\" or the quote character.
-qtext = { !(DQUOTE | "\\" ) ~ (printable_us_ascii | obs_qtext) }
+qtext = { !(DQUOTE | "\\" ) ~ (printable_us_ascii | UTF8_non_ascii | obs_qtext) }
 qcontent = { qtext | quoted_pair }
 quoted_string = { CFWS? ~ DQUOTE ~ (FWS? ~ qcontent)* ~ FWS? ~ DQUOTE ~ CFWS? }