Merging dev into main (#19)

* Updated README to include blocked-list and restructured * Adding minor suggestion by Guy Co-authored-by: Guy Dumais <[email protected]> * update the readme with the blocked-list details * update with new REST route * Update README.md Co-authored-by: koryf <[email protected]> * Update README.md Co-authored-by: koryf <[email protected]> * keeping the old REST path until 3.0 is released. * Update pii_dict format according to deid 3.0.0beta3 * Fix failing pre-commit hook fails on directories (#17) * Add test for get flagged lines * Skip PII flag check for directories --------- Co-authored-by: ketakipai <[email protected]> Co-authored-by: ketakipai <[email protected]> Co-authored-by: Guy Dumais <[email protected]> Co-authored-by: Guy Dumais <[email protected]> Co-authored-by: koryf <[email protected]>
privateai · Feb 8, 2023 · 8bf1476 · 8bf1476
1 parent f2b7fa6
commit 8bf1476
Show file tree

Hide file tree

Showing 8 changed files with 36 additions and 6 deletions.
diff --git a/pii_check/pii_check_hook.py b/pii_check/pii_check_hook.py
@@ -37,7 +37,7 @@ def get_payload(content, enabled_entity_list, blocked_list):
 def get_flagged_lines(files):
     flagged = []
     for file in files:
-        if os.path.exists(file):
+        if os.path.exists(file) and not os.path.isdir(file):
             with open(file, "r") as fp:
                 lines = fp.readlines()
                 start_flag = False
@@ -77,8 +77,8 @@ def locate_pii_in_files(content, files, checked, pii_dict):
                 for number, line in enumerate(lines, 1):
                     if content in line:
                         if (
-                            pii_dict["stt_idx"],
-                            pii_dict["end_idx"],
+                            pii_dict["location"]["stt_idx"],
+                            pii_dict["location"]["end_idx"],
                             number,
                             file,
                         ) in checked:
@@ -115,16 +115,16 @@ def check_for_pii(url, api_key, enabled_entity_list, blocked_list):
             continue
         for pii_dict in item["entities"]:
             line, file = locate_pii_in_files(content, files, checked, pii_dict)
-            checked.append((pii_dict["stt_idx"], pii_dict["end_idx"], line, file))
+            checked.append((pii_dict["location"]["stt_idx"], pii_dict["location"]["end_idx"], line, file))
             skip = False
             for item in flagged:
                 if line > item[0] and line < item[1] and file == item[2]:
                     skip = True
                     break
             if skip == False:
                 msg.append(
-                    f"PII found - type: {pii_dict['best_label']}, line number: {line}, file: {file}, start index: {pii_dict['stt_idx'] + 1}, end "
-                    f"index: {pii_dict['end_idx'] + 1} "
+                    f"PII found - type: {pii_dict['best_label']}, line number: {line}, file: {file}, start index: {pii_dict['location']['stt_idx'] + 1}, end "
+                    f"index: {pii_dict['location']['end_idx'] + 1} "
                 )
 
     if not msg:

diff --git a/requirements-test.txt b/requirements-test.txt
@@ -0,0 +1,4 @@
+pytest==7.2.1
+pytest-check==2.1.2
+python-dotenv==0.19.0
+requests==2.28.1
diff --git a/tests/__init__.py b/tests/__init__.py
diff --git a/tests/test_data/dir_with_files/file_with_pii.txt b/tests/test_data/dir_with_files/file_with_pii.txt
@@ -0,0 +1 @@
+Credit card number: 1234 5678 9101 1123
diff --git a/tests/test_data/dir_with_files/file_with_pii_flag b/tests/test_data/dir_with_files/file_with_pii_flag
@@ -0,0 +1,10 @@
+PII_CHECK:OFF
+Some content in between the flags. Ideally this content won't be checked for PII.
+Below is a dummy PII to check this
+Credit card number: 1234 5678 9101 1123
+CVV: 123
+PII_CHECK:ON
+
+Some content where the check will be performed.
+Credit card number: 1234 5678 9101 1123
+CVV: 123
diff --git a/tests/test_data/dir_with_files/file_without_pii.txt b/tests/test_data/dir_with_files/file_without_pii.txt
@@ -0,0 +1 @@
+Here's some content.
diff --git a/tests/test_data/symlink_of_dir_with_files b/tests/test_data/symlink_of_dir_with_files
@@ -0,0 +1 @@
+./dir_with_files
diff --git a/tests/test_get_flagged_lines.py b/tests/test_get_flagged_lines.py
@@ -0,0 +1,13 @@
+import pytest_check as check
+from pii_check.pii_check_hook import get_flagged_lines
+
+
+def test_get_flagged_lines():
+    files = [
+        "tests/test_data/dir_with_files/file_with_pii.txt", "tests/test_data/dir_with_files/file_without_pii.txt",
+        "tests/test_data/dir_with_files/file_with_pii_flag_on", "tests/test_data/dir_with_files/file_with_pii_flag_off",
+        "tests/test_data/dir_with_files/file_with_pii_flag", "tests/test_data/symlink_of_dir_with_files"
+    ]
+    res = get_flagged_lines(files)
+    check.equal(res, [(1, 6, 'tests/test_data/dir_with_files/file_with_pii_flag')])
+