From 7dca5bd9ca3d2701ef33e5b1a7925adc36bfd6d2 Mon Sep 17 00:00:00 2001 From: sramsrinivasan <107146913+sramsrinivasan@users.noreply.github.com> Date: Thu, 14 Jul 2022 15:19:57 -0500 Subject: [PATCH] feat: Remove dependency on tables list for custom query (#541) --- data_validation/__main__.py | 2 +- data_validation/config_manager.py | 10 ++++++---- docs/examples.md | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/data_validation/__main__.py b/data_validation/__main__.py index 2555638ca..0c781247f 100644 --- a/data_validation/__main__.py +++ b/data_validation/__main__.py @@ -246,7 +246,7 @@ def build_config_managers_from_args(args): is_filesystem = source_client._source_type == "FileSystem" tables_list = cli_tools.get_tables_list( - args.tables_list, default_value=[], is_filesystem=is_filesystem + args.tables_list, default_value=[{}], is_filesystem=is_filesystem ) for table_obj in tables_list: diff --git a/data_validation/config_manager.py b/data_validation/config_manager.py index c7dd0a1f3..6364e9ee8 100644 --- a/data_validation/config_manager.py +++ b/data_validation/config_manager.py @@ -401,13 +401,15 @@ def build_config_manager( consts.CONFIG_TYPE: config_type, consts.CONFIG_SOURCE_CONN_NAME: source_conn_name, consts.CONFIG_TARGET_CONN_NAME: target_conn_name, - consts.CONFIG_TABLE_NAME: table_obj[consts.CONFIG_TABLE_NAME], - consts.CONFIG_SCHEMA_NAME: table_obj[consts.CONFIG_SCHEMA_NAME], + consts.CONFIG_TABLE_NAME: table_obj.get(consts.CONFIG_TABLE_NAME, None), + consts.CONFIG_SCHEMA_NAME: table_obj.get(consts.CONFIG_SCHEMA_NAME, None), consts.CONFIG_TARGET_SCHEMA_NAME: table_obj.get( - consts.CONFIG_TARGET_SCHEMA_NAME, table_obj[consts.CONFIG_SCHEMA_NAME] + consts.CONFIG_TARGET_SCHEMA_NAME, + table_obj.get(consts.CONFIG_SCHEMA_NAME, None), ), consts.CONFIG_TARGET_TABLE_NAME: table_obj.get( - consts.CONFIG_TARGET_TABLE_NAME, table_obj[consts.CONFIG_TABLE_NAME] + consts.CONFIG_TARGET_TABLE_NAME, + table_obj.get(consts.CONFIG_TABLE_NAME, None), ), consts.CONFIG_LABELS: labels, consts.CONFIG_THRESHOLD: threshold, diff --git a/docs/examples.md b/docs/examples.md index c103c6244..e954b15b4 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -228,7 +228,7 @@ validations: #### Run a custom query column validation ````shell script -data-validation validate --custom-query-type column custom-query --source-query-file source_query.sql --target-query-file target_query.sql -sc my_bq_conn -tc my_bq_conn -tbls bigquery-public-data.new_york_citibike.citibike_stations +data-validation validate custom-query --custom-query-type column --source-query-file source_query.sql --target-query-file target_query.sql -sc my_bq_conn -tc my_bq_conn ```` #### Run a custom query validation with sum aggregation