feat: Remove dependency on tables list for custom query

GoogleCloudPlatform · Jul 13, 2022 · 11df129 · 11df129
1 parent f7ed739
commit 11df129
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -236,9 +236,9 @@ data-validation (--verbose or -v) validate custom-query
                         Target connection details
                         See: *Connections* section for each data source
   --tables-list or -tbls SOURCE_SCHEMA.SOURCE_TABLE=TARGET_SCHEMA.TARGET_TABLE
-                        Comma separated list of tables in the form schema.table=target_schema.target_table
-                        Target schema name and table name are optional.
-                        i.e 'bigquery-public-data.new_york_citibike.citibike_trips'
+                      Comma separated list of tables in the form schema.table=target_schema.target_table
+                      Target schema name and table name are optional.
+                      i.e 'bigquery-public-data.new_york_citibike.citibike_trips'
   --custom-query-type CUSTOM_QUERY_TYPE, -cqt CUSTOM_QUERY_TYPE
                         Type of custom query validation: ('row'|'column')
                         Enter 'column' for custom query column validation

diff --git a/data_validation/__main__.py b/data_validation/__main__.py
@@ -246,7 +246,7 @@ def build_config_managers_from_args(args):
 
     is_filesystem = source_client._source_type == "FileSystem"
     tables_list = cli_tools.get_tables_list(
-        args.tables_list, default_value=[], is_filesystem=is_filesystem
+        args.tables_list, default_value=[{}], is_filesystem=is_filesystem
     )
 
     for table_obj in tables_list:

diff --git a/data_validation/config_manager.py b/data_validation/config_manager.py
@@ -401,13 +401,15 @@ def build_config_manager(
             consts.CONFIG_TYPE: config_type,
             consts.CONFIG_SOURCE_CONN_NAME: source_conn_name,
             consts.CONFIG_TARGET_CONN_NAME: target_conn_name,
-            consts.CONFIG_TABLE_NAME: table_obj[consts.CONFIG_TABLE_NAME],
-            consts.CONFIG_SCHEMA_NAME: table_obj[consts.CONFIG_SCHEMA_NAME],
+            consts.CONFIG_TABLE_NAME: table_obj.get(consts.CONFIG_TABLE_NAME, None),
+            consts.CONFIG_SCHEMA_NAME: table_obj.get(consts.CONFIG_SCHEMA_NAME, None),
             consts.CONFIG_TARGET_SCHEMA_NAME: table_obj.get(
-                consts.CONFIG_TARGET_SCHEMA_NAME, table_obj[consts.CONFIG_SCHEMA_NAME]
+                consts.CONFIG_TARGET_SCHEMA_NAME,
+                table_obj.get(consts.CONFIG_SCHEMA_NAME, None),
             ),
             consts.CONFIG_TARGET_TABLE_NAME: table_obj.get(
-                consts.CONFIG_TARGET_TABLE_NAME, table_obj[consts.CONFIG_TABLE_NAME]
+                consts.CONFIG_TARGET_TABLE_NAME,
+                table_obj.get(consts.CONFIG_TABLE_NAME, None),
             ),
             consts.CONFIG_LABELS: labels,
             consts.CONFIG_THRESHOLD: threshold,

diff --git a/docs/examples.md b/docs/examples.md
@@ -228,7 +228,7 @@ validations:
 
 #### Run a custom query column validation
 ````shell script
-data-validation validate --custom-query-type column custom-query --source-query-file source_query.sql --target-query-file target_query.sql -sc my_bq_conn -tc my_bq_conn -tbls bigquery-public-data.new_york_citibike.citibike_stations
+data-validation validate custom-query --custom-query-type column --source-query-file source_query.sql --target-query-file target_query.sql -sc my_bq_conn -tc my_bq_conn
 ````
 
 #### Run a custom query validation with sum aggregation