Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: primary key casting #521

Merged
merged 3 commits into from
Jun 27, 2022
Merged

fix: primary key casting #521

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4a44d54
passing cast to comparison field
renzokuken Jun 17, 2022
3e70ed3
primary key casting works
renzokuken Jun 23, 2022
d5b7490
missed file upload
renzokuken Jun 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion data_validation/query_builder/query_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,8 @@ def compile(self, ibis_table):
# Fields are supplied on compile or on build
comparison_field = ibis_table[self.field_name]
alias = self.alias or self.field_name
if self.cast:
comparison_field = comparison_field.cast(self.cast)
comparison_field = comparison_field.name(alias)

return comparison_field
Expand Down Expand Up @@ -493,6 +495,8 @@ def compile(self, data_client, schema_name, table_name):
calc_table = calc_table.mutate(
self.compile_calculated_fields(calc_table, n)
)
if self.comparison_fields:
calc_table = calc_table.mutate(self.compile_comparison_fields(calc_table))
compiled_filters = self.compile_filter_fields(table)
filtered_table = (
calc_table.filter(compiled_filters) if compiled_filters else calc_table
Expand All @@ -508,7 +512,7 @@ def compile(self, data_client, schema_name, table_name):
self.compile_aggregate_fields(filtered_table)
)
else:
query = grouped_table
query = filtered_table
renzokuken marked this conversation as resolved.
Show resolved Hide resolved

if self.limit:
query = query.limit(self.limit)
Expand Down
27 changes: 21 additions & 6 deletions data_validation/validation_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from data_validation.query_builder.query_builder import (
AggregateField,
CalculatedField,
ComparisonField,
FilterField,
GroupedField,
QueryBuilder,
Expand Down Expand Up @@ -227,18 +228,25 @@ def add_query_group(self, grouped_field):
self.group_aliases[alias] = grouped_field

def add_primary_key(self, primary_key):
"""Add ComparionField to Queries
"""Add ComparisonField to Queries

Args:
primary_key (Dict): An object with source, target, and cast info
"""
source_field_name = primary_key[consts.CONFIG_SOURCE_COLUMN]
target_field_name = primary_key[consts.CONFIG_TARGET_COLUMN]
# grab calc field metadata
alias = primary_key[consts.CONFIG_FIELD_ALIAS]
alias = primary_key.get(consts.CONFIG_FIELD_ALIAS)
cast = primary_key.get(consts.CONFIG_CAST)
# check if valid calc field and return correct object
self.source_builder.add_comparison_field(source_field_name)
self.target_builder.add_comparison_field(target_field_name)
source_field = ComparisonField(
field_name=source_field_name, alias=alias, cast=cast
)
target_field = ComparisonField(
field_name=target_field_name, alias=alias, cast=cast
)
self.source_builder.add_comparison_field(source_field)
self.target_builder.add_comparison_field(target_field)
self.primary_keys[alias] = primary_key

def add_filter(self, filter_field):
Expand Down Expand Up @@ -287,9 +295,16 @@ def add_comparison_field(self, comparison_field):
target_field_name = comparison_field[consts.CONFIG_TARGET_COLUMN]
# grab calc field metadata
alias = comparison_field[consts.CONFIG_FIELD_ALIAS]
cast = comparison_field.get(consts.CONFIG_CAST)
source_field = ComparisonField(
field_name=source_field_name, alias=alias, cast=cast
)
target_field = ComparisonField(
field_name=target_field_name, alias=alias, cast=cast
)
# check if valid calc field and return correct object
self.source_builder.add_comparison_field(source_field_name)
self.target_builder.add_comparison_field(target_field_name)
self.source_builder.add_comparison_field(source_field)
self.target_builder.add_comparison_field(target_field)
self._metadata[alias] = metadata.ValidationMetadata(
aggregation_type=None,
validation_type=self.validation_type,
Expand Down