Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: handle numeric datatype mapping in teradata schema and fix int mapping as per teradata doc #874

Merged
merged 3 commits into from
May 31, 2023
Merged

fix: handle numeric datatype mapping in teradata schema and fix int mapping as per teradata doc #874

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8327a56
fix: handle numeric datatype mapping in teradata schema and fix int m…
sharangagarwal May 30, 2023
be33def
fix: Change cast from float64 to float as float64 doesnot exist in te…
sharangagarwal May 30, 2023
7cc8568
chore: Change allowlist to incorporate fix
sharangagarwal May 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions tests/system/data_sources/test_teradata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,11 +241,11 @@ def test_schema_validation_core_types_to_bigquery():
"--filter-status=fail",
(
# Teradata integrals go to BigQuery INT64.
"--allow-list=int8:int64,int16:int64,"
"--allow-list=int8:int64,int16:int64,int32:int64,"
# Teradata NUMBERS that map to BigQuery NUMERIC.
# "decimal(20,0):decimal(38,9),decimal(10,2):decimal(38,9),"
"decimal(20,0):decimal(38,9),decimal(10,2):decimal(38,9),"
# When fix issue 838 then uncomment line above and remove line below.
"float64:decimal(38,9),"
# "float64:decimal(38,9),"
# Teradata NUMBERS that map to BigQuery BIGNUMERIC.
# When issue-839 is resolved we need to edit the line below as appropriate.
"decimal(38,0):decimal(38,9)"
Expand Down Expand Up @@ -297,7 +297,6 @@ def test_column_validation_core_types():
def test_column_validation_core_types_to_bigquery():
parser = cli_tools.configure_arg_parser()
# TODO Add col_datetime,col_tstz to --sum string below when issue-762 is complete.
# TODO Add col_dec_20,col_dec_38,col_dec_10_2 to --sum/min/max string below when issue-838 is complete.
args = parser.parse_args(
[
"validate",
Expand All @@ -306,9 +305,9 @@ def test_column_validation_core_types_to_bigquery():
"-tc=bq-conn",
"-tbls=udf.dvt_core_types=pso_data_validator.dvt_core_types",
"--filter-status=fail",
"--sum=col_int8,col_int16,col_int32,col_int64,col_float32,col_float64,col_varchar_30,col_char_2,col_string,col_date",
"--min=col_int8,col_int16,col_int32,col_int64,col_float32,col_float64,col_varchar_30,col_char_2,col_string,col_date",
"--max=col_int8,col_int16,col_int32,col_int64,col_float32,col_float64,col_varchar_30,col_char_2,col_string,col_date",
"--sum=col_int8,col_int16,col_int32,col_int64,col_float32,col_float64,col_varchar_30,col_char_2,col_string,col_date,col_dec_20,col_dec_38,col_dec_10_2",
"--min=col_int8,col_int16,col_int32,col_int64,col_float32,col_float64,col_varchar_30,col_char_2,col_string,col_date,col_dec_20,col_dec_38,col_dec_10_2",
"--max=col_int8,col_int16,col_int32,col_int64,col_float32,col_float64,col_varchar_30,col_char_2,col_string,col_date,col_dec_20,col_dec_38,col_dec_10_2",
]
)
config_managers = main.build_config_managers_from_args(args)
Expand Down
21 changes: 14 additions & 7 deletions third_party/ibis/ibis_teradata/datatypes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,16 @@ def to_ibis_from_N(cls, col_data, return_ibis_type=True):

@classmethod
def to_ibis_from_D(cls, col_data, return_ibis_type=True):
precision = int(col_data.get("DecimalTotalDigits", col_data.get("Decimal Total Digits", 20)))
scale = int(col_data.get("DecimalFractionalDigits", col_data.get("Decimal Fractional Digits", 4)))
precision = int(
col_data.get("DecimalTotalDigits", col_data.get("Decimal Total Digits", 20))
)
scale = int(
col_data.get(
"DecimalFractionalDigits", col_data.get("Decimal Fractional Digits", 4)
)
)
if return_ibis_type:
return dt.float64
return dt.Decimal(precision, scale)
value_type = "DECIMAL(%d, %d)" % (precision, scale)
return value_type

Expand All @@ -71,7 +77,7 @@ def to_ibis_from_F(cls, col_data, return_ibis_type=True):
@classmethod
def to_ibis_from_I(cls, col_data, return_ibis_type=True):
if return_ibis_type:
return dt.int64
return dt.int32
return "INT"

@classmethod
Expand All @@ -83,13 +89,13 @@ def to_ibis_from_I1(cls, col_data, return_ibis_type=True):
@classmethod
def to_ibis_from_I2(cls, col_data, return_ibis_type=True):
if return_ibis_type:
return dt.int8
return dt.int16
return "INT"

@classmethod
def to_ibis_from_I8(cls, col_data, return_ibis_type=True):
if return_ibis_type:
return dt.int16
return dt.int64
return "INT"

@classmethod
Expand All @@ -116,6 +122,7 @@ def to_ibis_from_SZ(cls, col_data, return_ibis_type=True):

ibis_type_to_teradata_type = Dispatcher("ibis_type_to_teradata_type")


@ibis_type_to_teradata_type.register(dt.DataType)
def trans_default(t):
return ibis_type_to_teradata_type(t, TypeTranslationContext())
Expand All @@ -133,7 +140,7 @@ def trans_string(t, context):

@ibis_type_to_teradata_type.register(dt.Floating, TypeTranslationContext)
def trans_float64(t, context):
return "FLOAT64"
return "FLOAT"


@ibis_type_to_teradata_type.register(dt.Integer, TypeTranslationContext)
Expand Down