From 333eadbf9136a78e51822fcae5cd22f4490bdd6e Mon Sep 17 00:00:00 2001 From: Sharang Agarwal <117446992+sharangagarwal@users.noreply.github.com> Date: Wed, 31 May 2023 22:21:19 +0530 Subject: [PATCH] fix: handle numeric datatype mapping in teradata schema and fix int mapping as per teradata doc (#874) * fix: handle numeric datatype mapping in teradata schema and fix int mapping as per doc * fix: Change cast from float64 to float as float64 doesnot exist in teradata --- tests/system/data_sources/test_teradata.py | 13 ++++++------- third_party/ibis/ibis_teradata/datatypes.py | 21 ++++++++++++++------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/system/data_sources/test_teradata.py b/tests/system/data_sources/test_teradata.py index 59e3a4095..fb5657616 100644 --- a/tests/system/data_sources/test_teradata.py +++ b/tests/system/data_sources/test_teradata.py @@ -241,11 +241,11 @@ def test_schema_validation_core_types_to_bigquery(): "--filter-status=fail", ( # Teradata integrals go to BigQuery INT64. - "--allow-list=int8:int64,int16:int64," + "--allow-list=int8:int64,int16:int64,int32:int64," # Teradata NUMBERS that map to BigQuery NUMERIC. - # "decimal(20,0):decimal(38,9),decimal(10,2):decimal(38,9)," + "decimal(20,0):decimal(38,9),decimal(10,2):decimal(38,9)," # When fix issue 838 then uncomment line above and remove line below. - "float64:decimal(38,9)," + # "float64:decimal(38,9)," # Teradata NUMBERS that map to BigQuery BIGNUMERIC. # When issue-839 is resolved we need to edit the line below as appropriate. "decimal(38,0):decimal(38,9)" @@ -297,7 +297,6 @@ def test_column_validation_core_types(): def test_column_validation_core_types_to_bigquery(): parser = cli_tools.configure_arg_parser() # TODO Add col_datetime,col_tstz to --sum string below when issue-762 is complete. - # TODO Add col_dec_20,col_dec_38,col_dec_10_2 to --sum/min/max string below when issue-838 is complete. args = parser.parse_args( [ "validate", @@ -306,9 +305,9 @@ def test_column_validation_core_types_to_bigquery(): "-tc=bq-conn", "-tbls=udf.dvt_core_types=pso_data_validator.dvt_core_types", "--filter-status=fail", - "--sum=col_int8,col_int16,col_int32,col_int64,col_float32,col_float64,col_varchar_30,col_char_2,col_string,col_date", - "--min=col_int8,col_int16,col_int32,col_int64,col_float32,col_float64,col_varchar_30,col_char_2,col_string,col_date", - "--max=col_int8,col_int16,col_int32,col_int64,col_float32,col_float64,col_varchar_30,col_char_2,col_string,col_date", + "--sum=col_int8,col_int16,col_int32,col_int64,col_float32,col_float64,col_varchar_30,col_char_2,col_string,col_date,col_dec_20,col_dec_38,col_dec_10_2", + "--min=col_int8,col_int16,col_int32,col_int64,col_float32,col_float64,col_varchar_30,col_char_2,col_string,col_date,col_dec_20,col_dec_38,col_dec_10_2", + "--max=col_int8,col_int16,col_int32,col_int64,col_float32,col_float64,col_varchar_30,col_char_2,col_string,col_date,col_dec_20,col_dec_38,col_dec_10_2", ] ) config_managers = main.build_config_managers_from_args(args) diff --git a/third_party/ibis/ibis_teradata/datatypes.py b/third_party/ibis/ibis_teradata/datatypes.py index 5a73cdba6..f0c812225 100644 --- a/third_party/ibis/ibis_teradata/datatypes.py +++ b/third_party/ibis/ibis_teradata/datatypes.py @@ -55,10 +55,16 @@ def to_ibis_from_N(cls, col_data, return_ibis_type=True): @classmethod def to_ibis_from_D(cls, col_data, return_ibis_type=True): - precision = int(col_data.get("DecimalTotalDigits", col_data.get("Decimal Total Digits", 20))) - scale = int(col_data.get("DecimalFractionalDigits", col_data.get("Decimal Fractional Digits", 4))) + precision = int( + col_data.get("DecimalTotalDigits", col_data.get("Decimal Total Digits", 20)) + ) + scale = int( + col_data.get( + "DecimalFractionalDigits", col_data.get("Decimal Fractional Digits", 4) + ) + ) if return_ibis_type: - return dt.float64 + return dt.Decimal(precision, scale) value_type = "DECIMAL(%d, %d)" % (precision, scale) return value_type @@ -71,7 +77,7 @@ def to_ibis_from_F(cls, col_data, return_ibis_type=True): @classmethod def to_ibis_from_I(cls, col_data, return_ibis_type=True): if return_ibis_type: - return dt.int64 + return dt.int32 return "INT" @classmethod @@ -83,13 +89,13 @@ def to_ibis_from_I1(cls, col_data, return_ibis_type=True): @classmethod def to_ibis_from_I2(cls, col_data, return_ibis_type=True): if return_ibis_type: - return dt.int8 + return dt.int16 return "INT" @classmethod def to_ibis_from_I8(cls, col_data, return_ibis_type=True): if return_ibis_type: - return dt.int16 + return dt.int64 return "INT" @classmethod @@ -116,6 +122,7 @@ def to_ibis_from_SZ(cls, col_data, return_ibis_type=True): ibis_type_to_teradata_type = Dispatcher("ibis_type_to_teradata_type") + @ibis_type_to_teradata_type.register(dt.DataType) def trans_default(t): return ibis_type_to_teradata_type(t, TypeTranslationContext()) @@ -133,7 +140,7 @@ def trans_string(t, context): @ibis_type_to_teradata_type.register(dt.Floating, TypeTranslationContext) def trans_float64(t, context): - return "FLOAT64" + return "FLOAT" @ibis_type_to_teradata_type.register(dt.Integer, TypeTranslationContext)