diff --git a/tests/resources/oracle_test_tables.sql b/tests/resources/oracle_test_tables.sql index d72be23d..32b62dbe 100644 --- a/tests/resources/oracle_test_tables.sql +++ b/tests/resources/oracle_test_tables.sql @@ -100,6 +100,7 @@ CREATE TABLE pso_data_validator.dvt_ora2pg_types , col_tstz TIMESTAMP(6) WITH TIME ZONE --, col_tsltz TIMESTAMP(6) WITH LOCAL TIME ZONE , col_raw RAW(16) +, col_long_raw LONG RAW , col_blob BLOB , col_clob CLOB , col_nclob NCLOB @@ -116,7 +117,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES ,DATE'1970-01-01',TIMESTAMP'1970-01-01 00:00:01.123456' ,to_timestamp_tz('1970-01-01 00:00:01.123456 00:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM') --,to_timestamp_tz('1970-01-01 00:00:01.123456 00:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM') -,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT'),'DVT A','DVT A' +,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT') +,UTL_RAW.CAST_TO_RAW('DVT'),'DVT A','DVT A' ); INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES (2,2222,123456789,123456789012345678,1234567890123456789012345 @@ -127,7 +129,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES ,DATE'1970-01-02',TIMESTAMP'1970-01-02 00:00:01.123456' ,to_timestamp_tz('1970-01-02 00:00:02.123456 -02:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM') --,to_timestamp_tz('1970-01-02 00:00:02.123456 -02:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM') -,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT DVT'),'DVT B','DVT B' +,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT DVT') +,UTL_RAW.CAST_TO_RAW('DVT DVT'),'DVT B','DVT B' ); INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES (3,3333,123456789,123456789012345678,1234567890123456789012345 @@ -138,7 +141,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES ,DATE'1970-01-03',TIMESTAMP'1970-01-03 00:00:01.123456' ,to_timestamp_tz('1970-01-03 00:00:03.123456 -03:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM') --,to_timestamp_tz('1970-01-03 00:00:03.123456 -03:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM') -,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT DVT DVT'),'DVT C','DVT C' +,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT DVT DVT') +,UTL_RAW.CAST_TO_RAW('DVT DVT DVT'),'DVT C','DVT C' ); COMMIT; diff --git a/tests/resources/postgresql_test_tables.sql b/tests/resources/postgresql_test_tables.sql index c8104e5e..eb608204 100644 --- a/tests/resources/postgresql_test_tables.sql +++ b/tests/resources/postgresql_test_tables.sql @@ -74,6 +74,7 @@ CREATE TABLE pso_data_validator.dvt_ora2pg_types , col_tstz timestamp(6) with time zone --, col_tsltz timestamp(6) with time zone , col_raw bytea +, col_long_raw bytea , col_blob bytea , col_clob text , col_nclob text @@ -90,7 +91,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES ,DATE'1970-01-01',TIMESTAMP'1970-01-01 00:00:01.123456' ,TIMESTAMP WITH TIME ZONE'1970-01-01 00:00:01.123456 +00:00' --,TIMESTAMP WITH TIME ZONE'1970-01-01 00:00:01.123456 +00:00' -,CAST('DVT' AS BYTEA),CAST('DVT' AS BYTEA),'DVT A','DVT A') +,CAST('DVT' AS BYTEA),CAST('DVT' AS BYTEA) +,CAST('DVT' AS BYTEA),'DVT A','DVT A') ,(2,2222,123456789,123456789012345678,1234567890123456789012345 ,123.12,123.11 --,123400,0.002 @@ -99,7 +101,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES ,DATE'1970-01-02',TIMESTAMP'1970-01-02 00:00:01.123456' ,TIMESTAMP WITH TIME ZONE'1970-01-02 00:00:02.123456 -02:00' --,TIMESTAMP WITH TIME ZONE'1970-01-02 00:00:02.123456 -02:00' -,CAST('DVT' AS BYTEA),CAST('DVT DVT' AS BYTEA),'DVT B','DVT B') +,CAST('DVT' AS BYTEA),CAST('DVT DVT' AS BYTEA) +,CAST('DVT DVT' AS BYTEA),'DVT B','DVT B') ,(3,3333,123456789,123456789012345678,1234567890123456789012345 ,123.123,123.11 --,123400,0.003 @@ -108,7 +111,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES ,DATE'1970-01-03',TIMESTAMP'1970-01-03 00:00:01.123456' ,TIMESTAMP WITH TIME ZONE'1970-01-03 00:00:03.123456 -03:00' --,TIMESTAMP WITH TIME ZONE'1970-01-03 00:00:03.123456 -03:00' -,CAST('DVT' AS BYTEA),CAST('DVT DVT DVT' AS BYTEA),'DVT C','DVT C' +,CAST('DVT' AS BYTEA),CAST('DVT DVT DVT' AS BYTEA) +,CAST('DVT DVT DVT' AS BYTEA),'DVT C','DVT C' ); /* Following table used for validating generating table partitions */ diff --git a/tests/system/data_sources/test_oracle.py b/tests/system/data_sources/test_oracle.py index a9ee24e5..292f7206 100644 --- a/tests/system/data_sources/test_oracle.py +++ b/tests/system/data_sources/test_oracle.py @@ -77,6 +77,7 @@ "col_ts", "col_tstz", "col_raw", + "col_long_raw", "col_blob", "col_clob", "col_nclob", @@ -324,18 +325,22 @@ def test_column_validation_core_types_to_bigquery(): ) def test_column_validation_oracle_to_postgres(): parser = cli_tools.configure_arg_parser() - count_cols = ",".join(ORA2PG_COLUMNS) + count_cols = ",".join([_ for _ in ORA2PG_COLUMNS if _ not in ("col_long_raw")]) # TODO Change sum_cols and min_cols to include col_char_2,col_nchar_2 when issue-842 is complete. # TODO Change sum_cols to include col_num_18 when issue-1007 is complete. sum_cols = ",".join( [ _ for _ in ORA2PG_COLUMNS - if _ not in ("col_char_2", "col_nchar_2", "col_num_18") + if _ not in ("col_char_2", "col_nchar_2", "col_num_18", "col_long_raw") ] ) min_cols = ",".join( - [_ for _ in ORA2PG_COLUMNS if _ not in ("col_char_2", "col_nchar_2")] + [ + _ + for _ in ORA2PG_COLUMNS + if _ not in ("col_char_2", "col_nchar_2", "col_long_raw") + ] ) args = parser.parse_args( [ @@ -423,7 +428,7 @@ def test_row_validation_core_types_to_bigquery(): ) def test_row_validation_oracle_to_postgres(): # TODO Change hash_cols below to include col_tstz when issue-706 is complete. - # TODO col_raw is blocked by issue-773 (is it even reasonable to expect binary columns to work here?) + # TODO col_raw/col_long_raw are blocked by issue-773 (is it even reasonable to expect binary columns to work here?) # TODO Change hash_cols below to include col_nvarchar_30,col_nchar_2 when issue-772 is complete. # Excluded col_float32,col_float64 due to the lossy nature of BINARY_FLOAT/DOUBLE. # Excluded CLOB/NCLOB/BLOB columns because lob values cannot be concatenated @@ -437,6 +442,7 @@ def test_row_validation_oracle_to_postgres(): "col_clob", "col_nclob", "col_raw", + "col_long_raw", "col_float32", "col_float64", "col_tstz", diff --git a/third_party/ibis/ibis_oracle/datatypes.py b/third_party/ibis/ibis_oracle/datatypes.py index aafdda6f..b6619bb5 100644 --- a/third_party/ibis/ibis_oracle/datatypes.py +++ b/third_party/ibis/ibis_oracle/datatypes.py @@ -68,6 +68,7 @@ def _get_type(col: _FieldDescription) -> dt.DataType: cx_Oracle.DB_TYPE_TIMESTAMP: dt.Timestamp, cx_Oracle.DB_TYPE_TIMESTAMP_TZ: dt.Timestamp(timezone="UTC"), cx_Oracle.DB_TYPE_RAW: dt.Binary, + cx_Oracle.DB_TYPE_LONG_RAW: dt.Binary, cx_Oracle.DB_TYPE_BFILE: dt.Binary, cx_Oracle.DB_TYPE_NUMBER: dt.Decimal, cx_Oracle.DB_TYPE_LONG: dt.String, @@ -78,6 +79,15 @@ def _get_type(col: _FieldDescription) -> dt.DataType: cx_Oracle.DB_TYPE_BINARY_DOUBLE: dt.Float64, } +# SQL Alchemy doesn't support LONG RAW which drops us into Ibis 5.1.0 method: +# _handle_failed_column_type_inference(). +# This method has a flaw that is doesn't prefix the table name with the schema +# which causes a table not found exception. +# I (nj@2024-02-15) felt the simplest change was to add LONG RAW to SQL Alchemy +# as a RAW variant, as below. +if "LONG RAW" not in OracleDialect_cx_oracle.ischema_names: + OracleDialect_cx_oracle.ischema_names["LONG RAW"] = oracle.RAW + @dt.dtype.register(OracleDialect_cx_oracle, sa.dialects.oracle.CLOB) def sa_oracle_CLOB(_, satype, nullable=True):