fix: Add support for cx_Oracle's DB_TYPE_LONG_RAW (#1095)

* Add DB_TYPE_LONG_RAW to type mapping * fix: Patch SQL Alchemy to support Oracle LONG RAW * tests: Exclude col_long_raw from tests --------- Co-authored-by: nj1973 <[email protected]>
GoogleCloudPlatform · Feb 20, 2024 · 90547ef · 90547ef
1 parent d463038
commit 90547ef
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 10 deletions.
diff --git a/tests/resources/oracle_test_tables.sql b/tests/resources/oracle_test_tables.sql
@@ -100,6 +100,7 @@ CREATE TABLE pso_data_validator.dvt_ora2pg_types
 ,   col_tstz        TIMESTAMP(6) WITH TIME ZONE
 --,   col_tsltz       TIMESTAMP(6) WITH LOCAL TIME ZONE
 ,   col_raw         RAW(16)
+,   col_long_raw    LONG RAW
 ,   col_blob        BLOB
 ,   col_clob        CLOB
 ,   col_nclob       NCLOB
@@ -116,7 +117,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
 ,DATE'1970-01-01',TIMESTAMP'1970-01-01 00:00:01.123456'
 ,to_timestamp_tz('1970-01-01 00:00:01.123456 00:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')
 --,to_timestamp_tz('1970-01-01 00:00:01.123456 00:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')
-,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT'),'DVT A','DVT A'
+,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT')
+,UTL_RAW.CAST_TO_RAW('DVT'),'DVT A','DVT A'
 );
 INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
 (2,2222,123456789,123456789012345678,1234567890123456789012345
@@ -127,7 +129,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
 ,DATE'1970-01-02',TIMESTAMP'1970-01-02 00:00:01.123456'
 ,to_timestamp_tz('1970-01-02 00:00:02.123456 -02:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')
 --,to_timestamp_tz('1970-01-02 00:00:02.123456 -02:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')
-,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT DVT'),'DVT B','DVT B'
+,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT DVT')
+,UTL_RAW.CAST_TO_RAW('DVT DVT'),'DVT B','DVT B'
 );
 INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
 (3,3333,123456789,123456789012345678,1234567890123456789012345
@@ -138,7 +141,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
 ,DATE'1970-01-03',TIMESTAMP'1970-01-03 00:00:01.123456'
 ,to_timestamp_tz('1970-01-03 00:00:03.123456 -03:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')
 --,to_timestamp_tz('1970-01-03 00:00:03.123456 -03:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')
-,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT DVT DVT'),'DVT C','DVT C'
+,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT DVT DVT')
+,UTL_RAW.CAST_TO_RAW('DVT DVT DVT'),'DVT C','DVT C'
 );
 COMMIT;
 

diff --git a/tests/resources/postgresql_test_tables.sql b/tests/resources/postgresql_test_tables.sql
@@ -74,6 +74,7 @@ CREATE TABLE pso_data_validator.dvt_ora2pg_types
 ,   col_tstz        timestamp(6) with time zone
 --,   col_tsltz       timestamp(6) with time zone
 ,   col_raw         bytea
+,   col_long_raw    bytea
 ,   col_blob        bytea
 ,   col_clob        text
 ,   col_nclob       text
@@ -90,7 +91,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
 ,DATE'1970-01-01',TIMESTAMP'1970-01-01 00:00:01.123456'
 ,TIMESTAMP WITH TIME ZONE'1970-01-01 00:00:01.123456 +00:00'
 --,TIMESTAMP WITH TIME ZONE'1970-01-01 00:00:01.123456 +00:00'
-,CAST('DVT' AS BYTEA),CAST('DVT' AS BYTEA),'DVT A','DVT A')
+,CAST('DVT' AS BYTEA),CAST('DVT' AS BYTEA)
+,CAST('DVT' AS BYTEA),'DVT A','DVT A')
 ,(2,2222,123456789,123456789012345678,1234567890123456789012345
 ,123.12,123.11
 --,123400,0.002
@@ -99,7 +101,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
 ,DATE'1970-01-02',TIMESTAMP'1970-01-02 00:00:01.123456'
 ,TIMESTAMP WITH TIME ZONE'1970-01-02 00:00:02.123456 -02:00'
 --,TIMESTAMP WITH TIME ZONE'1970-01-02 00:00:02.123456 -02:00'
-,CAST('DVT' AS BYTEA),CAST('DVT DVT' AS BYTEA),'DVT B','DVT B')
+,CAST('DVT' AS BYTEA),CAST('DVT DVT' AS BYTEA)
+,CAST('DVT DVT' AS BYTEA),'DVT B','DVT B')
 ,(3,3333,123456789,123456789012345678,1234567890123456789012345
 ,123.123,123.11
 --,123400,0.003
@@ -108,7 +111,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
 ,DATE'1970-01-03',TIMESTAMP'1970-01-03 00:00:01.123456'
 ,TIMESTAMP WITH TIME ZONE'1970-01-03 00:00:03.123456 -03:00'
 --,TIMESTAMP WITH TIME ZONE'1970-01-03 00:00:03.123456 -03:00'
-,CAST('DVT' AS BYTEA),CAST('DVT DVT DVT' AS BYTEA),'DVT C','DVT C'
+,CAST('DVT' AS BYTEA),CAST('DVT DVT DVT' AS BYTEA)
+,CAST('DVT DVT DVT' AS BYTEA),'DVT C','DVT C'
 );
 
  /* Following table used for validating generating table partitions */

diff --git a/tests/system/data_sources/test_oracle.py b/tests/system/data_sources/test_oracle.py
@@ -77,6 +77,7 @@
     "col_ts",
     "col_tstz",
     "col_raw",
+    "col_long_raw",
     "col_blob",
     "col_clob",
     "col_nclob",
@@ -324,18 +325,22 @@ def test_column_validation_core_types_to_bigquery():
 )
 def test_column_validation_oracle_to_postgres():
     parser = cli_tools.configure_arg_parser()
-    count_cols = ",".join(ORA2PG_COLUMNS)
+    count_cols = ",".join([_ for _ in ORA2PG_COLUMNS if _ not in ("col_long_raw")])
     # TODO Change sum_cols and min_cols to include col_char_2,col_nchar_2 when issue-842 is complete.
     # TODO Change sum_cols to include col_num_18 when issue-1007 is complete.
     sum_cols = ",".join(
         [
             _
             for _ in ORA2PG_COLUMNS
-            if _ not in ("col_char_2", "col_nchar_2", "col_num_18")
+            if _ not in ("col_char_2", "col_nchar_2", "col_num_18", "col_long_raw")
         ]
     )
     min_cols = ",".join(
-        [_ for _ in ORA2PG_COLUMNS if _ not in ("col_char_2", "col_nchar_2")]
+        [
+            _
+            for _ in ORA2PG_COLUMNS
+            if _ not in ("col_char_2", "col_nchar_2", "col_long_raw")
+        ]
     )
     args = parser.parse_args(
         [
@@ -423,7 +428,7 @@ def test_row_validation_core_types_to_bigquery():
 )
 def test_row_validation_oracle_to_postgres():
     # TODO Change hash_cols below to include col_tstz when issue-706 is complete.
-    # TODO col_raw is blocked by issue-773 (is it even reasonable to expect binary columns to work here?)
+    # TODO col_raw/col_long_raw are blocked by issue-773 (is it even reasonable to expect binary columns to work here?)
     # TODO Change hash_cols below to include col_nvarchar_30,col_nchar_2 when issue-772 is complete.
     # Excluded col_float32,col_float64 due to the lossy nature of BINARY_FLOAT/DOUBLE.
     # Excluded CLOB/NCLOB/BLOB columns because lob values cannot be concatenated
@@ -437,6 +442,7 @@ def test_row_validation_oracle_to_postgres():
                 "col_clob",
                 "col_nclob",
                 "col_raw",
+                "col_long_raw",
                 "col_float32",
                 "col_float64",
                 "col_tstz",

diff --git a/third_party/ibis/ibis_oracle/datatypes.py b/third_party/ibis/ibis_oracle/datatypes.py
@@ -68,6 +68,7 @@ def _get_type(col: _FieldDescription) -> dt.DataType:
     cx_Oracle.DB_TYPE_TIMESTAMP: dt.Timestamp,
     cx_Oracle.DB_TYPE_TIMESTAMP_TZ: dt.Timestamp(timezone="UTC"),
     cx_Oracle.DB_TYPE_RAW: dt.Binary,
+    cx_Oracle.DB_TYPE_LONG_RAW: dt.Binary,
     cx_Oracle.DB_TYPE_BFILE: dt.Binary,
     cx_Oracle.DB_TYPE_NUMBER: dt.Decimal,
     cx_Oracle.DB_TYPE_LONG: dt.String,
@@ -78,6 +79,15 @@ def _get_type(col: _FieldDescription) -> dt.DataType:
     cx_Oracle.DB_TYPE_BINARY_DOUBLE: dt.Float64,
 }
 
+# SQL Alchemy doesn't support LONG RAW which drops us into Ibis 5.1.0 method:
+# _handle_failed_column_type_inference().
+# This method has a flaw that is doesn't prefix the table name with the schema
+# which causes a table not found exception.
+# I (nj@2024-02-15) felt the simplest change was to add LONG RAW to SQL Alchemy
+# as a RAW variant, as below.
+if "LONG RAW" not in OracleDialect_cx_oracle.ischema_names:
+    OracleDialect_cx_oracle.ischema_names["LONG RAW"] = oracle.RAW
+
 
 @dt.dtype.register(OracleDialect_cx_oracle, sa.dialects.oracle.CLOB)
 def sa_oracle_CLOB(_, satype, nullable=True):