Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: Add support for cx_Oracle's DB_TYPE_LONG_RAW #1095

Merged
merged 3 commits into from
Feb 20, 2024
Merged

fix: Add support for cx_Oracle's DB_TYPE_LONG_RAW #1095

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1aef0b6
Add DB_TYPE_LONG_RAW to type mapping
helensilva14 Feb 14, 2024
f1c70e4
fix: Patch SQL Alchemy to support Oracle LONG RAW
nj1973 Feb 15, 2024
c470fec
tests: Exclude col_long_raw from tests
nj1973 Feb 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions tests/resources/oracle_test_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ CREATE TABLE pso_data_validator.dvt_ora2pg_types
, col_tstz TIMESTAMP(6) WITH TIME ZONE
--, col_tsltz TIMESTAMP(6) WITH LOCAL TIME ZONE
, col_raw RAW(16)
, col_long_raw LONG RAW
, col_blob BLOB
, col_clob CLOB
, col_nclob NCLOB
Expand All @@ -116,7 +117,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
,DATE'1970-01-01',TIMESTAMP'1970-01-01 00:00:01.123456'
,to_timestamp_tz('1970-01-01 00:00:01.123456 00:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')
--,to_timestamp_tz('1970-01-01 00:00:01.123456 00:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')
,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT'),'DVT A','DVT A'
,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT')
,UTL_RAW.CAST_TO_RAW('DVT'),'DVT A','DVT A'
);
INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
(2,2222,123456789,123456789012345678,1234567890123456789012345
Expand All @@ -127,7 +129,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
,DATE'1970-01-02',TIMESTAMP'1970-01-02 00:00:01.123456'
,to_timestamp_tz('1970-01-02 00:00:02.123456 -02:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')
--,to_timestamp_tz('1970-01-02 00:00:02.123456 -02:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')
,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT DVT'),'DVT B','DVT B'
,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT DVT')
,UTL_RAW.CAST_TO_RAW('DVT DVT'),'DVT B','DVT B'
);
INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
(3,3333,123456789,123456789012345678,1234567890123456789012345
Expand All @@ -138,7 +141,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
,DATE'1970-01-03',TIMESTAMP'1970-01-03 00:00:01.123456'
,to_timestamp_tz('1970-01-03 00:00:03.123456 -03:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')
--,to_timestamp_tz('1970-01-03 00:00:03.123456 -03:00','YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')
,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT DVT DVT'),'DVT C','DVT C'
,UTL_RAW.CAST_TO_RAW('DVT'),UTL_RAW.CAST_TO_RAW('DVT DVT DVT')
,UTL_RAW.CAST_TO_RAW('DVT DVT DVT'),'DVT C','DVT C'
);
COMMIT;

Expand Down
10 changes: 7 additions & 3 deletions tests/resources/postgresql_test_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ CREATE TABLE pso_data_validator.dvt_ora2pg_types
, col_tstz timestamp(6) with time zone
--, col_tsltz timestamp(6) with time zone
, col_raw bytea
, col_long_raw bytea
, col_blob bytea
, col_clob text
, col_nclob text
Expand All @@ -90,7 +91,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
,DATE'1970-01-01',TIMESTAMP'1970-01-01 00:00:01.123456'
,TIMESTAMP WITH TIME ZONE'1970-01-01 00:00:01.123456 +00:00'
--,TIMESTAMP WITH TIME ZONE'1970-01-01 00:00:01.123456 +00:00'
,CAST('DVT' AS BYTEA),CAST('DVT' AS BYTEA),'DVT A','DVT A')
,CAST('DVT' AS BYTEA),CAST('DVT' AS BYTEA)
,CAST('DVT' AS BYTEA),'DVT A','DVT A')
,(2,2222,123456789,123456789012345678,1234567890123456789012345
,123.12,123.11
--,123400,0.002
Expand All @@ -99,7 +101,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
,DATE'1970-01-02',TIMESTAMP'1970-01-02 00:00:01.123456'
,TIMESTAMP WITH TIME ZONE'1970-01-02 00:00:02.123456 -02:00'
--,TIMESTAMP WITH TIME ZONE'1970-01-02 00:00:02.123456 -02:00'
,CAST('DVT' AS BYTEA),CAST('DVT DVT' AS BYTEA),'DVT B','DVT B')
,CAST('DVT' AS BYTEA),CAST('DVT DVT' AS BYTEA)
,CAST('DVT DVT' AS BYTEA),'DVT B','DVT B')
,(3,3333,123456789,123456789012345678,1234567890123456789012345
,123.123,123.11
--,123400,0.003
Expand All @@ -108,7 +111,8 @@ INSERT INTO pso_data_validator.dvt_ora2pg_types VALUES
,DATE'1970-01-03',TIMESTAMP'1970-01-03 00:00:01.123456'
,TIMESTAMP WITH TIME ZONE'1970-01-03 00:00:03.123456 -03:00'
--,TIMESTAMP WITH TIME ZONE'1970-01-03 00:00:03.123456 -03:00'
,CAST('DVT' AS BYTEA),CAST('DVT DVT DVT' AS BYTEA),'DVT C','DVT C'
,CAST('DVT' AS BYTEA),CAST('DVT DVT DVT' AS BYTEA)
,CAST('DVT DVT DVT' AS BYTEA),'DVT C','DVT C'
);

/* Following table used for validating generating table partitions */
Expand Down
14 changes: 10 additions & 4 deletions tests/system/data_sources/test_oracle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
"col_ts",
"col_tstz",
"col_raw",
"col_long_raw",
"col_blob",
"col_clob",
"col_nclob",
Expand Down Expand Up @@ -324,18 +325,22 @@ def test_column_validation_core_types_to_bigquery():
)
def test_column_validation_oracle_to_postgres():
parser = cli_tools.configure_arg_parser()
count_cols = ",".join(ORA2PG_COLUMNS)
count_cols = ",".join([_ for _ in ORA2PG_COLUMNS if _ not in ("col_long_raw")])
# TODO Change sum_cols and min_cols to include col_char_2,col_nchar_2 when issue-842 is complete.
# TODO Change sum_cols to include col_num_18 when issue-1007 is complete.
sum_cols = ",".join(
[
_
for _ in ORA2PG_COLUMNS
if _ not in ("col_char_2", "col_nchar_2", "col_num_18")
if _ not in ("col_char_2", "col_nchar_2", "col_num_18", "col_long_raw")
]
)
min_cols = ",".join(
[_ for _ in ORA2PG_COLUMNS if _ not in ("col_char_2", "col_nchar_2")]
[
_
for _ in ORA2PG_COLUMNS
if _ not in ("col_char_2", "col_nchar_2", "col_long_raw")
]
)
args = parser.parse_args(
[
Expand Down Expand Up @@ -423,7 +428,7 @@ def test_row_validation_core_types_to_bigquery():
)
def test_row_validation_oracle_to_postgres():
# TODO Change hash_cols below to include col_tstz when issue-706 is complete.
# TODO col_raw is blocked by issue-773 (is it even reasonable to expect binary columns to work here?)
# TODO col_raw/col_long_raw are blocked by issue-773 (is it even reasonable to expect binary columns to work here?)
# TODO Change hash_cols below to include col_nvarchar_30,col_nchar_2 when issue-772 is complete.
# Excluded col_float32,col_float64 due to the lossy nature of BINARY_FLOAT/DOUBLE.
# Excluded CLOB/NCLOB/BLOB columns because lob values cannot be concatenated
Expand All @@ -437,6 +442,7 @@ def test_row_validation_oracle_to_postgres():
"col_clob",
"col_nclob",
"col_raw",
"col_long_raw",
"col_float32",
"col_float64",
"col_tstz",
Expand Down
10 changes: 10 additions & 0 deletions third_party/ibis/ibis_oracle/datatypes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ def _get_type(col: _FieldDescription) -> dt.DataType:
cx_Oracle.DB_TYPE_TIMESTAMP: dt.Timestamp,
cx_Oracle.DB_TYPE_TIMESTAMP_TZ: dt.Timestamp(timezone="UTC"),
cx_Oracle.DB_TYPE_RAW: dt.Binary,
cx_Oracle.DB_TYPE_LONG_RAW: dt.Binary,
cx_Oracle.DB_TYPE_BFILE: dt.Binary,
cx_Oracle.DB_TYPE_NUMBER: dt.Decimal,
cx_Oracle.DB_TYPE_LONG: dt.String,
Expand All @@ -78,6 +79,15 @@ def _get_type(col: _FieldDescription) -> dt.DataType:
cx_Oracle.DB_TYPE_BINARY_DOUBLE: dt.Float64,
}

# SQL Alchemy doesn't support LONG RAW which drops us into Ibis 5.1.0 method:
# _handle_failed_column_type_inference().
# This method has a flaw that is doesn't prefix the table name with the schema
# which causes a table not found exception.
# I (nj@2024-02-15) felt the simplest change was to add LONG RAW to SQL Alchemy
# as a RAW variant, as below.
if "LONG RAW" not in OracleDialect_cx_oracle.ischema_names:
OracleDialect_cx_oracle.ischema_names["LONG RAW"] = oracle.RAW


@dt.dtype.register(OracleDialect_cx_oracle, sa.dialects.oracle.CLOB)
def sa_oracle_CLOB(_, satype, nullable=True):
Expand Down