Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: to_char support for Oracle and Postgres #632

Merged
merged 3 commits into from
Nov 10, 2022
Merged

feat: to_char support for Oracle and Postgres #632

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a9885ed
feat: to_char support for oracle and postgres
sramsrinivasan Nov 8, 2022
07bcc09
support user defined to_char fmt
nehanene15 Nov 8, 2022
221e30f
lint
nehanene15 Nov 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions data_validation/query_builder/query_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,13 +307,16 @@ def hash(config, fields):
)

@staticmethod
def ifnull(config, fields):
config["default_string"] = (
ibis.literal("DEFAULT_REPLACEMENT_STRING")
if config.get("default_null_string") is None
else ibis.literal(config.get("default_null_string"))
def to_char(config, fields):
fmt = ibis.literal(config.get("default_to_char_fmt", "FM90.099"))
return CalculatedField(
ibis.expr.api.NumericValue.to_char, config, fields, fmt=fmt
)
fields = [fields[0], config["default_string"]]

@staticmethod
def ifnull(config, fields):
default_null_string = ibis.literal(config.get("default_null_string", "DEFAULT_REPLACEMENT_STRING"))
fields = [fields[0], default_null_string]
return CalculatedField(
ibis.expr.api.ValueExpr.fillna,
config,
Expand Down
20 changes: 18 additions & 2 deletions third_party/ibis/ibis_addon/operations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
from data_validation.clients import _raise_missing_client_error
from ibis_bigquery.compiler import reduction as bq_reduction, BigQueryExprTranslator
from ibis.expr.operations import Arg, Comparison, Reduction, ValueOp
from ibis.expr.types import BinaryValue, IntegerColumn, StringValue
from ibis.expr.types import BinaryValue, IntegerColumn, StringValue, NumericValue
from ibis.backends.impala.compiler import ImpalaExprTranslator
from ibis.backends.pandas import client as _pandas_client
from ibis.backends.base_sqlalchemy.alchemy import AlchemyExprTranslator
Expand Down Expand Up @@ -66,6 +66,10 @@ class HashBytes(ValueOp):
how = Arg(rlz.isin({"sha256", "farm_fingerprint"}))
output_type = rlz.shape_like("arg", "binary")

class ToChar(ValueOp):
arg = Arg(rlz.one_of([rlz.value(dt.Decimal), rlz.value(dt.float64)]))
fmt = Arg(rlz.string)
output_type = rlz.shape_like("arg", dt.string)

class RawSQL(Comparison):
pass
Expand Down Expand Up @@ -95,6 +99,10 @@ def compile_hashbytes(binary_value, how):
return HashBytes(binary_value, how=how).to_expr()


def compile_to_char(numeric_value, fmt):
return ToChar(numeric_value, fmt=fmt).to_expr()


def format_hash_bigquery(translator, expr):
arg, how = expr.op().args
compiled_arg = translator.translate(arg)
Expand Down Expand Up @@ -188,13 +196,19 @@ def sa_format_hashbytes_postgres(translator, expr):
hash_func = sa.func.sha256(convert)
return sa.func.encode(hash_func, sa.sql.literal_column("'hex'"))

def sa_format_to_char(translator, expr):
arg, fmt = expr.op().args
compiled_arg = translator.translate(arg)
compiled_fmt = translator.translate(fmt)
return sa.func.to_char(compiled_arg, compiled_fmt)

_pandas_client._inferable_pandas_dtypes["floating"] = _pandas_client.dt.float64
IntegerColumn.bit_xor = ibis.expr.api._agg_function("bit_xor", BitXor, True)
BinaryValue.hash = compile_hash
StringValue.hash = compile_hash
BinaryValue.hashbytes = compile_hashbytes
StringValue.hashbytes = compile_hashbytes
NumericValue.to_char = compile_to_char
BigQueryExprTranslator._registry[BitXor] = bq_reduction("BIT_XOR")
BigQueryExprTranslator._registry[Hash] = format_hash_bigquery
BigQueryExprTranslator._registry[HashBytes] = format_hashbytes_bigquery
Expand All @@ -209,7 +223,9 @@ def sa_format_hashbytes_postgres(translator, expr):
ImpalaExprTranslator._registry[HashBytes] = format_hashbytes_hive
OracleExprTranslator._registry[RawSQL] = sa_format_raw_sql
OracleExprTranslator._registry[HashBytes] = sa_format_hashbytes_oracle
OracleExprTranslator._registry[ToChar] = sa_format_to_char
TeradataExprTranslator._registry[RawSQL] = format_raw_sql
TeradataExprTranslator._registry[HashBytes] = format_hashbytes_teradata
PostgreSQLExprTranslator._registry[HashBytes] = sa_format_hashbytes_postgres
PostgreSQLExprTranslator._registry[RawSQL] = sa_format_raw_sql
PostgreSQLExprTranslator._registry[RawSQL] = sa_format_raw_sql
PostgreSQLExprTranslator._registry[ToChar] = sa_format_to_char