sqlglot.optimizer.canonicalize API documentation

Edit on GitHub

# sqlglot.optimizer.canonicalize

View Source

      1from __future__ import annotations
      2
      3import itertools
      4import typing as t
      5
      6from sqlglot import exp
      7from sqlglot.helper import is_date_unit, is_iso_date, is_iso_datetime
      8
      9
     10def canonicalize(expression: exp.Expression) -> exp.Expression:
     11    """Converts a sql expression into a standard form.
     12
     13    This method relies on annotate_types because many of the
     14    conversions rely on type inference.
     15
     16    Args:
     17        expression: The expression to canonicalize.
     18    """
     19
     20    def _canonicalize(expression: exp.Expression) -> exp.Expression:
     21        expression = add_text_to_concat(expression)
     22        expression = replace_date_funcs(expression)
     23        expression = coerce_type(expression)
     24        expression = remove_redundant_casts(expression)
     25        expression = ensure_bools(expression, _replace_int_predicate)
     26        expression = remove_ascending_order(expression)
     27        return expression
     28
     29    return exp.replace_tree(expression, _canonicalize)
     30
     31
     32def add_text_to_concat(node: exp.Expression) -> exp.Expression:
     33    if isinstance(node, exp.Add) and node.type and node.type.this in exp.DataType.TEXT_TYPES:
     34        node = exp.Concat(expressions=[node.left, node.right])
     35    return node
     36
     37
     38def replace_date_funcs(node: exp.Expression) -> exp.Expression:
     39    if (
     40        isinstance(node, (exp.Date, exp.TsOrDsToDate))
     41        and not node.expressions
     42        and not node.args.get("zone")
     43    ):
     44        return exp.cast(node.this, to=exp.DataType.Type.DATE)
     45    if isinstance(node, exp.Timestamp) and not node.args.get("zone"):
     46        if not node.type:
     47            from sqlglot.optimizer.annotate_types import annotate_types
     48
     49            node = annotate_types(node)
     50        return exp.cast(node.this, to=node.type or exp.DataType.Type.TIMESTAMP)
     51
     52    return node
     53
     54
     55COERCIBLE_DATE_OPS = (
     56    exp.Add,
     57    exp.Sub,
     58    exp.EQ,
     59    exp.NEQ,
     60    exp.GT,
     61    exp.GTE,
     62    exp.LT,
     63    exp.LTE,
     64    exp.NullSafeEQ,
     65    exp.NullSafeNEQ,
     66)
     67
     68
     69def coerce_type(node: exp.Expression) -> exp.Expression:
     70    if isinstance(node, COERCIBLE_DATE_OPS):
     71        _coerce_date(node.left, node.right)
     72    elif isinstance(node, exp.Between):
     73        _coerce_date(node.this, node.args["low"])
     74    elif isinstance(node, exp.Extract) and not node.expression.type.is_type(
     75        *exp.DataType.TEMPORAL_TYPES
     76    ):
     77        _replace_cast(node.expression, exp.DataType.Type.DATETIME)
     78    elif isinstance(node, (exp.DateAdd, exp.DateSub, exp.DateTrunc)):
     79        _coerce_timeunit_arg(node.this, node.unit)
     80    elif isinstance(node, exp.DateDiff):
     81        _coerce_datediff_args(node)
     82
     83    return node
     84
     85
     86def remove_redundant_casts(expression: exp.Expression) -> exp.Expression:
     87    if (
     88        isinstance(expression, exp.Cast)
     89        and expression.this.type
     90        and expression.to.this == expression.this.type.this
     91    ):
     92        return expression.this
     93    return expression
     94
     95
     96def ensure_bools(
     97    expression: exp.Expression, replace_func: t.Callable[[exp.Expression], None]
     98) -> exp.Expression:
     99    if isinstance(expression, exp.Connector):
    100        replace_func(expression.left)
    101        replace_func(expression.right)
    102    elif isinstance(expression, exp.Not):
    103        replace_func(expression.this)
    104        # We can't replace num in CASE x WHEN num ..., because it's not the full predicate
    105    elif isinstance(expression, exp.If) and not (
    106        isinstance(expression.parent, exp.Case) and expression.parent.this
    107    ):
    108        replace_func(expression.this)
    109    elif isinstance(expression, (exp.Where, exp.Having)):
    110        replace_func(expression.this)
    111
    112    return expression
    113
    114
    115def remove_ascending_order(expression: exp.Expression) -> exp.Expression:
    116    if isinstance(expression, exp.Ordered) and expression.args.get("desc") is False:
    117        # Convert ORDER BY a ASC to ORDER BY a
    118        expression.set("desc", None)
    119
    120    return expression
    121
    122
    123def _coerce_date(a: exp.Expression, b: exp.Expression) -> None:
    124    for a, b in itertools.permutations([a, b]):
    125        if isinstance(b, exp.Interval):
    126            a = _coerce_timeunit_arg(a, b.unit)
    127        if (
    128            a.type
    129            and a.type.this in exp.DataType.TEMPORAL_TYPES
    130            and b.type
    131            and b.type.this in exp.DataType.TEXT_TYPES
    132        ):
    133            _replace_cast(b, exp.DataType.Type.DATETIME)
    134
    135
    136def _coerce_timeunit_arg(arg: exp.Expression, unit: t.Optional[exp.Expression]) -> exp.Expression:
    137    if not arg.type:
    138        return arg
    139
    140    if arg.type.this in exp.DataType.TEXT_TYPES:
    141        date_text = arg.name
    142        is_iso_date_ = is_iso_date(date_text)
    143
    144        if is_iso_date_ and is_date_unit(unit):
    145            return arg.replace(exp.cast(arg.copy(), to=exp.DataType.Type.DATE))
    146
    147        # An ISO date is also an ISO datetime, but not vice versa
    148        if is_iso_date_ or is_iso_datetime(date_text):
    149            return arg.replace(exp.cast(arg.copy(), to=exp.DataType.Type.DATETIME))
    150
    151    elif arg.type.this == exp.DataType.Type.DATE and not is_date_unit(unit):
    152        return arg.replace(exp.cast(arg.copy(), to=exp.DataType.Type.DATETIME))
    153
    154    return arg
    155
    156
    157def _coerce_datediff_args(node: exp.DateDiff) -> None:
    158    for e in (node.this, node.expression):
    159        if e.type.this not in exp.DataType.TEMPORAL_TYPES:
    160            e.replace(exp.cast(e.copy(), to=exp.DataType.Type.DATETIME))
    161
    162
    163def _replace_cast(node: exp.Expression, to: exp.DataType.Type) -> None:
    164    node.replace(exp.cast(node.copy(), to=to))
    165
    166
    167# this was originally designed for presto, there is a similar transform for tsql
    168# this is different in that it only operates on int types, this is because
    169# presto has a boolean type whereas tsql doesn't (people use bits)
    170# with y as (select true as x) select x = 0 FROM y -- illegal presto query
    171def _replace_int_predicate(expression: exp.Expression) -> None:
    172    if isinstance(expression, exp.Coalesce):
    173        for child in expression.iter_expressions():
    174            _replace_int_predicate(child)
    175    elif expression.type and expression.type.this in exp.DataType.INTEGER_TYPES:
    176        expression.replace(expression.neq(0))

def canonicalize( expression: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression: View Source

    11def canonicalize(expression: exp.Expression) -> exp.Expression:
    12    """Converts a sql expression into a standard form.
    13
    14    This method relies on annotate_types because many of the
    15    conversions rely on type inference.
    16
    17    Args:
    18        expression: The expression to canonicalize.
    19    """
    20
    21    def _canonicalize(expression: exp.Expression) -> exp.Expression:
    22        expression = add_text_to_concat(expression)
    23        expression = replace_date_funcs(expression)
    24        expression = coerce_type(expression)
    25        expression = remove_redundant_casts(expression)
    26        expression = ensure_bools(expression, _replace_int_predicate)
    27        expression = remove_ascending_order(expression)
    28        return expression
    29
    30    return exp.replace_tree(expression, _canonicalize)

Converts a sql expression into a standard form.

This method relies on annotate_types because many of the conversions rely on type inference.

###### Arguments:

- expression: The expression to canonicalize.

def add_text_to_concat(node: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression: View Source

    33def add_text_to_concat(node: exp.Expression) -> exp.Expression:
    34    if isinstance(node, exp.Add) and node.type and node.type.this in exp.DataType.TEXT_TYPES:
    35        node = exp.Concat(expressions=[node.left, node.right])
    36    return node

def replace_date_funcs(node: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression: View Source

    39def replace_date_funcs(node: exp.Expression) -> exp.Expression:
    40    if (
    41        isinstance(node, (exp.Date, exp.TsOrDsToDate))
    42        and not node.expressions
    43        and not node.args.get("zone")
    44    ):
    45        return exp.cast(node.this, to=exp.DataType.Type.DATE)
    46    if isinstance(node, exp.Timestamp) and not node.args.get("zone"):
    47        if not node.type:
    48            from sqlglot.optimizer.annotate_types import annotate_types
    49
    50            node = annotate_types(node)
    51        return exp.cast(node.this, to=node.type or exp.DataType.Type.TIMESTAMP)
    52
    53    return node

COERCIBLE_DATE_OPS = (<class 'sqlglot.expressions.Add'>, <class 'sqlglot.expressions.Sub'>, <class 'sqlglot.expressions.EQ'>, <class 'sqlglot.expressions.NEQ'>, <class 'sqlglot.expressions.GT'>, <class 'sqlglot.expressions.GTE'>, <class 'sqlglot.expressions.LT'>, <class 'sqlglot.expressions.LTE'>, <class 'sqlglot.expressions.NullSafeEQ'>, <class 'sqlglot.expressions.NullSafeNEQ'>)
def coerce_type(node: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression: View Source

    70def coerce_type(node: exp.Expression) -> exp.Expression:
    71    if isinstance(node, COERCIBLE_DATE_OPS):
    72        _coerce_date(node.left, node.right)
    73    elif isinstance(node, exp.Between):
    74        _coerce_date(node.this, node.args["low"])
    75    elif isinstance(node, exp.Extract) and not node.expression.type.is_type(
    76        *exp.DataType.TEMPORAL_TYPES
    77    ):
    78        _replace_cast(node.expression, exp.DataType.Type.DATETIME)
    79    elif isinstance(node, (exp.DateAdd, exp.DateSub, exp.DateTrunc)):
    80        _coerce_timeunit_arg(node.this, node.unit)
    81    elif isinstance(node, exp.DateDiff):
    82        _coerce_datediff_args(node)
    83
    84    return node

def remove_redundant_casts( expression: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression: View Source

    87def remove_redundant_casts(expression: exp.Expression) -> exp.Expression:
    88    if (
    89        isinstance(expression, exp.Cast)
    90        and expression.this.type
    91        and expression.to.this == expression.this.type.this
    92    ):
    93        return expression.this
    94    return expression

def ensure_bools( expression: sqlglot.expressions.Expression, replace_func: Callable[[sqlglot.expressions.Expression], NoneType]) -> sqlglot.expressions.Expression: View Source

     97def ensure_bools(
     98    expression: exp.Expression, replace_func: t.Callable[[exp.Expression], None]
     99) -> exp.Expression:
    100    if isinstance(expression, exp.Connector):
    101        replace_func(expression.left)
    102        replace_func(expression.right)
    103    elif isinstance(expression, exp.Not):
    104        replace_func(expression.this)
    105        # We can't replace num in CASE x WHEN num ..., because it's not the full predicate
    106    elif isinstance(expression, exp.If) and not (
    107        isinstance(expression.parent, exp.Case) and expression.parent.this
    108    ):
    109        replace_func(expression.this)
    110    elif isinstance(expression, (exp.Where, exp.Having)):
    111        replace_func(expression.this)
    112
    113    return expression

def remove_ascending_order( expression: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression: View Source

    116def remove_ascending_order(expression: exp.Expression) -> exp.Expression:
    117    if isinstance(expression, exp.Ordered) and expression.args.get("desc") is False:
    118        # Convert ORDER BY a ASC to ORDER BY a
    119        expression.set("desc", None)
    120
    121    return expression