Move filters to function

.gitignore

@@ -146,3 +146,5 @@ cython_debug/
 .idea
 test_artifacts/
 artifacts/
+
+pylintrc

rasgotransforms/rasgotransforms/macros/bigquery/aggregate_metrics.sql

@@ -1,4 +1,3 @@
-{% from 'filter.sql' import get_filter_statement %}
 {% from 'secondary_calculation.sql' import render_secondary_calculations %}
 
 {% macro calculate_timeseries_metric_values(

rasgotransforms/rasgotransforms/macros/distinct_values.sql

@@ -1,5 +1,3 @@
-{% from 'filter.sql' import get_filter_statement %}
-
 {% macro get_distinct_vals(
     columns,
     target_metric,

rasgotransforms/rasgotransforms/macros/expression_metrics.sql

@@ -1,5 +1,4 @@
 {% from 'aggregate_metrics.sql' import calculate_timeseries_metric_values %}
-{% from 'filter.sql' import combine_filters %}
 {% from 'secondary_calculation.sql' import render_secondary_calculations %}
 
 {% macro calculate_expression_metric_values(

rasgotransforms/rasgotransforms/macros/filter.sql

@@ -1,59 +0,0 @@
-{% macro get_filter_statement(filters) %}
-{% if filters %}
-{% if filters is string %}
-{{ filters }}
-{% else %}
-{% set logical_operator = namespace(value='AND') %}
-{% for filter in filters %}
-    {% if filter is not string %}
-        {% if filter is not mapping %}
-            {% set filter = dict(filter) %}
-        {% endif %}
-        {% if filter is mapping and 'compoundBoolean' in filter and filter['compoundBoolean'] %}
-            {% set logical_operator.value = filter['compoundBoolean'] %}
-        {% endif %}
-    {% endif %}
-{% endfor %}
-(
-    {% for filter in filters %}
-    {% if filter is not string and filter is not mapping %}
-    {% set filter = dict(filter) %}
-    {% endif %}
-    {% if 'columnName' in filter %}
-        {% do filter.__setitem__('column_name', filter.columnName) %}
-    {% endif %}
-    {% if 'comparisonValue' in filter %}
-        {% do filter.__setitem__('comparison_value', filter.comparisonValue) %}
-    {% endif %}
-    {% if filter is not mapping %}
-    {{ logical_operator.value + ' ' if not loop.first }}{{ filter }}
-    {% elif filter.operator|upper == 'CONTAINS' %}
-    {{ logical_operator.value + ' ' if not loop.first }}{{ filter.column_name }} like '%{{ filter.comparison_value }}%'
-    {% else %}
-    {{ logical_operator.value + ' ' if not loop.first }}{{ filter.column_name }} {{ filter.operator }} {{ filter.comparison_value }}
-    {% endif %}
-    {% endfor %}
-)
-{% endif %}
-{% else %}
-true
-{% endif %}
-{% endmacro %}
-
-
-{% macro combine_filters(filters_a, filters_b, condition) %}
-{% set condition = condition if condition is defined else 'AND' %}
-{% if filters_a and not filters_b %}
-{{ get_filter_statement(filters_a) }}
-{% elif filters_b and not filters_a %}
-{{ get_filter_statement(filters_b) }}
-{% elif not filters_a and not filters_b %}
-true
-{% else %}
-(
-    {{ get_filter_statement(filters_a)|indent }}
-    {{ condition }}
-    {{ get_filter_statement(filters_b)|indent }}
-)
-{% endif %}
-{% endmacro %}

rasgotransforms/rasgotransforms/macros/snowflake/aggregate_metrics.sql

@@ -1,4 +1,3 @@
-{% from 'filter.sql' import get_filter_statement %}
 {% from 'secondary_calculation.sql' import render_secondary_calculations %}
 
 {% macro calculate_timeseries_metric_values(

rasgotransforms/rasgotransforms/render/environment.py

@@ -1,9 +1,9 @@
 import re
 from datetime import datetime, timedelta
 from itertools import combinations, permutations, product
-from typing import Callable, Optional, Dict, Union
-from pathlib import Path
 from os.path import getmtime
+from pathlib import Path
+from typing import Callable, Dict, List, Optional, Union
 
 from jinja2 import Environment, BaseLoader
 from jinja2.exceptions import TemplateNotFound
@@ -12,19 +12,34 @@
 from rasgotransforms.exceptions import RenderException
 from rasgotransforms.main import DataWarehouse
 
+ALLOWED_OPERATORS = (
+    ">",
+    "<",
+    "=",
+    "<>",
+    ">=",
+    "<=",
+    "CONTAINS",
+    "IS NULL",
+    "IS NOT NULL",
+    "NOT CONTAINS",
+    "IN",
+    "NOT IN",
+)
+
 
 class RasgoEnvironment(Environment):
     def __init__(self, run_query: Optional[Callable] = None, dw_type: Optional[str] = None, *args, **kwargs):
         super().__init__(*args, extensions=self.rasgo_extensions, loader=RasgoLoader(), **kwargs)
         if not dw_type:
-            dw_type = 'snowflake'
+            dw_type = "snowflake"
         self._dw_type = DataWarehouse(dw_type)
         for filter_name, method in self.rasgo_filters.items():
             self.filters[filter_name] = method
         for name, value in self.rasgo_globals.items():
             self.globals[name] = value
         self._run_query = run_query
-        self.globals['run_query'] = self._run_query
+        self.globals["run_query"] = self._run_query
 
     @property
     def dw_type(self) -> DataWarehouse:
@@ -36,14 +51,16 @@ def dw_type(self, dw_type: str):
 
     @property
     def rasgo_extensions(self):
-        return ['jinja2.ext.do', 'jinja2.ext.loopcontrols']
+        return ["jinja2.ext.do", "jinja2.ext.loopcontrols"]
 
     @property
     def rasgo_globals(self):
         return {
             "min": min,
             "max": max,
             "cleanse_name": cleanse_template_symbol,
+            "get_filter_statement": get_filter_statement,
+            "combine_filters": combine_filters,
             "raise_exception": raise_exception,
             "itertools": {"combinations": combinations, "permutations": permutations, "product": product},
             "dw_type": lambda: self.dw_type.value,
@@ -73,40 +90,83 @@ def render(
 
         Source columns can is a mapping of table names to columns (e.g. {table_name: {column_name: column_type}})
         """
-        arguments['source_table'] = source_table
+        arguments["source_table"] = source_table
 
         if not override_globals:
             override_globals = {}
 
-        if source_columns and 'get_columns' not in override_globals:
+        if source_columns and "get_columns" not in override_globals:
 
             def get_columns(fqtn):
                 return source_columns[fqtn]
 
-            override_globals['get_columns'] = get_columns
-        if 'get_columns' in override_globals:
-            self.globals['get_columns'] = override_globals['get_columns']
+            override_globals["get_columns"] = get_columns
+        if "get_columns" in override_globals:
+            self.globals["get_columns"] = override_globals["get_columns"]
         try:
             template = self.from_string(source_code)
             rendered = template.render(**arguments, **override_globals)
         except Exception as e:
-            raise RenderException(e)
+            raise RenderException(e) from e
         return trim_blank_lines(rendered)
 
 
 def cleanse_template_symbol(symbol: str) -> str:
-    symbol = str(symbol).strip().replace(' ', '_').replace('-', '_')
-    symbol = re.sub('[^A-Za-z0-9_]+', '', symbol)
-    symbol = '_' + symbol if not symbol or symbol[0].isdecimal() else symbol
+    symbol = str(symbol).strip().replace(" ", "_").replace("-", "_")
+    symbol = re.sub("[^A-Za-z0-9_]+", "", symbol)
+    symbol = "_" + symbol if not symbol or symbol[0].isdecimal() else symbol
     return symbol
 
 
+def combine_filters(filters_a: Union[List, str], filters_b: Union[List, str], condition: str) -> str:
+    """
+    Parse & combine multiple filters, return a single SQL statement
+    """
+    condition = condition or "AND"
+    if filters_a and not filters_b:
+        return get_filter_statement(filters_a)
+    elif filters_b and not filters_a:
+        return get_filter_statement(filters_b)
+    elif not filters_a and not filters_b:
+        return "true"
+    return f"({get_filter_statement(filters_a)} {condition} {get_filter_statement(filters_b)})"
+
+
+def get_filter_statement(filters: Union[List, str]) -> str:
+    """
+    Parse a list of string or dict filters to a simple SQL string
+    """
+    if isinstance(filters, str):
+        return filters
+
+    filter_string = ""
+    for fil in filters:
+        fil: dict
+
+        # Handle variable casing from past versions: only support snake evnetually
+        column_name = fil.get("column_name", fil.get("columnName"))
+        operator = fil.get("operator")
+        comparison_value = fil.get("comparison_value", fil.get("comparisonValue"))
+        compound_boolean = fil.get("compound_boolean", fil.get("compoundBoolean"), "AND")
+
+        # Handle override operators
+        if operator not in ALLOWED_OPERATORS:
+            raise_exception(f"operator {operator} is not supported")
+        if operator == "CONTAINS":
+            operator = "LIKE"
+
+        compound_boolean = "" if filter_string == "" else compound_boolean
+        filter_string += f"{compound_boolean} {column_name} {operator} {comparison_value} "
+
+    return filter_string
+
+
 def raise_exception(message: str) -> None:
     raise Exception(message)
 
 
 def trim_blank_lines(sql: str) -> str:
-    return re.sub(r'[\n][\s]*\n', '\n', sql)
+    return re.sub(r"[\n][\s]*\n", "\n", sql)
 
 
 def get_timedelta(time_grain: str, interval: int) -> timedelta:
@@ -133,7 +193,7 @@ def get_timedelta(time_grain: str, interval: int) -> timedelta:
 class RasgoLoader(BaseLoader):
     def __init__(self, root_path=None):
         if not root_path:
-            root_path = Path(__file__).parent.parent / 'macros'
+            root_path = Path(__file__).parent.parent / "macros"
         self.root_path = root_path
 
     def get_source(self, environment: RasgoEnvironment, template):

rasgotransforms/rasgotransforms/transforms/clean/bigquery/clean.sql

@@ -56,7 +56,7 @@
     {{ output }} as {{ output_col_name }}
 {%- endmacro -%}
 
-{%- macro get_filter_statement(columns) -%}
+{%- macro get_where_clause(columns) -%}
     {%- set filter_statements = [] -%}
     {%- for column in columns.keys() %}
         {%- set output_col_name = column if columns[column].name is not defined else cleanse_name(columns[column].name) -%}
@@ -68,7 +68,7 @@
         {{ "where " if loop.first else "" }}{{ filter_statement }}{{ " \n    and " if not loop.last else "" }}
     {%- endfor %}
 {%- endmacro -%}
-{%- set filter_statement = get_filter_statement(columns) -%}
+{%- set filter_statement = get_where_clause(columns) -%}
 
 {%- if filter_statement == "" -%}
 select

rasgotransforms/rasgotransforms/transforms/clean/clean.sql

@@ -34,7 +34,7 @@
     {{ output }} as {{ output_col_name }}
 {%- endmacro -%}
 
-{%- macro get_filter_statement(columns) -%}
+{%- macro get_where_clause(columns) -%}
     {%- set filter_statements = [] -%}
     {%- for column in columns.keys() %}
         {%- set output_col_name = column if columns[column].name is not defined else cleanse_name(columns[column].name) -%}
@@ -46,7 +46,7 @@
         {{ "where " if loop.first else "" }}{{ filter_statement }}{{ " \n    and " if not loop.last else "" }}
     {%- endfor %}
 {%- endmacro -%}
-{%- set filter_statement = get_filter_statement(columns) -%}
+{%- set filter_statement = get_where_clause(columns) -%}
 
 {%- if filter_statement == "" -%}
 select

rasgotransforms/rasgotransforms/transforms/filter/filter.sql

@@ -1,4 +1,3 @@
-{% from 'filter.sql' import get_filter_statement %}
 {% if items is not defined %}
     {% if filter_statements is not defined %}
         {{ raise_exception('items is empty: there are no filters to apply') }}

rasgotransforms/rasgotransforms/transforms/plot/plot.sql

@@ -2,7 +2,6 @@
 {% from 'expression_metrics.sql' import calculate_expression_metric_values %}
 {% from 'distinct_values.sql' import get_distinct_vals %}
 {% from 'pivot.sql' import pivot_plot_values %}
-{% from 'filter.sql' import get_filter_statement, combine_filters %}
 {% from 'secondary_calculation.sql' import render_secondary_calculations, adjust_start_date %}
 {% set dimensions = group_by if group_by is defined else [] %}
 {% set flatten = flatten if flatten is defined else true %}