From b5e984c1e9d4855d9854dd119bfbfaba22b34494 Mon Sep 17 00:00:00 2001 From: Nam Nd Date: Fri, 8 Jan 2016 17:28:36 +0700 Subject: [PATCH 1/7] add feature having clause --- cubes/cells.py | 9 +++++--- cubes/sql/browser.py | 17 +++++++++++++-- cubes/sql/query.py | 52 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/cubes/cells.py b/cubes/cells.py index b85287cb..6f11ae3e 100644 --- a/cubes/cells.py +++ b/cubes/cells.py @@ -36,12 +36,13 @@ class Cell(object): """Part of a cube determined by slicing dimensions. Immutable object.""" - def __init__(self, cube=None, cuts=None): + def __init__(self, cube=None, cuts=None, having_clauses=None): if not isinstance(cube, Cube): raise ArgumentError("Cell cube should be sublcass of Cube, " "provided: %s" % type(cube).__name__) self.cube = cube self.cuts = cuts if cuts is not None else [] + self.having_clauses = having_clauses if having_clauses is not None else [] def __and__(self, other): """Returns a new cell that is a conjunction of the two provided @@ -51,13 +52,15 @@ def __and__(self, other): "cubes '%s' and '%s'." % (self.name, other.name)) cuts = self.cuts + other.cuts - return Cell(self.cube, cuts=cuts) + having_clauses = self.having_clauses + other.having_clauses + return Cell(self.cube, cuts=cuts, having_clause=having_clauses) def to_dict(self): """Returns a dictionary representation of the cell""" result = { "cube": str(self.cube.name), - "cuts": [cut.to_dict() for cut in self.cuts] + "cuts": [cut.to_dict() for cut in self.cuts], + "having_clauses": [clause.to_dict() for clause in self.having_clauses] } return result diff --git a/cubes/sql/browser.py b/cubes/sql/browser.py index bea8676c..78ed263f 100644 --- a/cubes/sql/browser.py +++ b/cubes/sql/browser.py @@ -579,12 +579,25 @@ def aggregation_statement(self, cell, aggregates, drilldown=None, else: selection += aggregate_cols + # namnd added: + # HAVING + # ------ + having_clauses = context.clause_for_having(cell) + havings = having_clauses["condition"] + group_clauses = having_clauses["groups"] + if group_by is None: + group_by = [] + for group in group_clauses: + if group not in group_by: + group_by.append(group) + statement = sql.expression.select(selection, from_obj=context.star, use_labels=True, whereclause=condition, - group_by=group_by) - + group_by=group_by, + having=havings) + # print("statement: {}".format(statement)) return (statement, context.get_labels(statement.columns)) def _log_statement(self, statement, label=None): diff --git a/cubes/sql/query.py b/cubes/sql/query.py index 27c6cda6..df14dc01 100644 --- a/cubes/sql/query.py +++ b/cubes/sql/query.py @@ -1111,3 +1111,55 @@ def column_for_split(self, split_cell, label=None): return split_column.label(label) + # namnd added + def clause_for_having(self, cell): + """Returns a clause for having clause and attr for group. If cell is empty, not contain having or cell is + `None` then returns `None`.""" + + if not cell: + return None + + clauses = self.clauses_for_having(cell.having_clauses) + condition = and_(*clauses["condition"]) + clauses["condition"] = condition + return clauses + + # namnd added + def clauses_for_having(self, having_clauses): + clauses = [] + groups = [] + for cut in having_clauses: + hierarchy = str(cut.hierarchy) if cut.hierarchy else None + if isinstance(cut, PointCut): + path = cut.path + hav_conds = self.having_condition(str(cut.dimension), + path, + hierarchy, cut.invert) + clauses.append(hav_conds["condition"]) + groups += hav_conds["group"] + # return one dict + dict_clause = {"groups": groups, "condition": clauses} + return dict_clause + + # namnd added + def having_condition(self, dim, path, hierarchy=None, invert=False): + """Returns a dict of `Condition` tuple (`attributes`, `conditions`, + `group_by`) dimension `dim` point at `path` and list group attrs use having. It is a compound + condition - one equality condition for each path element in form: + ``level[i].key = path[i]``""" + conditions = [] + groups = [] + levels = self.level_keys(dim, hierarchy, path) + for level_key, value in zip(levels, path): + # Prepare condition: dimension.level_key = path_value + column = self.column(level_key) + conditions.append(column == value) + groups.append(column) + + condition = sql.expression.and_(*conditions) + + if invert: + condition = sql.expression.not_(condition) + + dict_condition = {"group": groups, "condition": condition} + return dict_condition From 96f0bbe44b0274e8fd782df2fdc7b53f895e1037 Mon Sep 17 00:00:00 2001 From: Nam Nd Date: Sun, 10 Jan 2016 01:46:19 +0700 Subject: [PATCH 2/7] update feature having clause, support full cut type --- cubes/cells.py | 10 +++---- cubes/sql/browser.py | 25 +++++++++-------- cubes/sql/query.py | 65 +++++++++++++++----------------------------- 3 files changed, 40 insertions(+), 60 deletions(-) diff --git a/cubes/cells.py b/cubes/cells.py index 6f11ae3e..98acc298 100644 --- a/cubes/cells.py +++ b/cubes/cells.py @@ -36,13 +36,13 @@ class Cell(object): """Part of a cube determined by slicing dimensions. Immutable object.""" - def __init__(self, cube=None, cuts=None, having_clauses=None): + def __init__(self, cube=None, cuts=None, having_cuts=None): if not isinstance(cube, Cube): raise ArgumentError("Cell cube should be sublcass of Cube, " "provided: %s" % type(cube).__name__) self.cube = cube self.cuts = cuts if cuts is not None else [] - self.having_clauses = having_clauses if having_clauses is not None else [] + self.having_cuts = having_cuts if having_cuts is not None else [] def __and__(self, other): """Returns a new cell that is a conjunction of the two provided @@ -52,15 +52,15 @@ def __and__(self, other): "cubes '%s' and '%s'." % (self.name, other.name)) cuts = self.cuts + other.cuts - having_clauses = self.having_clauses + other.having_clauses - return Cell(self.cube, cuts=cuts, having_clause=having_clauses) + having_cuts = self.having_cuts + other.having_cuts + return Cell(self.cube, cuts=cuts, having_cuts=having_cuts) def to_dict(self): """Returns a dictionary representation of the cell""" result = { "cube": str(self.cube.name), "cuts": [cut.to_dict() for cut in self.cuts], - "having_clauses": [clause.to_dict() for clause in self.having_clauses] + "having_cuts": [clause.to_dict() for clause in self.having_cuts] } return result diff --git a/cubes/sql/browser.py b/cubes/sql/browser.py index 78ed263f..3fa70dc6 100644 --- a/cubes/sql/browser.py +++ b/cubes/sql/browser.py @@ -579,25 +579,26 @@ def aggregation_statement(self, cell, aggregates, drilldown=None, else: selection += aggregate_cols - # namnd added: - # HAVING + # madman: HAVING # ------ - having_clauses = context.clause_for_having(cell) - havings = having_clauses["condition"] - group_clauses = having_clauses["groups"] - if group_by is None: - group_by = [] - for group in group_clauses: - if group not in group_by: - group_by.append(group) + having_clauses = None + colums_and_havings = context.colums_and_having_cut_for_cell(cell) + if colums_and_havings is not None: + having_clauses = colums_and_havings[1] + group_clauses = colums_and_havings[0] + if group_by is None: + group_by = [] + for group in group_clauses: + if group not in group_by: + group_by.append(group) statement = sql.expression.select(selection, from_obj=context.star, use_labels=True, whereclause=condition, group_by=group_by, - having=havings) - # print("statement: {}".format(statement)) + having=having_clauses) + return (statement, context.get_labels(statement.columns)) def _log_statement(self, statement, label=None): diff --git a/cubes/sql/query.py b/cubes/sql/query.py index df14dc01..41907fb7 100644 --- a/cubes/sql/query.py +++ b/cubes/sql/query.py @@ -1111,55 +1111,34 @@ def column_for_split(self, split_cell, label=None): return split_column.label(label) - # namnd added - def clause_for_having(self, cell): - """Returns a clause for having clause and attr for group. If cell is empty, not contain having or cell is + # madman: get having clause and attributes + def colums_and_having_cut_for_cell(self, cell): + """Returns attributes and having clause. If cell is empty, not contain having or cell is `None` then returns `None`.""" if not cell: return None - clauses = self.clauses_for_having(cell.having_clauses) - condition = and_(*clauses["condition"]) - clauses["condition"] = condition - return clauses + having_cuts = cell.having_cuts + hav_condition = and_(*self.conditions_for_cuts(having_cuts)) - # namnd added - def clauses_for_having(self, having_clauses): - clauses = [] - groups = [] - for cut in having_clauses: - hierarchy = str(cut.hierarchy) if cut.hierarchy else None - if isinstance(cut, PointCut): - path = cut.path - hav_conds = self.having_condition(str(cut.dimension), - path, - hierarchy, cut.invert) - clauses.append(hav_conds["condition"]) - groups += hav_conds["group"] - # return one dict - dict_clause = {"groups": groups, "condition": clauses} - return dict_clause - - # namnd added - def having_condition(self, dim, path, hierarchy=None, invert=False): - """Returns a dict of `Condition` tuple (`attributes`, `conditions`, - `group_by`) dimension `dim` point at `path` and list group attrs use having. It is a compound - condition - one equality condition for each path element in form: - ``level[i].key = path[i]``""" - conditions = [] - groups = [] - levels = self.level_keys(dim, hierarchy, path) - for level_key, value in zip(levels, path): - # Prepare condition: dimension.level_key = path_value - column = self.column(level_key) - conditions.append(column == value) - groups.append(column) + if hav_condition is None: + return None - condition = sql.expression.and_(*conditions) + colums = self.colums_in_having_cuts(having_cuts) - if invert: - condition = sql.expression.not_(condition) + return (colums, hav_condition) + + # madman: get attributes in having cuts + def colums_in_having_cuts(self, having_cus): + + columns = [] + + for cut in having_cus: + hierarchy = str(cut.hierarchy) if cut.hierarchy else None + levels = self.hierarchies[(str(cut.dimension), hierarchy)] + for level_key in levels: + column = self.column(level_key) + columns.append(column) - dict_condition = {"group": groups, "condition": condition} - return dict_condition + return columns From 60396b1b5ceed6893b153e93b41ccd073715752d Mon Sep 17 00:00:00 2001 From: Nam Nd Date: Mon, 11 Jan 2016 10:11:00 +0700 Subject: [PATCH 3/7] fix having cut, only support PointCut --- cubes/sql/query.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/cubes/sql/query.py b/cubes/sql/query.py index 41907fb7..e3276d2d 100644 --- a/cubes/sql/query.py +++ b/cubes/sql/query.py @@ -1120,7 +1120,7 @@ def colums_and_having_cut_for_cell(self, cell): return None having_cuts = cell.having_cuts - hav_condition = and_(*self.conditions_for_cuts(having_cuts)) + hav_condition = and_(*self.conditions_for_having_cuts(having_cuts)) if hav_condition is None: return None @@ -1142,3 +1142,26 @@ def colums_in_having_cuts(self, having_cus): columns.append(column) return columns + + # madman: get condition in having cuts + def conditions_for_having_cuts(self, having_cuts): + """ + Having cuts has only support type PointCut + """ + + conditions = [] + + for cut in having_cuts: + hierarchy = str(cut.hierarchy) if cut.hierarchy else None + + if isinstance(cut, PointCut): + path = cut.path + condition = self.condition_for_point(str(cut.dimension), + path, + hierarchy, cut.invert) + else: + raise ArgumentError("Having cut has not support type %s" % type(cut)) + + conditions.append(condition) + + return conditions \ No newline at end of file From 4ff48f9d9c2ba81ee25836b7006695782f01896f Mon Sep 17 00:00:00 2001 From: Nam Nd Date: Tue, 12 Jan 2016 18:33:20 +0700 Subject: [PATCH 4/7] fix SetCut --- cubes/sql/query.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/cubes/sql/query.py b/cubes/sql/query.py index e3276d2d..2615601a 100644 --- a/cubes/sql/query.py +++ b/cubes/sql/query.py @@ -966,19 +966,10 @@ def conditions_for_cuts(self, cuts): hierarchy, cut.invert) elif isinstance(cut, SetCut): - set_conds = [] - - for path in cut.paths: - condition = self.condition_for_point(str(cut.dimension), - path, + condition = self.condition_for_set(str(cut.dimension), + cut.paths, str(cut.hierarchy), - invert=False) - set_conds.append(condition) - - condition = sql.expression.or_(*set_conds) - - if cut.invert: - condition = sql.expression.not_(condition) + invert=cut.invert) elif isinstance(cut, RangeCut): condition = self.range_condition(str(cut.dimension), @@ -1164,4 +1155,25 @@ def conditions_for_having_cuts(self, having_cuts): conditions.append(condition) - return conditions \ No newline at end of file + return conditions + + # madman: fix setcut + def condition_for_set(self, dim, path, hierarchy=None, invert=False): + """Returns a `Condition` tuple (`attributes`, `conditions`, + `group_by`) dimension `dim` point at `path`. It is a compound + condition - one equality condition for each path element in form: + ``level[i].key IN (path[i])``""" + conditions = [] + levels = self.level_keys(dim, hierarchy, path) + for level_key, value in zip(levels, path): + column = self.column(level_key) + values = [] + for v in value: + values.append([v]) + if invert: + condition = sql.expression.tuple_(column).notin_(values) + else: + condition = sql.expression.tuple_(column).in_(values) + conditions.append(condition) + condition = sql.expression.and_(*conditions) + return condition \ No newline at end of file From be3a2e08d70ed6b802fe4a13bd26dd3fbc78b55f Mon Sep 17 00:00:00 2001 From: Nam Nd Date: Wed, 13 Jan 2016 15:51:37 +0700 Subject: [PATCH 5/7] edit feature filter set for summary or not --- cubes/sql/browser.py | 2 +- cubes/sql/query.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cubes/sql/browser.py b/cubes/sql/browser.py index 3fa70dc6..869156d5 100644 --- a/cubes/sql/browser.py +++ b/cubes/sql/browser.py @@ -564,7 +564,7 @@ def aggregation_statement(self, cell, aggregates, drilldown=None, # WHERE # ----- - condition = context.condition_for_cell(cell) + condition = context.condition_for_cell(cell, for_summary) group_by = selection[:] if not for_summary else None diff --git a/cubes/sql/query.py b/cubes/sql/query.py index 2615601a..f0a96a67 100644 --- a/cubes/sql/query.py +++ b/cubes/sql/query.py @@ -938,18 +938,18 @@ def get_columns(self, refs): return [self._columns[ref] for ref in refs] - def condition_for_cell(self, cell): + def condition_for_cell(self, cell, for_summary=None): """Returns a condition for cell `cell`. If cell is empty or cell is `None` then returns `None`.""" if not cell: return None - condition = and_(*self.conditions_for_cuts(cell.cuts)) + condition = and_(*self.conditions_for_cuts(cell.cuts, for_summary)) return condition - def conditions_for_cuts(self, cuts): + def conditions_for_cuts(self, cuts, for_summary=None): """Constructs conditions for all cuts in the `cell`. Returns a list of SQL conditional expressions. """ @@ -966,7 +966,8 @@ def conditions_for_cuts(self, cuts): hierarchy, cut.invert) elif isinstance(cut, SetCut): - condition = self.condition_for_set(str(cut.dimension), + if not for_summary or cut.hidden is not True: + condition = self.condition_for_set(str(cut.dimension), cut.paths, str(cut.hierarchy), invert=cut.invert) From 42ffdde25541b7db7e4ac41d80ca19659f491a7b Mon Sep 17 00:00:00 2001 From: Nam Nd Date: Fri, 15 Jan 2016 00:57:00 +0700 Subject: [PATCH 6/7] add function sum_distinct --- cubes/sql/functions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cubes/sql/functions.py b/cubes/sql/functions.py index 6c280c84..04fbcfb8 100644 --- a/cubes/sql/functions.py +++ b/cubes/sql/functions.py @@ -153,6 +153,11 @@ def __init__(self, name): function = lambda x: sql.functions.count(sql.expression.distinct(x)) super(FactCountDistinctFunction, self).__init__(name, function) +class FactSumDistinctFunction(AggregateFunction): + def __init__(self, name): + """Creates a function that provides distinct fact (record) counts.""" + function = lambda x: sql.functions.sum(sql.expression.distinct(x)) + super(FactSumDistinctFunction, self).__init__(name, function) class avg(ReturnTypeFromArgs): pass @@ -169,6 +174,7 @@ class variance(ReturnTypeFromArgs): _functions = ( SummaryCoalescingFunction("sum", sql.functions.sum), + FactSumDistinctFunction("sum_distinct"), SummaryCoalescingFunction("count_nonempty", sql.functions.count), FactCountFunction("count"), FactCountDistinctFunction("count_distinct"), From 2932e35181c0f9f6e6214a69d97486faccf2ee63 Mon Sep 17 00:00:00 2001 From: Nam Nd Date: Thu, 21 Jan 2016 18:34:21 +0700 Subject: [PATCH 7/7] fixed cut in summary --- cubes/sql/query.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cubes/sql/query.py b/cubes/sql/query.py index f0a96a67..fb058a43 100644 --- a/cubes/sql/query.py +++ b/cubes/sql/query.py @@ -959,6 +959,7 @@ def conditions_for_cuts(self, cuts, for_summary=None): for cut in cuts: hierarchy = str(cut.hierarchy) if cut.hierarchy else None + condition = None if isinstance(cut, PointCut): path = cut.path condition = self.condition_for_point(str(cut.dimension), @@ -981,7 +982,8 @@ def conditions_for_cuts(self, cuts, for_summary=None): else: raise ArgumentError("Unknown cut type %s" % type(cut)) - conditions.append(condition) + if condition is not None: + conditions.append(condition) return conditions