Skip to content

I
incubator-superset

qq_22812535 / incubator-superset
与 Fork 源项目一致

从无法访问的项目Fork

通知 3
Star 0
Fork 0
I
incubator-superset

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

未验证 提交 35c15b8b 编写于 作者: Y Yongjie Zhao 提交者： Ville Brofeldt
浏览文件
操作

refactor: from superset.utils.core break down date_parser (#12408)

上级 90915db6
隐藏空白更改
内联 并排
Showing with 753 addition and 704 deletion
superset/common/query_object.py
浏览文件 @ 35c15b8b
......@@ -28,12 +28,8 @@ from superset import app, is_feature_enabled
from superset.exceptions import QueryObjectValidationError
from superset.typing import Metric
from superset.utils import pandas_postprocessing
from superset.utils.core import (
DTTM_ALIAS,
get_since_until,
json_int_dttm_ser,
parse_human_timedelta,
)
from superset.utils.core import DTTM_ALIAS, json_int_dttm_ser
from superset.utils.date_parser import get_since_until, parse_human_timedelta
from superset.views.utils import get_time_range_endpoints
config = app.config
......
superset/connectors/druid/models.py
浏览文件 @ 35c15b8b
......@@ -57,6 +57,7 @@ from superset.models.core import Database
from superset.models.helpers import AuditMixinNullable, ImportExportMixin, QueryResult
from superset.typing import FilterValues, Granularity, Metric, QueryObjectDict
from superset.utils import core as utils
from superset.utils.date_parser import parse_human_datetime, parse_human_timedelta
try:
import requests
......@@ -777,7 +778,7 @@ class DruidDatasource(Model, BaseDatasource):
granularity["timeZone"] = timezone
if origin:
dttm = utils.parse_human_datetime(origin)
dttm = parse_human_datetime(origin)
assert dttm
granularity["origin"] = dttm.isoformat()
......@@ -795,7 +796,7 @@ class DruidDatasource(Model, BaseDatasource):
else:
granularity["type"] = "duration"
granularity["duration"] = (
utils.parse_human_timedelta(period_name).total_seconds() # type: ignore
parse_human_timedelta(period_name).total_seconds() # type: ignore
* 1000
)
return granularity
......@@ -938,7 +939,7 @@ class DruidDatasource(Model, BaseDatasource):
)
# TODO: Use Lexicographic TopNMetricSpec once supported by PyDruid
if self.fetch_values_from:
from_dttm = utils.parse_human_datetime(self.fetch_values_from)
from_dttm = parse_human_datetime(self.fetch_values_from)
assert from_dttm
else:
from_dttm = datetime(1970, 1, 1)
......@@ -1426,7 +1427,7 @@ class DruidDatasource(Model, BaseDatasource):
time_offset = DruidDatasource.time_offset(query_obj["granularity"])
def increment_timestamp(ts: str) -> datetime:
dt = utils.parse_human_datetime(ts).replace(tzinfo=DRUID_TZ)
dt = parse_human_datetime(ts).replace(tzinfo=DRUID_TZ)
return dt + timedelta(milliseconds=time_offset)
if DTTM_ALIAS in df.columns and time_offset:
......
superset/migrations/versions/3dda56f1c4c6_migrate_num_period_compare_and_period_.py
浏览文件 @ 35c15b8b
......@@ -33,7 +33,7 @@ from sqlalchemy import Column, Integer, String, Text
from sqlalchemy.ext.declarative import declarative_base
from superset import db
from superset.utils.core import parse_human_timedelta
from superset.utils.date_parser import parse_human_timedelta
revision = "3dda56f1c4c6"
down_revision = "bddc498dd179"
......
superset/tasks/cache.py
浏览文件 @ 35c15b8b
......@@ -31,7 +31,7 @@ from superset.models.core import Log
from superset.models.dashboard import Dashboard
from superset.models.slice import Slice
from superset.models.tags import Tag, TaggedObject
from superset.utils.core import parse_human_datetime
from superset.utils.date_parser import parse_human_datetime
from superset.views.utils import build_extra_filters
logger = get_task_logger(__name__)
......
superset/utils/core.py
浏览文件 @ 35c15b8b
......@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
"""Utility functions used across Superset"""
import calendar
import decimal
import errno
import functools
......@@ -39,7 +38,6 @@ from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import formatdate
from enum import Enum
from time import struct_time
from timeit import default_timer
from types import TracebackType
from typing import (
......@@ -65,29 +63,14 @@ import bleach
import markdown as md
import numpy as np
import pandas as pd
import parsedatetime
import sqlalchemy as sa
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.backends.openssl.x509 import _Certificate
from dateutil.parser import parse
from dateutil.relativedelta import relativedelta
from flask import current_app, flash, g, Markup, render_template
from flask_appbuilder import SQLA
from flask_appbuilder.security.sqla.models import Role, User
from flask_babel import gettext as __, lazy_gettext as _
from holidays import CountryHoliday
from pyparsing import (
CaselessKeyword,
Forward,
Group,
Optional as ppOptional,
ParseException,
ParseResults,
pyparsing_common,
quotedString,
Suppress,
)
from flask_babel import gettext as __
from sqlalchemy import event, exc, select, Text
from sqlalchemy.dialects.mysql import MEDIUMTEXT
from sqlalchemy.engine import Connection, Engine
......@@ -443,58 +426,6 @@ def list_minus(l: List[Any], minus: List[Any]) -> List[Any]:
return [o for o in l if o not in minus]
def parse_human_datetime(human_readable: str) -> datetime:
"""
Returns ``datetime.datetime`` from human readable strings
>>> from datetime import date, timedelta
>>> from dateutil.relativedelta import relativedelta
>>> parse_human_datetime('2015-04-03')
datetime.datetime(2015, 4, 3, 0, 0)
>>> parse_human_datetime('2/3/1969')
datetime.datetime(1969, 2, 3, 0, 0)
>>> parse_human_datetime('now') <= datetime.now()
True
>>> parse_human_datetime('yesterday') <= datetime.now()
True
>>> date.today() - timedelta(1) == parse_human_datetime('yesterday').date()
True
>>> year_ago_1 = parse_human_datetime('one year ago').date()
>>> year_ago_2 = (datetime.now() - relativedelta(years=1)).date()
>>> year_ago_1 == year_ago_2
True
>>> year_after_1 = parse_human_datetime('2 years after').date()
>>> year_after_2 = (datetime.now() + relativedelta(years=2)).date()
>>> year_after_1 == year_after_2
True
"""
try:
dttm = parse(human_readable)
except Exception: # pylint: disable=broad-except
try:
cal = parsedatetime.Calendar()
parsed_dttm, parsed_flags = cal.parseDT(human_readable)
# when time is not extracted, we 'reset to midnight'
if parsed_flags & 2 == 0:
parsed_dttm = parsed_dttm.replace(hour=0, minute=0, second=0)
dttm = dttm_from_timetuple(parsed_dttm.utctimetuple())
except Exception as ex:
logger.exception(ex)
raise ValueError("Couldn't parse date string [{}]".format(human_readable))
return dttm
def dttm_from_timetuple(date_: struct_time) -> datetime:
return datetime(
date_.tm_year,
date_.tm_mon,
date_.tm_mday,
date_.tm_hour,
date_.tm_min,
date_.tm_sec,
)
def md5_hex(data: str) -> str:
return hashlib.md5(data.encode()).hexdigest()
......@@ -516,39 +447,6 @@ class DashboardEncoder(json.JSONEncoder):
return json.JSONEncoder(sort_keys=True).default(o)
def parse_human_timedelta(
human_readable: Optional[str], source_time: Optional[datetime] = None,
) -> timedelta:
"""
Returns ``datetime.timedelta`` from natural language time deltas
>>> parse_human_timedelta('1 day') == timedelta(days=1)
True
"""
cal = parsedatetime.Calendar()
source_dttm = dttm_from_timetuple(
source_time.timetuple() if source_time else datetime.now().timetuple()
)
modified_dttm = dttm_from_timetuple(cal.parse(human_readable or "", source_dttm)[0])
return modified_dttm - source_dttm
def parse_past_timedelta(
delta_str: str, source_time: Optional[datetime] = None
) -> timedelta:
"""
Takes a delta like '1 year' and finds the timedelta for that period in
the past, then represents that past timedelta in positive terms.
parse_human_timedelta('1 year') find the timedelta 1 year in the future.
parse_past_timedelta('1 year') returns -datetime.timedelta(-365)
or datetime.timedelta(365).
"""
return -parse_human_timedelta(
delta_str if delta_str.startswith("-") else f"-{delta_str}", source_time,
)
class JSONEncodedDict(TypeDecorator): # pylint: disable=abstract-method
"""Represents an immutable structure as a json-encoded string."""
......@@ -1254,347 +1152,6 @@ def ensure_path_exists(path: str) -> None:
raise
class EvalText: # pylint: disable=too-few-public-methods
def __init__(self, tokens: ParseResults) -> None:
self.value = tokens[0]
def eval(self) -> str:
# strip quotes
return self.value[1:-1]
class EvalDateTimeFunc: # pylint: disable=too-few-public-methods
def __init__(self, tokens: ParseResults) -> None:
self.value = tokens[1]
def eval(self) -> datetime:
return parse_human_datetime(self.value.eval())
class EvalDateAddFunc: # pylint: disable=too-few-public-methods
def __init__(self, tokens: ParseResults) -> None:
self.value = tokens[1]
def eval(self) -> datetime:
dttm_expression, delta, unit = self.value
dttm = dttm_expression.eval()
if unit.lower() == "quarter":
delta = delta * 3
unit = "month"
return dttm + parse_human_timedelta(f"{delta} {unit}s", dttm)
class EvalDateTruncFunc: # pylint: disable=too-few-public-methods
def __init__(self, tokens: ParseResults) -> None:
self.value = tokens[1]
def eval(self) -> datetime:
dttm_expression, unit = self.value
dttm = dttm_expression.eval()
if unit == "year":
dttm = dttm.replace(
month=1, day=1, hour=0, minute=0, second=0, microsecond=0
)
elif unit == "month":
dttm = dttm.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
elif unit == "week":
dttm = dttm - relativedelta(days=dttm.weekday())
dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0)
elif unit == "day":
dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0)
elif unit == "hour":
dttm = dttm.replace(minute=0, second=0, microsecond=0)
elif unit == "minute":
dttm = dttm.replace(second=0, microsecond=0)
else:
dttm = dttm.replace(microsecond=0)
return dttm
class EvalLastDayFunc: # pylint: disable=too-few-public-methods
def __init__(self, tokens: ParseResults) -> None:
self.value = tokens[1]
def eval(self) -> datetime:
dttm_expression, unit = self.value
dttm = dttm_expression.eval()
if unit == "year":
return dttm.replace(
month=12, day=31, hour=0, minute=0, second=0, microsecond=0
)
if unit == "month":
return dttm.replace(
day=calendar.monthrange(dttm.year, dttm.month)[1],
hour=0,
minute=0,
second=0,
microsecond=0,
)
# unit == "week":
mon = dttm - relativedelta(days=dttm.weekday())
mon = mon.replace(hour=0, minute=0, second=0, microsecond=0)
return mon + relativedelta(days=6)
class EvalHolidayFunc: # pylint: disable=too-few-public-methods
def __init__(self, tokens: ParseResults) -> None:
self.value = tokens[1]
def eval(self) -> datetime:
holiday = self.value[0].eval()
dttm, country = [None, None]
if len(self.value) >= 2:
dttm = self.value[1].eval()
if len(self.value) == 3:
country = self.value[2]
holiday_year = dttm.year if dttm else parse_human_datetime("today").year
country = country.eval() if country else "US"
holiday_lookup = CountryHoliday(country, years=[holiday_year], observed=False)
searched_result = holiday_lookup.get_named(holiday)
if len(searched_result) == 1:
return dttm_from_timetuple(searched_result[0].timetuple())
raise ValueError(_("Unable to find such a holiday: [{}]").format(holiday))
@memoized()
def datetime_parser() -> ParseResults: # pylint: disable=too-many-locals
( # pylint: disable=invalid-name
DATETIME,
DATEADD,
DATETRUNC,
LASTDAY,
HOLIDAY,
YEAR,
QUARTER,
MONTH,
WEEK,
DAY,
HOUR,
MINUTE,
SECOND,
) = map(
CaselessKeyword,
"datetime dateadd datetrunc lastday holiday "
"year quarter month week day hour minute second".split(),
)
lparen, rparen, comma = map(Suppress, "(),")
int_operand = pyparsing_common.signed_integer().setName("int_operand")
text_operand = quotedString.setName("text_operand").setParseAction(EvalText)
# allow expression to be used recursively
datetime_func = Forward().setName("datetime")
dateadd_func = Forward().setName("dateadd")
datetrunc_func = Forward().setName("datetrunc")
lastday_func = Forward().setName("lastday")
holiday_func = Forward().setName("holiday")
date_expr = (
datetime_func | dateadd_func | datetrunc_func | lastday_func | holiday_func
)
datetime_func <<= (DATETIME + lparen + text_operand + rparen).setParseAction(
EvalDateTimeFunc
)
dateadd_func <<= (
DATEADD
+ lparen
+ Group(
date_expr
+ comma
+ int_operand
+ comma
+ (YEAR | QUARTER | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND)
+ ppOptional(comma)
)
+ rparen
).setParseAction(EvalDateAddFunc)
datetrunc_func <<= (
DATETRUNC
+ lparen
+ Group(
date_expr
+ comma
+ (YEAR | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND)
+ ppOptional(comma)
)
+ rparen
).setParseAction(EvalDateTruncFunc)
lastday_func <<= (
LASTDAY
+ lparen
+ Group(date_expr + comma + (YEAR | MONTH | WEEK) + ppOptional(comma))
+ rparen
).setParseAction(EvalLastDayFunc)
holiday_func <<= (
HOLIDAY
+ lparen
+ Group(
text_operand
+ ppOptional(comma)
+ ppOptional(date_expr)
+ ppOptional(comma)
+ ppOptional(text_operand)
+ ppOptional(comma)
)
+ rparen
).setParseAction(EvalHolidayFunc)
return date_expr
def datetime_eval(datetime_expression: Optional[str] = None) -> Optional[datetime]:
if datetime_expression:
try:
return datetime_parser().parseString(datetime_expression)[0].eval()
except ParseException as error:
raise ValueError(error)
return None
# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
def get_since_until(
time_range: Optional[str] = None,
since: Optional[str] = None,
until: Optional[str] = None,
time_shift: Optional[str] = None,
relative_start: Optional[str] = None,
relative_end: Optional[str] = None,
) -> Tuple[Optional[datetime], Optional[datetime]]:
"""Return `since` and `until` date time tuple from string representations of
time_range, since, until and time_shift.
This functiom supports both reading the keys separately (from `since` and
`until`), as well as the new `time_range` key. Valid formats are:
- ISO 8601
- X days/years/hours/day/year/weeks
- X days/years/hours/day/year/weeks ago
- X days/years/hours/day/year/weeks from now
- freeform
Additionally, for `time_range` (these specify both `since` and `until`):
- Last day
- Last week
- Last month
- Last quarter
- Last year
- No filter
- Last X seconds/minutes/hours/days/weeks/months/years
- Next X seconds/minutes/hours/days/weeks/months/years
"""
separator = " : "
_relative_start = relative_start if relative_start else "today"
_relative_end = relative_end if relative_end else "today"
if time_range == "No filter":
return None, None
if time_range and time_range.startswith("Last") and separator not in time_range:
time_range = time_range + separator + _relative_end
if time_range and time_range.startswith("Next") and separator not in time_range:
time_range = _relative_start + separator + time_range
if (
time_range
and time_range.startswith("previous calendar week")
and separator not in time_range
):
time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, WEEK), WEEK) : DATETRUNC(DATETIME('today'), WEEK)" # pylint: disable=line-too-long
if (
time_range
and time_range.startswith("previous calendar month")
and separator not in time_range
):
time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, MONTH), MONTH) : DATETRUNC(DATETIME('today'), MONTH)" # pylint: disable=line-too-long
if (
time_range
and time_range.startswith("previous calendar year")
and separator not in time_range
):
time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, YEAR), YEAR) : DATETRUNC(DATETIME('today'), YEAR)" # pylint: disable=line-too-long
if time_range and separator in time_range:
time_range_lookup = [
(
r"^last\s+(day|week|month|quarter|year)$",
lambda unit: f"DATEADD(DATETIME('{_relative_start}'), -1, {unit})",
),
(
r"^last\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$",
lambda delta, unit: f"DATEADD(DATETIME('{_relative_start}'), -{int(delta)}, {unit})", # pylint: disable=line-too-long
),
(
r"^next\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$",
lambda delta, unit: f"DATEADD(DATETIME('{_relative_end}'), {int(delta)}, {unit})", # pylint: disable=line-too-long
),
(
r"^(DATETIME.*|DATEADD.*|DATETRUNC.*|LASTDAY.*|HOLIDAY.*)$",
lambda text: text,
),
]
since_and_until_partition = [_.strip() for _ in time_range.split(separator, 1)]
since_and_until: List[Optional[str]] = []
for part in since_and_until_partition:
if not part:
# if since or until is "", set as None
since_and_until.append(None)
continue
# Is it possible to match to time_range_lookup
matched = False
for pattern, fn in time_range_lookup:
result = re.search(pattern, part, re.IGNORECASE)
if result:
matched = True
# converted matched time_range to "formal time expressions"
since_and_until.append(fn(*result.groups())) # type: ignore
if not matched:
# default matched case
since_and_until.append(f"DATETIME('{part}')")
_since, _until = map(datetime_eval, since_and_until)
else:
since = since or ""
if since:
since = add_ago_to_since(since)
_since = parse_human_datetime(since) if since else None
_until = (
parse_human_datetime(until)
if until
else parse_human_datetime(_relative_end)
)
if time_shift:
time_delta = parse_past_timedelta(time_shift)
_since = _since if _since is None else (_since - time_delta)
_until = _until if _until is None else (_until - time_delta)
if _since and _until and _since > _until:
raise ValueError(_("From date cannot be larger than to date"))
return _since, _until
def add_ago_to_since(since: str) -> str:
"""
Backwards compatibility hack. Without this slices with since: 7 days will
be treated as 7 days in the future.
:param str since:
:returns: Since with ago added if necessary
:rtype: str
"""
since_words = since.split(" ")
grains = ["days", "years", "hours", "day", "year", "weeks"]
if len(since_words) == 2 and since_words[1] in grains:
since += " ago"
return since
def convert_legacy_filters_into_adhoc( # pylint: disable=invalid-name
form_data: FormData,
) -> None:
......
superset/utils/date_parser.py 0 → 100644
浏览文件 @ 35c15b8b
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import calendar
import logging
import re
from datetime import datetime, timedelta
from time import struct_time
from typing import List, Optional, Tuple
import parsedatetime
from dateutil.parser import parse
from dateutil.relativedelta import relativedelta
from flask_babel import lazy_gettext as _
from holidays import CountryHoliday
from pyparsing import (
CaselessKeyword,
Forward,
Group,
Optional as ppOptional,
ParseException,
ParseResults,
pyparsing_common,
quotedString,
Suppress,
)
from .core import memoized
logger = logging.getLogger(__name__)
def parse_human_datetime(human_readable: str) -> datetime:
"""
Returns ``datetime.datetime`` from human readable strings
>>> from datetime import date, timedelta
>>> from dateutil.relativedelta import relativedelta
>>> parse_human_datetime('2015-04-03')
datetime.datetime(2015, 4, 3, 0, 0)
>>> parse_human_datetime('2/3/1969')
datetime.datetime(1969, 2, 3, 0, 0)
>>> parse_human_datetime('now') <= datetime.now()
True
>>> parse_human_datetime('yesterday') <= datetime.now()
True
>>> date.today() - timedelta(1) == parse_human_datetime('yesterday').date()
True
>>> year_ago_1 = parse_human_datetime('one year ago').date()
>>> year_ago_2 = (datetime.now() - relativedelta(years=1)).date()
>>> year_ago_1 == year_ago_2
True
>>> year_after_1 = parse_human_datetime('2 years after').date()
>>> year_after_2 = (datetime.now() + relativedelta(years=2)).date()
>>> year_after_1 == year_after_2
True
"""
try:
dttm = parse(human_readable)
except Exception: # pylint: disable=broad-except
try:
cal = parsedatetime.Calendar()
parsed_dttm, parsed_flags = cal.parseDT(human_readable)
# when time is not extracted, we 'reset to midnight'
if parsed_flags & 2 == 0:
parsed_dttm = parsed_dttm.replace(hour=0, minute=0, second=0)
dttm = dttm_from_timetuple(parsed_dttm.utctimetuple())
except Exception as ex:
logger.exception(ex)
raise ValueError("Couldn't parse date string [{}]".format(human_readable))
return dttm
def dttm_from_timetuple(date_: struct_time) -> datetime:
return datetime(
date_.tm_year,
date_.tm_mon,
date_.tm_mday,
date_.tm_hour,
date_.tm_min,
date_.tm_sec,
)
def parse_human_timedelta(
human_readable: Optional[str], source_time: Optional[datetime] = None,
) -> timedelta:
"""
Returns ``datetime.timedelta`` from natural language time deltas
>>> parse_human_timedelta('1 day') == timedelta(days=1)
True
"""
cal = parsedatetime.Calendar()
source_dttm = dttm_from_timetuple(
source_time.timetuple() if source_time else datetime.now().timetuple()
)
modified_dttm = dttm_from_timetuple(cal.parse(human_readable or "", source_dttm)[0])
return modified_dttm - source_dttm
def parse_past_timedelta(
delta_str: str, source_time: Optional[datetime] = None
) -> timedelta:
"""
Takes a delta like '1 year' and finds the timedelta for that period in
the past, then represents that past timedelta in positive terms.
parse_human_timedelta('1 year') find the timedelta 1 year in the future.
parse_past_timedelta('1 year') returns -datetime.timedelta(-365)
or datetime.timedelta(365).
"""
return -parse_human_timedelta(
delta_str if delta_str.startswith("-") else f"-{delta_str}", source_time,
)
# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
def get_since_until(
time_range: Optional[str] = None,
since: Optional[str] = None,
until: Optional[str] = None,
time_shift: Optional[str] = None,
relative_start: Optional[str] = None,
relative_end: Optional[str] = None,
) -> Tuple[Optional[datetime], Optional[datetime]]:
"""Return `since` and `until` date time tuple from string representations of
time_range, since, until and time_shift.
This functiom supports both reading the keys separately (from `since` and
`until`), as well as the new `time_range` key. Valid formats are:
- ISO 8601
- X days/years/hours/day/year/weeks
- X days/years/hours/day/year/weeks ago
- X days/years/hours/day/year/weeks from now
- freeform
Additionally, for `time_range` (these specify both `since` and `until`):
- Last day
- Last week
- Last month
- Last quarter
- Last year
- No filter
- Last X seconds/minutes/hours/days/weeks/months/years
- Next X seconds/minutes/hours/days/weeks/months/years
"""
separator = " : "
_relative_start = relative_start if relative_start else "today"
_relative_end = relative_end if relative_end else "today"
if time_range == "No filter":
return None, None
if time_range and time_range.startswith("Last") and separator not in time_range:
time_range = time_range + separator + _relative_end
if time_range and time_range.startswith("Next") and separator not in time_range:
time_range = _relative_start + separator + time_range
if (
time_range
and time_range.startswith("previous calendar week")
and separator not in time_range
):
time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, WEEK), WEEK) : DATETRUNC(DATETIME('today'), WEEK)" # pylint: disable=line-too-long
if (
time_range
and time_range.startswith("previous calendar month")
and separator not in time_range
):
time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, MONTH), MONTH) : DATETRUNC(DATETIME('today'), MONTH)" # pylint: disable=line-too-long
if (
time_range
and time_range.startswith("previous calendar year")
and separator not in time_range
):
time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, YEAR), YEAR) : DATETRUNC(DATETIME('today'), YEAR)" # pylint: disable=line-too-long
if time_range and separator in time_range:
time_range_lookup = [
(
r"^last\s+(day|week|month|quarter|year)$",
lambda unit: f"DATEADD(DATETIME('{_relative_start}'), -1, {unit})",
),
(
r"^last\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$",
lambda delta, unit: f"DATEADD(DATETIME('{_relative_start}'), -{int(delta)}, {unit})", # pylint: disable=line-too-long
),
(
r"^next\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$",
lambda delta, unit: f"DATEADD(DATETIME('{_relative_end}'), {int(delta)}, {unit})", # pylint: disable=line-too-long
),
(
r"^(DATETIME.*|DATEADD.*|DATETRUNC.*|LASTDAY.*|HOLIDAY.*)$",
lambda text: text,
),
]
since_and_until_partition = [_.strip() for _ in time_range.split(separator, 1)]
since_and_until: List[Optional[str]] = []
for part in since_and_until_partition:
if not part:
# if since or until is "", set as None
since_and_until.append(None)
continue
# Is it possible to match to time_range_lookup
matched = False
for pattern, fn in time_range_lookup:
result = re.search(pattern, part, re.IGNORECASE)
if result:
matched = True
# converted matched time_range to "formal time expressions"
since_and_until.append(fn(*result.groups())) # type: ignore
if not matched:
# default matched case
since_and_until.append(f"DATETIME('{part}')")
_since, _until = map(datetime_eval, since_and_until)
else:
since = since or ""
if since:
since = add_ago_to_since(since)
_since = parse_human_datetime(since) if since else None
_until = (
parse_human_datetime(until)
if until
else parse_human_datetime(_relative_end)
)
if time_shift:
time_delta = parse_past_timedelta(time_shift)
_since = _since if _since is None else (_since - time_delta)
_until = _until if _until is None else (_until - time_delta)
if _since and _until and _since > _until:
raise ValueError(_("From date cannot be larger than to date"))
return _since, _until
def add_ago_to_since(since: str) -> str:
"""
Backwards compatibility hack. Without this slices with since: 7 days will
be treated as 7 days in the future.
:param str since:
:returns: Since with ago added if necessary
:rtype: str
"""
since_words = since.split(" ")
grains = ["days", "years", "hours", "day", "year", "weeks"]
if len(since_words) == 2 and since_words[1] in grains:
since += " ago"
return since
class EvalText: # pylint: disable=too-few-public-methods
def __init__(self, tokens: ParseResults) -> None:
self.value = tokens[0]
def eval(self) -> str:
# strip quotes
return self.value[1:-1]
class EvalDateTimeFunc: # pylint: disable=too-few-public-methods
def __init__(self, tokens: ParseResults) -> None:
self.value = tokens[1]
def eval(self) -> datetime:
return parse_human_datetime(self.value.eval())
class EvalDateAddFunc: # pylint: disable=too-few-public-methods
def __init__(self, tokens: ParseResults) -> None:
self.value = tokens[1]
def eval(self) -> datetime:
dttm_expression, delta, unit = self.value
dttm = dttm_expression.eval()
if unit.lower() == "quarter":
delta = delta * 3
unit = "month"
return dttm + parse_human_timedelta(f"{delta} {unit}s", dttm)
class EvalDateTruncFunc: # pylint: disable=too-few-public-methods
def __init__(self, tokens: ParseResults) -> None:
self.value = tokens[1]
def eval(self) -> datetime:
dttm_expression, unit = self.value
dttm = dttm_expression.eval()
if unit == "year":
dttm = dttm.replace(
month=1, day=1, hour=0, minute=0, second=0, microsecond=0
)
elif unit == "month":
dttm = dttm.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
elif unit == "week":
dttm = dttm - relativedelta(days=dttm.weekday())
dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0)
elif unit == "day":
dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0)
elif unit == "hour":
dttm = dttm.replace(minute=0, second=0, microsecond=0)
elif unit == "minute":
dttm = dttm.replace(second=0, microsecond=0)
else:
dttm = dttm.replace(microsecond=0)
return dttm
class EvalLastDayFunc: # pylint: disable=too-few-public-methods
def __init__(self, tokens: ParseResults) -> None:
self.value = tokens[1]
def eval(self) -> datetime:
dttm_expression, unit = self.value
dttm = dttm_expression.eval()
if unit == "year":
return dttm.replace(
month=12, day=31, hour=0, minute=0, second=0, microsecond=0
)
if unit == "month":
return dttm.replace(
day=calendar.monthrange(dttm.year, dttm.month)[1],
hour=0,
minute=0,
second=0,
microsecond=0,
)
# unit == "week":
mon = dttm - relativedelta(days=dttm.weekday())
mon = mon.replace(hour=0, minute=0, second=0, microsecond=0)
return mon + relativedelta(days=6)
class EvalHolidayFunc: # pylint: disable=too-few-public-methods
def __init__(self, tokens: ParseResults) -> None:
self.value = tokens[1]
def eval(self) -> datetime:
holiday = self.value[0].eval()
dttm, country = [None, None]
if len(self.value) >= 2:
dttm = self.value[1].eval()
if len(self.value) == 3:
country = self.value[2]
holiday_year = dttm.year if dttm else parse_human_datetime("today").year
country = country.eval() if country else "US"
holiday_lookup = CountryHoliday(country, years=[holiday_year], observed=False)
searched_result = holiday_lookup.get_named(holiday)
if len(searched_result) == 1:
return dttm_from_timetuple(searched_result[0].timetuple())
raise ValueError(_("Unable to find such a holiday: [{}]").format(holiday))
@memoized()
def datetime_parser() -> ParseResults: # pylint: disable=too-many-locals
( # pylint: disable=invalid-name
DATETIME,
DATEADD,
DATETRUNC,
LASTDAY,
HOLIDAY,
YEAR,
QUARTER,
MONTH,
WEEK,
DAY,
HOUR,
MINUTE,
SECOND,
) = map(
CaselessKeyword,
"datetime dateadd datetrunc lastday holiday "
"year quarter month week day hour minute second".split(),
)
lparen, rparen, comma = map(Suppress, "(),")
int_operand = pyparsing_common.signed_integer().setName("int_operand")
text_operand = quotedString.setName("text_operand").setParseAction(EvalText)
# allow expression to be used recursively
datetime_func = Forward().setName("datetime")
dateadd_func = Forward().setName("dateadd")
datetrunc_func = Forward().setName("datetrunc")
lastday_func = Forward().setName("lastday")
holiday_func = Forward().setName("holiday")
date_expr = (
datetime_func | dateadd_func | datetrunc_func | lastday_func | holiday_func
)
datetime_func <<= (DATETIME + lparen + text_operand + rparen).setParseAction(
EvalDateTimeFunc
)
dateadd_func <<= (
DATEADD
+ lparen
+ Group(
date_expr
+ comma
+ int_operand
+ comma
+ (YEAR | QUARTER | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND)
+ ppOptional(comma)
)
+ rparen
).setParseAction(EvalDateAddFunc)
datetrunc_func <<= (
DATETRUNC
+ lparen
+ Group(
date_expr
+ comma
+ (YEAR | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND)
+ ppOptional(comma)
)
+ rparen
).setParseAction(EvalDateTruncFunc)
lastday_func <<= (
LASTDAY
+ lparen
+ Group(date_expr + comma + (YEAR | MONTH | WEEK) + ppOptional(comma))
+ rparen
).setParseAction(EvalLastDayFunc)
holiday_func <<= (
HOLIDAY
+ lparen
+ Group(
text_operand
+ ppOptional(comma)
+ ppOptional(date_expr)
+ ppOptional(comma)
+ ppOptional(text_operand)
+ ppOptional(comma)
)
+ rparen
).setParseAction(EvalHolidayFunc)
return date_expr
def datetime_eval(datetime_expression: Optional[str] = None) -> Optional[datetime]:
if datetime_expression:
try:
return datetime_parser().parseString(datetime_expression)[0].eval()
except ParseException as error:
raise ValueError(error)
return None
superset/views/api.py
浏览文件 @ 35c15b8b
......@@ -29,7 +29,7 @@ from superset.legacy import update_time_range
from superset.models.slice import Slice
from superset.typing import FlaskResponse
from superset.utils import core as utils
from superset.utils.core import get_since_until
from superset.utils.date_parser import get_since_until
from superset.views.base import api, BaseSupersetView, handle_api_exception
get_time_range_schema = {"type": "string"}
......
superset/viz.py
浏览文件 @ 35c15b8b
......@@ -75,6 +75,7 @@ from superset.utils.core import (
QueryMode,
to_adhoc,
)
from superset.utils.date_parser import get_since_until, parse_past_timedelta
from superset.utils.dates import datetime_to_epoch
from superset.utils.hashing import md5_sha_from_str
......@@ -356,7 +357,7 @@ class BaseViz:
order_desc = form_data.get("order_desc", True)
try:
since, until = utils.get_since_until(
since, until = get_since_until(
relative_start=relative_start,
relative_end=relative_end,
time_range=form_data.get("time_range"),
......@@ -367,7 +368,7 @@ class BaseViz:
raise QueryObjectValidationError(str(ex))
time_shift = form_data.get("time_shift", "")
self.time_shift = utils.parse_past_timedelta(time_shift)
self.time_shift = parse_past_timedelta(time_shift)
from_dttm = None if since is None else (since - self.time_shift)
to_dttm = None if until is None else (until - self.time_shift)
if from_dttm and to_dttm and from_dttm > to_dttm:
......@@ -1004,7 +1005,7 @@ class CalHeatmapViz(BaseViz):
data[metric] = values
try:
start, end = utils.get_since_until(
start, end = get_since_until(
relative_start=relative_start,
relative_end=relative_end,
time_range=form_data.get("time_range"),
......@@ -1318,7 +1319,7 @@ class NVD3TimeSeriesViz(NVD3Viz):
for option in time_compare:
query_object = self.query_obj()
try:
delta = utils.parse_past_timedelta(option)
delta = parse_past_timedelta(option)
except ValueError as ex:
raise QueryObjectValidationError(str(ex))
query_object["inner_from_dttm"] = query_object["from_dttm"]
......
superset/viz_sip38.py
浏览文件 @ 35c15b8b
......@@ -63,6 +63,7 @@ from superset.utils.core import (
merge_extra_filters,
to_adhoc,
)
from superset.utils.date_parser import get_since_until, parse_past_timedelta
import dataclasses # isort:skip
......@@ -359,7 +360,7 @@ class BaseViz:
# default order direction
order_desc = form_data.get("order_desc", True)
since, until = utils.get_since_until(
since, until = get_since_until(
relative_start=relative_start,
relative_end=relative_end,
time_range=form_data.get("time_range"),
......@@ -367,7 +368,7 @@ class BaseViz:
until=form_data.get("until"),
)
time_shift = form_data.get("time_shift", "")
self.time_shift = utils.parse_past_timedelta(time_shift)
self.time_shift = parse_past_timedelta(time_shift)
from_dttm = None if since is None else (since - self.time_shift)
to_dttm = None if until is None else (until - self.time_shift)
if from_dttm and to_dttm and from_dttm > to_dttm:
......@@ -883,7 +884,7 @@ class CalHeatmapViz(BaseViz):
values[str(v / 10 ** 9)] = obj.get(metric)
data[metric] = values
start, end = utils.get_since_until(
start, end = get_since_until(
relative_start=relative_start,
relative_end=relative_end,
time_range=form_data.get("time_range"),
......@@ -1265,7 +1266,7 @@ class NVD3TimeSeriesViz(NVD3Viz):
for option in time_compare:
query_object = self.query_obj()
delta = utils.parse_past_timedelta(option)
delta = parse_past_timedelta(option)
query_object["inner_from_dttm"] = query_object["from_dttm"]
query_object["inner_to_dttm"] = query_object["to_dttm"]
......
tests/utils/date_parser_tests.py 0 → 100644
浏览文件 @ 35c15b8b
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from datetime import datetime, timedelta
from unittest.mock import patch
from superset.utils.date_parser import (
datetime_eval,
get_since_until,
parse_human_timedelta,
parse_past_timedelta,
)
from tests.base_tests import SupersetTestCase
def mock_parse_human_datetime(s):
if s == "now":
return datetime(2016, 11, 7, 9, 30, 10)
elif s == "today":
return datetime(2016, 11, 7)
elif s == "yesterday":
return datetime(2016, 11, 6)
elif s == "tomorrow":
return datetime(2016, 11, 8)
elif s == "Last year":
return datetime(2015, 11, 7)
elif s == "Last week":
return datetime(2015, 10, 31)
elif s == "Last 5 months":
return datetime(2016, 6, 7)
elif s == "Next 5 months":
return datetime(2017, 4, 7)
elif s in ["5 days", "5 days ago"]:
return datetime(2016, 11, 2)
elif s == "2018-01-01T00:00:00":
return datetime(2018, 1, 1)
elif s == "2018-12-31T23:59:59":
return datetime(2018, 12, 31, 23, 59, 59)
class TestDateParser(SupersetTestCase):
@patch("superset.utils.date_parser.parse_human_datetime", mock_parse_human_datetime)
def test_get_since_until(self):
result = get_since_until()
expected = None, datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until(" : now")
expected = None, datetime(2016, 11, 7, 9, 30, 10)
self.assertEqual(result, expected)
result = get_since_until("yesterday : tomorrow")
expected = datetime(2016, 11, 6), datetime(2016, 11, 8)
self.assertEqual(result, expected)
result = get_since_until("2018-01-01T00:00:00 : 2018-12-31T23:59:59")
expected = datetime(2018, 1, 1), datetime(2018, 12, 31, 23, 59, 59)
self.assertEqual(result, expected)
result = get_since_until("Last year")
expected = datetime(2015, 11, 7), datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until("Last quarter")
expected = datetime(2016, 8, 7), datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until("Last 5 months")
expected = datetime(2016, 6, 7), datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until("Next 5 months")
expected = datetime(2016, 11, 7), datetime(2017, 4, 7)
self.assertEqual(result, expected)
result = get_since_until(since="5 days")
expected = datetime(2016, 11, 2), datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until(since="5 days ago", until="tomorrow")
expected = datetime(2016, 11, 2), datetime(2016, 11, 8)
self.assertEqual(result, expected)
result = get_since_until(time_range="yesterday : tomorrow", time_shift="1 day")
expected = datetime(2016, 11, 5), datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until(time_range="5 days : now")
expected = datetime(2016, 11, 2), datetime(2016, 11, 7, 9, 30, 10)
self.assertEqual(result, expected)
result = get_since_until("Last week", relative_end="now")
expected = datetime(2016, 10, 31), datetime(2016, 11, 7, 9, 30, 10)
self.assertEqual(result, expected)
result = get_since_until("Last week", relative_start="now")
expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until("Last week", relative_start="now", relative_end="now")
expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7, 9, 30, 10)
self.assertEqual(result, expected)
result = get_since_until("previous calendar week")
expected = datetime(2016, 10, 31, 0, 0, 0), datetime(2016, 11, 7, 0, 0, 0)
self.assertEqual(result, expected)
result = get_since_until("previous calendar month")
expected = datetime(2016, 10, 1, 0, 0, 0), datetime(2016, 11, 1, 0, 0, 0)
self.assertEqual(result, expected)
result = get_since_until("previous calendar year")
expected = datetime(2015, 1, 1, 0, 0, 0), datetime(2016, 1, 1, 0, 0, 0)
self.assertEqual(result, expected)
with self.assertRaises(ValueError):
get_since_until(time_range="tomorrow : yesterday")
@patch("superset.utils.date_parser.parse_human_datetime", mock_parse_human_datetime)
def test_datetime_eval(self):
result = datetime_eval("datetime('now')")
expected = datetime(2016, 11, 7, 9, 30, 10)
self.assertEqual(result, expected)
result = datetime_eval("datetime('today' )")
expected = datetime(2016, 11, 7)
self.assertEqual(result, expected)
# Parse compact arguments spelling
result = datetime_eval("dateadd(datetime('today'),1,year,)")
expected = datetime(2017, 11, 7)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('today'), -2, year)")
expected = datetime(2014, 11, 7)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('today'), 2, quarter)")
expected = datetime(2017, 5, 7)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('today'), 3, month)")
expected = datetime(2017, 2, 7)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('today'), -3, week)")
expected = datetime(2016, 10, 17)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('today'), 3, day)")
expected = datetime(2016, 11, 10)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('now'), 3, hour)")
expected = datetime(2016, 11, 7, 12, 30, 10)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('now'), 40, minute)")
expected = datetime(2016, 11, 7, 10, 10, 10)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('now'), -11, second)")
expected = datetime(2016, 11, 7, 9, 29, 59)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), year)")
expected = datetime(2016, 1, 1, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), month)")
expected = datetime(2016, 11, 1, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), day)")
expected = datetime(2016, 11, 7, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), week)")
expected = datetime(2016, 11, 7, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), hour)")
expected = datetime(2016, 11, 7, 9, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), minute)")
expected = datetime(2016, 11, 7, 9, 30, 0)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), second)")
expected = datetime(2016, 11, 7, 9, 30, 10)
self.assertEqual(result, expected)
result = datetime_eval("lastday(datetime('now'), year)")
expected = datetime(2016, 12, 31, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("lastday(datetime('today'), month)")
expected = datetime(2016, 11, 30, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("holiday('Christmas')")
expected = datetime(2016, 12, 25, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("holiday('Labor day', datetime('2018-01-01T00:00:00'))")
expected = datetime(2018, 9, 3, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval(
"holiday('Boxing day', datetime('2018-01-01T00:00:00'), 'UK')"
)
expected = datetime(2018, 12, 26, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval(
"lastday(dateadd(datetime('2018-01-01T00:00:00'), 1, month), month)"
)
expected = datetime(2018, 2, 28, 0, 0, 0)
self.assertEqual(result, expected)
@patch("superset.utils.date_parser.datetime")
def test_parse_human_timedelta(self, mock_datetime):
mock_datetime.now.return_value = datetime(2019, 4, 1)
mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
self.assertEqual(parse_human_timedelta("now"), timedelta(0))
self.assertEqual(parse_human_timedelta("1 year"), timedelta(366))
self.assertEqual(parse_human_timedelta("-1 year"), timedelta(-365))
self.assertEqual(parse_human_timedelta(None), timedelta(0))
self.assertEqual(
parse_human_timedelta("1 month", datetime(2019, 4, 1)), timedelta(30),
)
self.assertEqual(
parse_human_timedelta("1 month", datetime(2019, 5, 1)), timedelta(31),
)
self.assertEqual(
parse_human_timedelta("1 month", datetime(2019, 2, 1)), timedelta(28),
)
self.assertEqual(
parse_human_timedelta("-1 month", datetime(2019, 2, 1)), timedelta(-31),
)
@patch("superset.utils.date_parser.datetime")
def test_parse_past_timedelta(self, mock_datetime):
mock_datetime.now.return_value = datetime(2019, 4, 1)
mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
self.assertEqual(parse_past_timedelta("1 year"), timedelta(365))
self.assertEqual(parse_past_timedelta("-1 year"), timedelta(365))
self.assertEqual(parse_past_timedelta("52 weeks"), timedelta(364))
self.assertEqual(parse_past_timedelta("1 month"), timedelta(31))
tests/utils_tests.py
浏览文件 @ 35c15b8b
......@@ -46,7 +46,6 @@ from superset.utils.core import (
get_iterable,
get_email_address_list,
get_or_create_db,
get_since_until,
get_stacktrace,
json_int_dttm_ser,
json_iso_dttm_ser,
......@@ -55,15 +54,12 @@ from superset.utils.core import (
merge_extra_filters,
merge_request_params,
parse_ssl_cert,
parse_human_timedelta,
parse_js_uri_path_item,
parse_past_timedelta,
split,
TimeRangeEndpoint,
validate_json,
zlib_compress,
zlib_decompress,
datetime_eval,
)
from superset.utils import schema
from superset.views.utils import (
......@@ -76,31 +72,6 @@ from tests.base_tests import SupersetTestCase
from .fixtures.certificates import ssl_certificate
def mock_parse_human_datetime(s):
if s == "now":
return datetime(2016, 11, 7, 9, 30, 10)
elif s == "today":
return datetime(2016, 11, 7)
elif s == "yesterday":
return datetime(2016, 11, 6)
elif s == "tomorrow":
return datetime(2016, 11, 8)
elif s == "Last year":
return datetime(2015, 11, 7)
elif s == "Last week":
return datetime(2015, 10, 31)
elif s == "Last 5 months":
return datetime(2016, 6, 7)
elif s == "Next 5 months":
return datetime(2017, 4, 7)
elif s in ["5 days", "5 days ago"]:
return datetime(2016, 11, 2)
elif s == "2018-01-01T00:00:00":
return datetime(2018, 1, 1)
elif s == "2018-12-31T23:59:59":
return datetime(2018, 12, 31, 23, 59, 59)
def mock_to_adhoc(filt, expressionType="SIMPLE", clause="where"):
result = {"clause": clause.upper(), "expressionType": expressionType}
......@@ -147,36 +118,6 @@ class TestUtils(SupersetTestCase):
assert isinstance(base_json_conv(uuid.uuid4()), str) is True
assert isinstance(base_json_conv(timedelta(0)), str) is True
@patch("superset.utils.core.datetime")
def test_parse_human_timedelta(self, mock_datetime):
mock_datetime.now.return_value = datetime(2019, 4, 1)
mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
self.assertEqual(parse_human_timedelta("now"), timedelta(0))
self.assertEqual(parse_human_timedelta("1 year"), timedelta(366))
self.assertEqual(parse_human_timedelta("-1 year"), timedelta(-365))
self.assertEqual(parse_human_timedelta(None), timedelta(0))
self.assertEqual(
parse_human_timedelta("1 month", datetime(2019, 4, 1)), timedelta(30),
)
self.assertEqual(
parse_human_timedelta("1 month", datetime(2019, 5, 1)), timedelta(31),
)
self.assertEqual(
parse_human_timedelta("1 month", datetime(2019, 2, 1)), timedelta(28),
)
self.assertEqual(
parse_human_timedelta("-1 month", datetime(2019, 2, 1)), timedelta(-31),
)
@patch("superset.utils.core.datetime")
def test_parse_past_timedelta(self, mock_datetime):
mock_datetime.now.return_value = datetime(2019, 4, 1)
mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
self.assertEqual(parse_past_timedelta("1 year"), timedelta(365))
self.assertEqual(parse_past_timedelta("-1 year"), timedelta(365))
self.assertEqual(parse_past_timedelta("52 weeks"), timedelta(364))
self.assertEqual(parse_past_timedelta("1 month"), timedelta(31))
def test_zlib_compression(self):
json_str = '{"test": 1}'
blob = zlib_compress(json_str)
......@@ -699,186 +640,6 @@ class TestUtils(SupersetTestCase):
self.assertEqual(instance.watcher, 4)
self.assertEqual(result1, result8)
@patch("superset.utils.core.parse_human_datetime", mock_parse_human_datetime)
def test_get_since_until(self):
result = get_since_until()
expected = None, datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until(" : now")
expected = None, datetime(2016, 11, 7, 9, 30, 10)
self.assertEqual(result, expected)
result = get_since_until("yesterday : tomorrow")
expected = datetime(2016, 11, 6), datetime(2016, 11, 8)
self.assertEqual(result, expected)
result = get_since_until("2018-01-01T00:00:00 : 2018-12-31T23:59:59")
expected = datetime(2018, 1, 1), datetime(2018, 12, 31, 23, 59, 59)
self.assertEqual(result, expected)
result = get_since_until("Last year")
expected = datetime(2015, 11, 7), datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until("Last quarter")
expected = datetime(2016, 8, 7), datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until("Last 5 months")
expected = datetime(2016, 6, 7), datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until("Next 5 months")
expected = datetime(2016, 11, 7), datetime(2017, 4, 7)
self.assertEqual(result, expected)
result = get_since_until(since="5 days")
expected = datetime(2016, 11, 2), datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until(since="5 days ago", until="tomorrow")
expected = datetime(2016, 11, 2), datetime(2016, 11, 8)
self.assertEqual(result, expected)
result = get_since_until(time_range="yesterday : tomorrow", time_shift="1 day")
expected = datetime(2016, 11, 5), datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until(time_range="5 days : now")
expected = datetime(2016, 11, 2), datetime(2016, 11, 7, 9, 30, 10)
self.assertEqual(result, expected)
result = get_since_until("Last week", relative_end="now")
expected = datetime(2016, 10, 31), datetime(2016, 11, 7, 9, 30, 10)
self.assertEqual(result, expected)
result = get_since_until("Last week", relative_start="now")
expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7)
self.assertEqual(result, expected)
result = get_since_until("Last week", relative_start="now", relative_end="now")
expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7, 9, 30, 10)
self.assertEqual(result, expected)
result = get_since_until("previous calendar week")
expected = datetime(2016, 10, 31, 0, 0, 0), datetime(2016, 11, 7, 0, 0, 0)
self.assertEqual(result, expected)
result = get_since_until("previous calendar month")
expected = datetime(2016, 10, 1, 0, 0, 0), datetime(2016, 11, 1, 0, 0, 0)
self.assertEqual(result, expected)
result = get_since_until("previous calendar year")
expected = datetime(2015, 1, 1, 0, 0, 0), datetime(2016, 1, 1, 0, 0, 0)
self.assertEqual(result, expected)
with self.assertRaises(ValueError):
get_since_until(time_range="tomorrow : yesterday")
@patch("superset.utils.core.parse_human_datetime", mock_parse_human_datetime)
def test_datetime_eval(self):
result = datetime_eval("datetime('now')")
expected = datetime(2016, 11, 7, 9, 30, 10)
self.assertEqual(result, expected)
result = datetime_eval("datetime('today' )")
expected = datetime(2016, 11, 7)
self.assertEqual(result, expected)
# Parse compact arguments spelling
result = datetime_eval("dateadd(datetime('today'),1,year,)")
expected = datetime(2017, 11, 7)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('today'), -2, year)")
expected = datetime(2014, 11, 7)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('today'), 2, quarter)")
expected = datetime(2017, 5, 7)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('today'), 3, month)")
expected = datetime(2017, 2, 7)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('today'), -3, week)")
expected = datetime(2016, 10, 17)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('today'), 3, day)")
expected = datetime(2016, 11, 10)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('now'), 3, hour)")
expected = datetime(2016, 11, 7, 12, 30, 10)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('now'), 40, minute)")
expected = datetime(2016, 11, 7, 10, 10, 10)
self.assertEqual(result, expected)
result = datetime_eval("dateadd(datetime('now'), -11, second)")
expected = datetime(2016, 11, 7, 9, 29, 59)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), year)")
expected = datetime(2016, 1, 1, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), month)")
expected = datetime(2016, 11, 1, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), day)")
expected = datetime(2016, 11, 7, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), week)")
expected = datetime(2016, 11, 7, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), hour)")
expected = datetime(2016, 11, 7, 9, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), minute)")
expected = datetime(2016, 11, 7, 9, 30, 0)
self.assertEqual(result, expected)
result = datetime_eval("datetrunc(datetime('now'), second)")
expected = datetime(2016, 11, 7, 9, 30, 10)
self.assertEqual(result, expected)
result = datetime_eval("lastday(datetime('now'), year)")
expected = datetime(2016, 12, 31, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("lastday(datetime('today'), month)")
expected = datetime(2016, 11, 30, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("holiday('Christmas')")
expected = datetime(2016, 12, 25, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval("holiday('Labor day', datetime('2018-01-01T00:00:00'))")
expected = datetime(2018, 9, 3, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval(
"holiday('Boxing day', datetime('2018-01-01T00:00:00'), 'UK')"
)
expected = datetime(2018, 12, 26, 0, 0, 0)
self.assertEqual(result, expected)
result = datetime_eval(
"lastday(dateadd(datetime('2018-01-01T00:00:00'), 1, month), month)"
)
expected = datetime(2018, 2, 28, 0, 0, 0)
self.assertEqual(result, expected)
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
def test_convert_legacy_filters_into_adhoc_where(self):
form_data = {"where": "a = 1"}
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或