*: Adapt to deprecation of datetime utc methods

Python 3.12 deprecates the utcnow and utcfromtimestamp methods and
discourages the use of naive datetimes to represent UTC. This was
previously the main way that Tornado used datetimes (since it was
the only option available in Python 2 before the introduction
of datetime.timezone.utc in Python 3.2).

- httpclient_test: Test-only change to test that both kinds of datetimes
  are supported in If-Modified-Since (this just calls
  httputil.format_timestamp)
- httputil: No functional changes, but format_timestamp's
  support for both naive and aware datetimes is now tested.
- locale: format_timestamp now supports aware datetimes (in
  addition to the existing support for naive datetimes).
- web: Cookie expirations internally use aware datetimes.
  StaticFileHandler.get_modified_time now supports both and the
  standard implementation returns aware.

It feels fragile that "naive" and "aware" datetimes are not distinct
types but subject to data-dependent behavior. This change uses
"aware" datetimes throughout Tornado, but some operations (comparisons
and subtraction) fail with mixed datetime types and if I missed any
in this change may cause errors if naive datetimes were used (where
previously naive datetimes would have been required). But that's
apparently the API we have to work with.

tornado/httputil.py

@@ -856,7 +856,8 @@ def format_timestamp(
 
     The argument may be a numeric timestamp as returned by `time.time`,
     a time tuple as returned by `time.gmtime`, or a `datetime.datetime`
-    object.
+    object. Naive `datetime.datetime` objects are assumed to represent
+    UTC; aware objects are converted to UTC before formatting.
 
     >>> format_timestamp(1359312200)
     'Sun, 27 Jan 2013 18:43:20 GMT'

tornado/locale.py

@@ -333,7 +333,7 @@ class Locale(object):
         shorter: bool = False,
         full_format: bool = False,
     ) -> str:
-        """Formats the given date (which should be GMT).
+        """Formats the given date.
 
         By default, we return a relative time (e.g., "2 minutes ago"). You
         can return an absolute date string with ``relative=False``.
@@ -343,10 +343,16 @@ class Locale(object):
 
         This method is primarily intended for dates in the past.
         For dates in the future, we fall back to full format.
+
+        .. versionchanged:: 6.4
+           Aware `datetime.datetime` objects are now supported (naive
+           datetimes are still assumed to be UTC).
         """
         if isinstance(date, (int, float)):
-            date = datetime.datetime.utcfromtimestamp(date)
-        now = datetime.datetime.utcnow()
+            date = datetime.datetime.fromtimestamp(date, datetime.timezone.utc)
+        if date.tzinfo is None:
+            date = date.replace(tzinfo=datetime.timezone.utc)
+        now = datetime.datetime.now(datetime.timezone.utc)
         if date > now:
             if relative and (date - now).seconds < 60:
                 # Due to click skew, things are some things slightly

tornado/test/httpclient_test.py

@@ -28,7 +28,7 @@ from tornado.iostream import IOStream
 from tornado.log import gen_log, app_log
 from tornado import netutil
 from tornado.testing import AsyncHTTPTestCase, bind_unused_port, gen_test, ExpectLog
-from tornado.test.util import skipOnTravis
+from tornado.test.util import skipOnTravis, ignore_deprecation
 from tornado.web import Application, RequestHandler, url
 from tornado.httputil import format_timestamp, HTTPHeaders
 
@@ -887,7 +887,15 @@ class HTTPRequestTestCase(unittest.TestCase):
         self.assertEqual(request.body, utf8("foo"))
 
     def test_if_modified_since(self):
-        http_date = datetime.datetime.utcnow()
+        http_date = datetime.datetime.now(datetime.timezone.utc)
+        request = HTTPRequest("http://example.com", if_modified_since=http_date)
+        self.assertEqual(
+            request.headers, {"If-Modified-Since": format_timestamp(http_date)}
+        )
+
+    def test_if_modified_since_naive_deprecated(self):
+        with ignore_deprecation():
+            http_date = datetime.datetime.utcnow()
         request = HTTPRequest("http://example.com", if_modified_since=http_date)
         self.assertEqual(
             request.headers, {"If-Modified-Since": format_timestamp(http_date)}

tornado/test/httputil_test.py

@@ -13,6 +13,7 @@ from tornado.httputil import (
 from tornado.escape import utf8, native_str
 from tornado.log import gen_log
 from tornado.testing import ExpectLog
+from tornado.test.util import ignore_deprecation
 
 import copy
 import datetime
@@ -412,8 +413,29 @@ class FormatTimestampTest(unittest.TestCase):
         self.assertEqual(9, len(tup))
         self.check(tup)
 
-    def test_datetime(self):
-        self.check(datetime.datetime.utcfromtimestamp(self.TIMESTAMP))
+    def test_utc_naive_datetime(self):
+        self.check(
+            datetime.datetime.fromtimestamp(
+                self.TIMESTAMP, datetime.timezone.utc
+            ).replace(tzinfo=None)
+        )
+
+    def test_utc_naive_datetime_deprecated(self):
+        with ignore_deprecation():
+            self.check(datetime.datetime.utcfromtimestamp(self.TIMESTAMP))
+
+    def test_utc_aware_datetime(self):
+        self.check(
+            datetime.datetime.fromtimestamp(self.TIMESTAMP, datetime.timezone.utc)
+        )
+
+    def test_other_aware_datetime(self):
+        # Other timezones are ignored; the timezone is always printed as GMT
+        self.check(
+            datetime.datetime.fromtimestamp(
+                self.TIMESTAMP, datetime.timezone(datetime.timedelta(hours=-4))
+            )
+        )
 
 
 # HTTPServerRequest is mainly tested incidentally to the server itself,

tornado/test/locale_test.py

@@ -91,45 +91,55 @@ class EnglishTest(unittest.TestCase):
             locale.format_date(date, full_format=True), "April 28, 2013 at 6:35 pm"
         )
 
-        now = datetime.datetime.utcnow()
-
-        self.assertEqual(
-            locale.format_date(now - datetime.timedelta(seconds=2), full_format=False),
-            "2 seconds ago",
-        )
-        self.assertEqual(
-            locale.format_date(now - datetime.timedelta(minutes=2), full_format=False),
-            "2 minutes ago",
-        )
-        self.assertEqual(
-            locale.format_date(now - datetime.timedelta(hours=2), full_format=False),
-            "2 hours ago",
-        )
-
-        self.assertEqual(
-            locale.format_date(
-                now - datetime.timedelta(days=1), full_format=False, shorter=True
-            ),
-            "yesterday",
-        )
-
-        date = now - datetime.timedelta(days=2)
-        self.assertEqual(
-            locale.format_date(date, full_format=False, shorter=True),
-            locale._weekdays[date.weekday()],
-        )
-
-        date = now - datetime.timedelta(days=300)
-        self.assertEqual(
-            locale.format_date(date, full_format=False, shorter=True),
-            "%s %d" % (locale._months[date.month - 1], date.day),
-        )
-
-        date = now - datetime.timedelta(days=500)
-        self.assertEqual(
-            locale.format_date(date, full_format=False, shorter=True),
-            "%s %d, %d" % (locale._months[date.month - 1], date.day, date.year),
-        )
+        aware_dt = datetime.datetime.now(datetime.timezone.utc)
+        naive_dt = aware_dt.replace(tzinfo=None)
+        for name, now in {"aware": aware_dt, "naive": naive_dt}.items():
+            with self.subTest(dt=name):
+                self.assertEqual(
+                    locale.format_date(
+                        now - datetime.timedelta(seconds=2), full_format=False
+                    ),
+                    "2 seconds ago",
+                )
+                self.assertEqual(
+                    locale.format_date(
+                        now - datetime.timedelta(minutes=2), full_format=False
+                    ),
+                    "2 minutes ago",
+                )
+                self.assertEqual(
+                    locale.format_date(
+                        now - datetime.timedelta(hours=2), full_format=False
+                    ),
+                    "2 hours ago",
+                )
+
+                self.assertEqual(
+                    locale.format_date(
+                        now - datetime.timedelta(days=1),
+                        full_format=False,
+                        shorter=True,
+                    ),
+                    "yesterday",
+                )
+
+                date = now - datetime.timedelta(days=2)
+                self.assertEqual(
+                    locale.format_date(date, full_format=False, shorter=True),
+                    locale._weekdays[date.weekday()],
+                )
+
+                date = now - datetime.timedelta(days=300)
+                self.assertEqual(
+                    locale.format_date(date, full_format=False, shorter=True),
+                    "%s %d" % (locale._months[date.month - 1], date.day),
+                )
+
+                date = now - datetime.timedelta(days=500)
+                self.assertEqual(
+                    locale.format_date(date, full_format=False, shorter=True),
+                    "%s %d, %d" % (locale._months[date.month - 1], date.day, date.year),
+                )
 
     def test_friendly_number(self):
         locale = tornado.locale.get("en_US")

tornado/test/web_test.py

@@ -404,10 +404,10 @@ class CookieTest(WebTestCase):
         match = re.match("foo=bar; expires=(?P<expires>.+); Path=/", header)
         assert match is not None
 
-        expires = datetime.datetime.utcnow() + datetime.timedelta(days=10)
-        parsed = email.utils.parsedate(match.groupdict()["expires"])
-        assert parsed is not None
-        header_expires = datetime.datetime(*parsed[:6])
+        expires = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
+            days=10
+        )
+        header_expires = email.utils.parsedate_to_datetime(match.groupdict()["expires"])
         self.assertTrue(abs((expires - header_expires).total_seconds()) < 10)
 
     def test_set_cookie_false_flags(self):
@@ -1697,11 +1697,10 @@ class DateHeaderTest(SimpleHandlerTestCase):
 
     def test_date_header(self):
         response = self.fetch("/")
-        parsed = email.utils.parsedate(response.headers["Date"])
-        assert parsed is not None
-        header_date = datetime.datetime(*parsed[:6])
+        header_date = email.utils.parsedate_to_datetime(response.headers["Date"])
         self.assertTrue(
-            header_date - datetime.datetime.utcnow() < datetime.timedelta(seconds=2)
+            header_date - datetime.datetime.now(datetime.timezone.utc)
+            < datetime.timedelta(seconds=2)
         )
 
 
@@ -3010,10 +3009,12 @@ class XSRFCookieKwargsTest(SimpleHandlerTestCase):
         match = re.match(".*; expires=(?P<expires>.+);.*", header)
         assert match is not None
 
-        expires = datetime.datetime.utcnow() + datetime.timedelta(days=2)
-        parsed = email.utils.parsedate(match.groupdict()["expires"])
-        assert parsed is not None
-        header_expires = datetime.datetime(*parsed[:6])
+        expires = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
+            days=2
+        )
+        header_expires = email.utils.parsedate_to_datetime(match.groupdict()["expires"])
+        if header_expires.tzinfo is None:
+            header_expires = header_expires.replace(tzinfo=datetime.timezone.utc)
         self.assertTrue(abs((expires - header_expires).total_seconds()) < 10)
 
 

tornado/web.py

@@ -647,7 +647,9 @@ class RequestHandler(object):
         if domain:
             morsel["domain"] = domain
         if expires_days is not None and not expires:
-            expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days)
+            expires = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
+                days=expires_days
+            )
         if expires:
             morsel["expires"] = httputil.format_timestamp(expires)
         if path:
@@ -698,7 +700,9 @@ class RequestHandler(object):
                 raise TypeError(
                     f"clear_cookie() got an unexpected keyword argument '{excluded_arg}'"
                 )
-        expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
+        expires = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(
+            days=365
+        )
         self.set_cookie(name, value="", expires=expires, **kwargs)
 
     def clear_all_cookies(self, **kwargs: Any) -> None:
@@ -2812,12 +2816,12 @@ class StaticFileHandler(RequestHandler):
         # content has not been modified
         ims_value = self.request.headers.get("If-Modified-Since")
         if ims_value is not None:
-            date_tuple = email.utils.parsedate(ims_value)
-            if date_tuple is not None:
-                if_since = datetime.datetime(*date_tuple[:6])
-                assert self.modified is not None
-                if if_since >= self.modified:
-                    return True
+            if_since = email.utils.parsedate_to_datetime(ims_value)
+            if if_since.tzinfo is None:
+                if_since = if_since.replace(tzinfo=datetime.timezone.utc)
+            assert self.modified is not None
+            if if_since >= self.modified:
+                return True
 
         return False
 
@@ -2981,6 +2985,10 @@ class StaticFileHandler(RequestHandler):
         object or None.
 
         .. versionadded:: 3.1
+
+        .. versionchanged:: 6.4
+           Now returns an aware datetime object instead of a naive one.
+           Subclasses that override this method may return either kind.
         """
         stat_result = self._stat()
         # NOTE: Historically, this used stat_result[stat.ST_MTIME],
@@ -2991,7 +2999,9 @@ class StaticFileHandler(RequestHandler):
         # consistency with the past (and because we have a unit test
         # that relies on this), we truncate the float here, although
         # I'm not sure that's the right thing to do.
-        modified = datetime.datetime.utcfromtimestamp(int(stat_result.st_mtime))
+        modified = datetime.datetime.fromtimestamp(
+            int(stat_result.st_mtime), datetime.timezone.utc
+        )
         return modified
 
     def get_content_type(self) -> str: