Fix out-of-range Python timestamps with pandas 3.1

iotdb-client/client-py/iotdb/utils/rpc_utils.py

@@ -17,6 +17,7 @@
 #
 import logging
 
+import numpy as np
 import pandas as pd
 from pandas._libs import OutOfBoundsDatetime
 from tzlocal import get_localzone_name
@@ -81,10 +82,16 @@ def convert_to_timestamp(time: int, precision: str, timezone: str):
     try:
         ts = pd.Timestamp(time, unit=precision, tz=timezone)
     except OutOfBoundsDatetime:
+        # A datetime64 scalar keeps the input unit; pandas 3.1 otherwise rescales
+        # numeric input to microseconds, which can overflow millisecond values.
         try:
-            ts = pd.Timestamp(time, unit=precision).tz_localize(timezone)
-        except NotImplementedError:
-            return pd.Timestamp(time, unit=precision)
+            ts = (
+                pd.Timestamp(np.datetime64(int(time), precision))
+                .tz_localize("UTC")
+                .tz_convert(timezone)
+            )
+        except (OutOfBoundsDatetime, NotImplementedError):
+            return pd.Timestamp(np.datetime64(int(time), precision))
     except ValueError:
         logger.warning(
             f"Timezone string '{timezone}' cannot be recognized by pandas. "
@@ -95,7 +102,7 @@ def convert_to_timestamp(time: int, precision: str, timezone: str):
         # Timestamp falls outside Python's stdlib datetime range (year < 1 or
         # > 9999). pandas >= 3.0 cannot attach a timezone to such Timestamps
         # because tz conversion relies on toordinal(). Return naive instead.
-        return pd.Timestamp(time, unit=precision)
+        return pd.Timestamp(np.datetime64(int(time), precision))
     # Construction succeeded but utcoffset() may still fail downstream (e.g.
     # in pd.DataFrame's tz alignment) when the Timestamp's year is outside
     # 1..9999 and tz is not the canonical UTC. Probe early and fall back to
@@ -105,7 +112,7 @@ def convert_to_timestamp(time: int, precision: str, timezone: str):
         try:
             ts.utcoffset()
         except NotImplementedError:
-            return pd.Timestamp(time, unit=precision)
+            return pd.Timestamp(np.datetime64(int(time), precision))
     return ts
 
 

iotdb-client/client-py/tests/unit/test_rpc_utils.py

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from iotdb.utils.rpc_utils import convert_to_timestamp
+
+
+@pytest.mark.parametrize("value", [-(2**63) + 1, 2**63 - 1])
+@pytest.mark.parametrize("timezone", ["UTC", "Asia/Shanghai"])
+def test_convert_timestamp_preserves_extreme_milliseconds(value, timezone):
+    timestamp = convert_to_timestamp(value, "ms", timezone)
+    assert isinstance(timestamp, pd.Timestamp)
+    assert not pd.isna(timestamp)
+    assert timestamp.asm8.astype("datetime64[ms]").astype(np.int64) == value