diff --git a/iotdb-client/client-py/iotdb/utils/rpc_utils.py b/iotdb-client/client-py/iotdb/utils/rpc_utils.py index 0912fd14d95de..e6aec4080c0cd 100644 --- a/iotdb-client/client-py/iotdb/utils/rpc_utils.py +++ b/iotdb-client/client-py/iotdb/utils/rpc_utils.py @@ -17,6 +17,7 @@ # import logging +import numpy as np import pandas as pd from pandas._libs import OutOfBoundsDatetime from tzlocal import get_localzone_name @@ -81,10 +82,16 @@ def convert_to_timestamp(time: int, precision: str, timezone: str): try: ts = pd.Timestamp(time, unit=precision, tz=timezone) except OutOfBoundsDatetime: + # A datetime64 scalar keeps the input unit; pandas 3.1 otherwise rescales + # numeric input to microseconds, which can overflow millisecond values. try: - ts = pd.Timestamp(time, unit=precision).tz_localize(timezone) - except NotImplementedError: - return pd.Timestamp(time, unit=precision) + ts = ( + pd.Timestamp(np.datetime64(int(time), precision)) + .tz_localize("UTC") + .tz_convert(timezone) + ) + except (OutOfBoundsDatetime, NotImplementedError): + return pd.Timestamp(np.datetime64(int(time), precision)) except ValueError: logger.warning( f"Timezone string '{timezone}' cannot be recognized by pandas. " @@ -95,7 +102,7 @@ def convert_to_timestamp(time: int, precision: str, timezone: str): # Timestamp falls outside Python's stdlib datetime range (year < 1 or # > 9999). pandas >= 3.0 cannot attach a timezone to such Timestamps # because tz conversion relies on toordinal(). Return naive instead. - return pd.Timestamp(time, unit=precision) + return pd.Timestamp(np.datetime64(int(time), precision)) # Construction succeeded but utcoffset() may still fail downstream (e.g. # in pd.DataFrame's tz alignment) when the Timestamp's year is outside # 1..9999 and tz is not the canonical UTC. Probe early and fall back to @@ -105,7 +112,7 @@ def convert_to_timestamp(time: int, precision: str, timezone: str): try: ts.utcoffset() except NotImplementedError: - return pd.Timestamp(time, unit=precision) + return pd.Timestamp(np.datetime64(int(time), precision)) return ts diff --git a/iotdb-client/client-py/tests/unit/test_rpc_utils.py b/iotdb-client/client-py/tests/unit/test_rpc_utils.py new file mode 100644 index 0000000000000..f68b294f80a0f --- /dev/null +++ b/iotdb-client/client-py/tests/unit/test_rpc_utils.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import numpy as np +import pandas as pd +import pytest + +from iotdb.utils.rpc_utils import convert_to_timestamp + + +@pytest.mark.parametrize("value", [-(2**63) + 1, 2**63 - 1]) +@pytest.mark.parametrize("timezone", ["UTC", "Asia/Shanghai"]) +def test_convert_timestamp_preserves_extreme_milliseconds(value, timezone): + timestamp = convert_to_timestamp(value, "ms", timezone) + assert isinstance(timestamp, pd.Timestamp) + assert not pd.isna(timestamp) + assert timestamp.asm8.astype("datetime64[ms]").astype(np.int64) == value