From 780d3ebc91c4b81ed4e605b800f6732b76e09ba3 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Thu, 12 Nov 2026 18:35:09 +0800 Subject: [PATCH 1/3] Fix out-of-range Python timestamps with pandas 3.1 --- .../client-py/iotdb/utils/rpc_utils.py | 17 +++-- .../client-py/tests/unit/test_rpc_utils.py | 65 +++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 iotdb-client/client-py/tests/unit/test_rpc_utils.py diff --git a/iotdb-client/client-py/iotdb/utils/rpc_utils.py b/iotdb-client/client-py/iotdb/utils/rpc_utils.py index 0912fd14d95de..e6aec4080c0cd 100644 --- a/iotdb-client/client-py/iotdb/utils/rpc_utils.py +++ b/iotdb-client/client-py/iotdb/utils/rpc_utils.py @@ -17,6 +17,7 @@ # import logging +import numpy as np import pandas as pd from pandas._libs import OutOfBoundsDatetime from tzlocal import get_localzone_name @@ -81,10 +82,16 @@ def convert_to_timestamp(time: int, precision: str, timezone: str): try: ts = pd.Timestamp(time, unit=precision, tz=timezone) except OutOfBoundsDatetime: + # A datetime64 scalar keeps the input unit; pandas 3.1 otherwise rescales + # numeric input to microseconds, which can overflow millisecond values. try: - ts = pd.Timestamp(time, unit=precision).tz_localize(timezone) - except NotImplementedError: - return pd.Timestamp(time, unit=precision) + ts = ( + pd.Timestamp(np.datetime64(int(time), precision)) + .tz_localize("UTC") + .tz_convert(timezone) + ) + except (OutOfBoundsDatetime, NotImplementedError): + return pd.Timestamp(np.datetime64(int(time), precision)) except ValueError: logger.warning( f"Timezone string '{timezone}' cannot be recognized by pandas. " @@ -95,7 +102,7 @@ def convert_to_timestamp(time: int, precision: str, timezone: str): # Timestamp falls outside Python's stdlib datetime range (year < 1 or # > 9999). pandas >= 3.0 cannot attach a timezone to such Timestamps # because tz conversion relies on toordinal(). Return naive instead. - return pd.Timestamp(time, unit=precision) + return pd.Timestamp(np.datetime64(int(time), precision)) # Construction succeeded but utcoffset() may still fail downstream (e.g. # in pd.DataFrame's tz alignment) when the Timestamp's year is outside # 1..9999 and tz is not the canonical UTC. Probe early and fall back to @@ -105,7 +112,7 @@ def convert_to_timestamp(time: int, precision: str, timezone: str): try: ts.utcoffset() except NotImplementedError: - return pd.Timestamp(time, unit=precision) + return pd.Timestamp(np.datetime64(int(time), precision)) return ts diff --git a/iotdb-client/client-py/tests/unit/test_rpc_utils.py b/iotdb-client/client-py/tests/unit/test_rpc_utils.py new file mode 100644 index 0000000000000..04b1c4bcd9d39 --- /dev/null +++ b/iotdb-client/client-py/tests/unit/test_rpc_utils.py @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import numpy as np +import pandas as pd +import pytest + +from iotdb.utils.Field import Field +from iotdb.utils.IoTDBConstants import TSDataType +from iotdb.utils.rpc_utils import convert_to_timestamp + + +@pytest.mark.parametrize("precision", ["ms", "us", "ns"]) +@pytest.mark.parametrize("value", [-(2**63) + 1, 2**63 - 1]) +@pytest.mark.parametrize( + "timezone", ["UTC", "Etc/UTC", "Asia/Shanghai", "America/New_York"] +) +@pytest.mark.parametrize("integer_type", [int, np.int64]) +def test_convert_timestamp_preserves_extreme_value( + precision, value, timezone, integer_type +): + timestamp = convert_to_timestamp(integer_type(value), precision, timezone) + assert isinstance(timestamp, pd.Timestamp) + assert not pd.isna(timestamp) + assert timestamp.asm8.astype(f"datetime64[{precision}]").astype(np.int64) == value + + +@pytest.mark.parametrize("precision", ["ms", "us", "ns"]) +def test_convert_timestamp_preserves_nat(precision): + assert convert_to_timestamp(-(2**63), precision, "Etc/UTC") is pd.NaT + + +@pytest.mark.parametrize( + "timezone", ["UTC", "Etc/UTC", "Asia/Shanghai", "America/New_York"] +) +def test_convert_timestamp_preserves_normal_timezone(timezone): + timestamp = convert_to_timestamp(1, "ms", timezone) + expected = pd.Timestamp("1970-01-01T00:00:00.001Z").tz_convert(timezone) + assert timestamp == expected + assert timestamp.utcoffset() == expected.utcoffset() + + +def test_timestamp_field_formats_extreme_milliseconds(): + field = Field(TSDataType.TIMESTAMP, 2**63 - 1, timezone="Etc/UTC", precision="ms") + assert ( + field.get_timestamp_value().asm8.astype("datetime64[ms]").astype(np.int64) + == 2**63 - 1 + ) + suffix = "+00:00" if field.get_timestamp_value().tzinfo is not None else "" + assert field.get_string_value() == "292278994-08-17T07:12:55.807" + suffix From 6cf0ab631a55eedc39b867d17e4bee64ac60be94 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Thu, 12 Nov 2026 18:58:41 +0800 Subject: [PATCH 2/3] Remove added timestamp regression tests --- .../client-py/tests/unit/test_rpc_utils.py | 65 ------------------- 1 file changed, 65 deletions(-) delete mode 100644 iotdb-client/client-py/tests/unit/test_rpc_utils.py diff --git a/iotdb-client/client-py/tests/unit/test_rpc_utils.py b/iotdb-client/client-py/tests/unit/test_rpc_utils.py deleted file mode 100644 index 04b1c4bcd9d39..0000000000000 --- a/iotdb-client/client-py/tests/unit/test_rpc_utils.py +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import numpy as np -import pandas as pd -import pytest - -from iotdb.utils.Field import Field -from iotdb.utils.IoTDBConstants import TSDataType -from iotdb.utils.rpc_utils import convert_to_timestamp - - -@pytest.mark.parametrize("precision", ["ms", "us", "ns"]) -@pytest.mark.parametrize("value", [-(2**63) + 1, 2**63 - 1]) -@pytest.mark.parametrize( - "timezone", ["UTC", "Etc/UTC", "Asia/Shanghai", "America/New_York"] -) -@pytest.mark.parametrize("integer_type", [int, np.int64]) -def test_convert_timestamp_preserves_extreme_value( - precision, value, timezone, integer_type -): - timestamp = convert_to_timestamp(integer_type(value), precision, timezone) - assert isinstance(timestamp, pd.Timestamp) - assert not pd.isna(timestamp) - assert timestamp.asm8.astype(f"datetime64[{precision}]").astype(np.int64) == value - - -@pytest.mark.parametrize("precision", ["ms", "us", "ns"]) -def test_convert_timestamp_preserves_nat(precision): - assert convert_to_timestamp(-(2**63), precision, "Etc/UTC") is pd.NaT - - -@pytest.mark.parametrize( - "timezone", ["UTC", "Etc/UTC", "Asia/Shanghai", "America/New_York"] -) -def test_convert_timestamp_preserves_normal_timezone(timezone): - timestamp = convert_to_timestamp(1, "ms", timezone) - expected = pd.Timestamp("1970-01-01T00:00:00.001Z").tz_convert(timezone) - assert timestamp == expected - assert timestamp.utcoffset() == expected.utcoffset() - - -def test_timestamp_field_formats_extreme_milliseconds(): - field = Field(TSDataType.TIMESTAMP, 2**63 - 1, timezone="Etc/UTC", precision="ms") - assert ( - field.get_timestamp_value().asm8.astype("datetime64[ms]").astype(np.int64) - == 2**63 - 1 - ) - suffix = "+00:00" if field.get_timestamp_value().tzinfo is not None else "" - assert field.get_string_value() == "292278994-08-17T07:12:55.807" + suffix From 11814177258c77407267e93c83fef41aa3e1f8d3 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Fri, 13 Nov 2026 09:54:29 +0800 Subject: [PATCH 3/3] Add focused extreme millisecond timestamp regression coverage --- .../client-py/tests/unit/test_rpc_utils.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 iotdb-client/client-py/tests/unit/test_rpc_utils.py diff --git a/iotdb-client/client-py/tests/unit/test_rpc_utils.py b/iotdb-client/client-py/tests/unit/test_rpc_utils.py new file mode 100644 index 0000000000000..f68b294f80a0f --- /dev/null +++ b/iotdb-client/client-py/tests/unit/test_rpc_utils.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import numpy as np +import pandas as pd +import pytest + +from iotdb.utils.rpc_utils import convert_to_timestamp + + +@pytest.mark.parametrize("value", [-(2**63) + 1, 2**63 - 1]) +@pytest.mark.parametrize("timezone", ["UTC", "Asia/Shanghai"]) +def test_convert_timestamp_preserves_extreme_milliseconds(value, timezone): + timestamp = convert_to_timestamp(value, "ms", timezone) + assert isinstance(timestamp, pd.Timestamp) + assert not pd.isna(timestamp) + assert timestamp.asm8.astype("datetime64[ms]").astype(np.int64) == value