Skip to content

Fix date_bin month bins counting intra-second milliseconds twice #19265

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
DaZuiZui wants to merge 1 commit into from
+67 −8
Closed

Fix date_bin month bins counting intra-second milliseconds twice #19265

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 11 additions & 8 deletions ...e/iotdb/calc/transformation/dag/column/unary/scalar/DateBinFunctionColumnTransformer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,19 +90,22 @@ private static LocalDateTime convertToLocalDateTime(long timestamp, ZoneId zoneI
}

private static long convertToTimestamp(LocalDateTime dateTime, ZoneId zoneId) {
// Converting LocalDateTime to Seconds since epoch
long epochMilliSecond = dateTime.atZone(zoneId).toInstant().toEpochMilli();
// Get the nanoseconds section
long nanoAdjustment = dateTime.getNano();
// Encode from the epoch second and the nanosecond-of-second. toEpochMilli() already
// contains the whole milliseconds of that second, so adding getNano() on top counts
// those milliseconds a second time.
Instant instant = dateTime.atZone(zoneId).toInstant();
long seconds = instant.getEpochSecond();
long nanoOfSecond = instant.getNano();

switch (CommonDescriptor.getInstance().getConfig().getTimestampPrecision()) {
case "ms":
return epochMilliSecond + nanoAdjustment / NANOSECONDS_IN_MILLISECOND;
return Math.addExact(
Math.multiplyExact(seconds, 1000L), nanoOfSecond / NANOSECONDS_IN_MILLISECOND);
case "us":
return TimeUnit.MILLISECONDS.toMicros(epochMilliSecond)
+ nanoAdjustment / NANOSECONDS_IN_MICROSECOND;
return Math.addExact(
Math.multiplyExact(seconds, 1_000_000L), nanoOfSecond / NANOSECONDS_IN_MICROSECOND);
case "ns":
return TimeUnit.MILLISECONDS.toNanos(epochMilliSecond) + nanoAdjustment;
return Math.addExact(Math.multiplyExact(seconds, 1_000_000_000L), nanoOfSecond);
default:
throw new IllegalArgumentException(
CalcMessages.EXCEPTION_UNKNOWN_PRECISION_0119BEB0
Expand Down
56 changes: 56 additions & 0 deletions ...ache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/DateBinFunctionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@

package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar;

import org.apache.iotdb.calc.transformation.dag.column.leaf.NullColumnTransformer;
import org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DateBinFunctionColumnTransformer;
import org.apache.iotdb.commons.conf.CommonDescriptor;

import org.junit.Before;
import org.junit.Test;

Expand All @@ -28,6 +32,9 @@
import java.util.TimeZone;

import static org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DateBinFunctionColumnTransformer.dateBin;
import static org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DateBinFunctionColumnTransformer.nextDateBin;
import static org.apache.tsfile.read.common.type.TimestampType.TIMESTAMP;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;

public class DateBinFunctionTest {
Expand Down Expand Up @@ -126,4 +133,53 @@ public void testSourceBeforeUnixEpoch() throws ParseException {
zoneId);
assertEquals(getTimestamp("1969-12-31 00:00:00.000"), result);
}

@Test
public void testMonthBinKeepsFractionalOrigin() throws ParseException {
long origin = 500L;
long oneMonthLater = getTimestamp("1970-02-01 00:00:00.500");
long justBeforeNextMonth = oneMonthLater - 1;

assertEquals(origin, dateBin(origin, origin, 1, 0, zoneId));
assertEquals(origin, dateBin(justBeforeNextMonth, origin, 1, 0, zoneId));
assertEquals(oneMonthLater, dateBin(oneMonthLater, origin, 1, 0, zoneId));

long marchOrigin = getTimestamp("2024-03-15 00:00:00.500");
long aprilOrigin = getTimestamp("2024-04-15 00:00:00.500");
assertEquals(marchOrigin, dateBin(marchOrigin, marchOrigin, 1, 0, zoneId));
assertEquals(aprilOrigin, dateBin(aprilOrigin, marchOrigin, 1, 0, zoneId));

long exactSecond = getTimestamp("2024-03-15 00:00:00.000");
long nextExactMonth = getTimestamp("2024-04-15 00:00:00.000");
assertEquals(exactSecond, dateBin(exactSecond, exactSecond, 1, 0, zoneId));
assertEquals(nextExactMonth, dateBin(nextExactMonth, exactSecond, 1, 0, zoneId));
}

@Test
public void testMonthBinStartEndAndGapFillKeepFractionalMillis() throws ParseException {
long origin = 500L;
DateBinFunctionColumnTransformer transformer =
new DateBinFunctionColumnTransformer(
TIMESTAMP, 1, 0, new NullColumnTransformer(), origin, zoneId);

long oneMonthLater = getTimestamp("1970-02-01 00:00:00.500");
assertArrayEquals(new long[] {origin, oneMonthLater}, transformer.dateBinStartEnd(origin));
assertEquals(oneMonthLater, nextDateBin(1, zoneId, origin));
}

@Test
public void testMonthBinKeepsSubMillisecondOriginUnderUsAndNs() {
String originalPrecision = CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
try {
// 1.5 ms after the epoch, expressed in microseconds and nanoseconds.
CommonDescriptor.getInstance().getConfig().setTimestampPrecision("us");
assertEquals(1500L, dateBin(1500L, 1500L, 1, 0, zoneId));
assertEquals(1500L, nextDateBin(1, zoneId, 1500L) - 2678400000000L);

CommonDescriptor.getInstance().getConfig().setTimestampPrecision("ns");
assertEquals(1_500_000L, dateBin(1_500_000L, 1_500_000L, 1, 0, zoneId));
} finally {
CommonDescriptor.getInstance().getConfig().setTimestampPrecision(originalPrecision);
}
}
}