From d0b363db95b6fb0c68c131af141012c1f0b061a6 Mon Sep 17 00:00:00 2001 From: Fabian Hueske Date: Thu, 7 May 2026 16:55:10 +0200 Subject: [PATCH 1/5] [FLINK-39781][table] Add LateralSnapshotJoinOperator with two-phase LOAD/JOIN execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stateful keyed two-input operator backing LATERAL SNAPSHOT joins. * Two phases (LOAD/JOIN) gated by an operator UnionListState; mixed-phase rescale collapses to LOAD. Flip triggers: build-side WM reaches loadCompletedTime, or a processing-time idle-timeout fallback. * Probe records are buffered in LOAD and joined on flip. Build-side changes are buffered per-key and applied lazily in event-time order on the next per-key access once the build-side WM advanced, preserving atomic -U/+U visibility. * Watermarks: build-side absorbed; probe-side held back in LOAD, forwarded in JOIN. NULL equi-keys filtered via JoinConditionWithNullFilters. * State TTL via keyed processing-time timers with a post-flip grace window; rearms are amortized so effective time-to-eviction is in [1.0×, 1.5×] stateTtlMs. Generated-By: Claude Opus 4.7 (1M context) --- .../snapshot/LateralSnapshotJoinOperator.java | 1078 +++++++++ .../LateralSnapshotJoinOperatorTest.java | 1963 +++++++++++++++++ 2 files changed, 3041 insertions(+) create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/join/snapshot/LateralSnapshotJoinOperator.java create mode 100644 flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/snapshot/LateralSnapshotJoinOperatorTest.java diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/join/snapshot/LateralSnapshotJoinOperator.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/join/snapshot/LateralSnapshotJoinOperator.java new file mode 100644 index 00000000000000..7cab71b9391b84 --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/join/snapshot/LateralSnapshotJoinOperator.java @@ -0,0 +1,1078 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.join.snapshot; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.functions.DefaultOpenContext; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.MapState; +import org.apache.flink.api.common.state.MapStateDescriptor; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.common.typeutils.base.EnumSerializer; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.Gauge; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.runtime.state.StateBackendLoader; +import org.apache.flink.runtime.state.StateInitializationContext; +import org.apache.flink.runtime.state.StateSnapshotContext; +import org.apache.flink.runtime.state.VoidNamespace; +import org.apache.flink.runtime.state.VoidNamespaceSerializer; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.InternalTimer; +import org.apache.flink.streaming.api.operators.InternalTimerService; +import org.apache.flink.streaming.api.operators.TimestampedCollector; +import org.apache.flink.streaming.api.operators.Triggerable; +import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.utils.JoinedRowData; +import org.apache.flink.table.runtime.generated.GeneratedJoinCondition; +import org.apache.flink.table.runtime.generated.JoinCondition; +import org.apache.flink.table.runtime.operators.join.JoinConditionWithNullFilters; +import org.apache.flink.table.runtime.operators.metrics.SimpleGauge; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.types.RowKind; +import org.apache.flink.util.Preconditions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ScheduledFuture; + +/** + * Stream operator implementing the {@code LATERAL SNAPSHOT} processing-time temporal table join. + * + *

The operator runs in two operator-wide phases that progress only forward, LOAD then JOIN: LOAD + * bootstraps the build-side table from its changelog up to {@code loadCompletedTime}, buffering + * probe rows meanwhile; JOIN then continuously joins probe rows against the materialized build-side + * state. How each input is handled depends on the current phase: + * + *

+ * + *

Watermark forwarding rules: + * + *

+ * + *

The flip from LOAD to JOIN phase is triggered by either: + * + *

+ * + *

State TTL eviction happens during JOIN phase and is implemented with keyed processing-time + * timers (matching the semantics of Flink's standard {@code StateTtlConfig}). + * + *

Streaming only: The operator joins data with processing-time semantics which can't be (easily) + * done in batch. Also, the operator's phase is held in union operator state, which is incompatible + * with finished operators. Joins against LATERAL SNAPSHOT functions should be translated to regular + * joins. The changes on the build-side would be consolidated into a final table and then be joined + * with the probe-side. All probe-side records are then joined against the same (and final) version + * of the build-side input. + */ +@Internal +public class LateralSnapshotJoinOperator extends AbstractStreamOperator + implements TwoInputStreamOperator, + Triggerable { + + // -------------------------- static final definitions -------------------------- + + private static final long serialVersionUID = 1L; + + private static final Logger LOG = LoggerFactory.getLogger(LateralSnapshotJoinOperator.class); + + /** Operator state names. */ + private static final String OPERATOR_PHASE_STATE_NAME = "lateral-snapshot-phase"; + + /** Keyed state names. */ + @VisibleForTesting static final String BUILD_TABLE_STATE_NAME = "build-table"; + + @VisibleForTesting static final String BUILD_CHANGE_BUFFER_STATE_NAME = "build-change-buffer"; + @VisibleForTesting static final String BUFFERED_AT_WM_STATE_NAME = "buffered-at-wm"; + @VisibleForTesting static final String PROBE_BUFFER_STATE_NAME = "probe-buffer"; + private static final String PROBE_BUFFER_SEQ_STATE_NAME = "probe-buffer-seq"; + private static final String TTL_EXPIRY_STATE_NAME = "ttl-expiry"; + + /** + * Single timer service for two kinds of per-key timers, distinguished by timer type (not by + * namespace): event-time timers drain a key's LOAD-buffered probes at the flip ({@link + * #onEventTime}), processing-time timers evict keyed state ({@link #onProcessingTime}). Both + * use {@link VoidNamespace} so no namespace is serialized per timer. + */ + private static final String TIMER_SERVICE_NAME = "lateral-snapshot-timers"; + + /** Gauge: probe-side records currently buffered during LOAD. */ + @VisibleForTesting + static final String NUM_PROBE_BUFFERED_METRIC_NAME = "numProbeSideRecordsBuffered"; + + /** Counter: keys evicted from state by TTL. */ + @VisibleForTesting + static final String NUM_STATE_TTL_EVICTIONS_METRIC_NAME = "numStateTtlEvictions"; + + /** Gauge: latest build-side watermark observed. */ + @VisibleForTesting + static final String CURRENT_BUILD_WM_METRIC_NAME = "currentBuildSideWatermark"; + + /** Gauge: latest probe-side watermark observed. */ + @VisibleForTesting + static final String CURRENT_PROBE_WM_METRIC_NAME = "currentProbeSideWatermark"; + + /** Gauge: current phase ordinal, 0 = LOAD, 1 = JOIN. */ + @VisibleForTesting static final String CURRENT_PHASE_METRIC_NAME = "currentPhase"; + + /** Gauge: max joined records emitted for a probe-side record. */ + @VisibleForTesting static final String MAX_JOIN_FAN_OUT_METRIC_NAME = "maxJoinFanOut"; + + /** Gauge: average joined records emitted per probe-side record. */ + @VisibleForTesting static final String AVG_JOIN_FAN_OUT_METRIC_NAME = "avgJoinFanOut"; + + /** Counter: probe-side records without a match. */ + @VisibleForTesting + static final String NUM_UNMATCHED_PROBE_METRIC_NAME = "numUnmatchedProbeRecords"; + + /** Counter: build-side retractions for a row not present in state. */ + @VisibleForTesting + static final String NUM_UNMATCHED_BUILD_RETRACTIONS_METRIC_NAME = + "numUnmatchedBuildRetractions"; + + /** Two-phase state machine. */ + enum Phase { + LOAD, + JOIN + } + + /** What triggered the {@link Phase#LOAD} to {@link Phase#JOIN} flip (for logging). */ + private enum FlipTrigger { + BUILD_WATERMARK, + IDLE_TIMEOUT + } + + // -------------------------- constructor args -------------------------- + + private final boolean isLeftOuterJoin; + private final InternalTypeInfo leftType; + private final InternalTypeInfo rightType; + + /** Field index of the build-side (right) row-time attribute. */ + private final int buildRowtimeIndex; + + private final GeneratedJoinCondition generatedJoinCondition; + + /** + * Per-equi-key flag indicating whether rows with a NULL in that key position must be filtered + * before the join condition runs (SQL semantics: {@code NULL = NULL} is not true). + */ + private final boolean[] filterNullKeys; + + /** + * Timestamp at which the build-side watermark must arrive for the operator to flip from {@code + * LOAD} to JOIN. + */ + private final long loadCompletedTime; + + /** + * Processing-time idle timeout duration (millis) on build-side watermarks. When configured, the + * operator flips to JOIN if no build-side watermark advance is seen for this duration. + */ + @Nullable private final Long loadCompletedIdleTimeoutMs; + + /** + * State TTL (millis) to clean up any keyed state during JOIN phase. We schedule TTL timers + * maxStateTtlMs ahead and check on minStateTtlMs before scheduling a new timer. This avoids + * rescheduling timers on every state access while still ensuring that keyed state is evicted + * after at most maxStateTtlMs of key inactivity during JOIN phase. If minStateTtlMs is set to + * 0, state TTL is disabled. + */ + private final long minStateTtlMs; + + private final long maxStateTtlMs; + + // -------------------------- transient runtime fields -------------------------- + + private transient JoinConditionWithNullFilters joinCondition; + private transient GenericRowData nullPaddedBuild; + private transient TimestampedCollector collector; + + private transient InternalTimerService timerService; + + private transient Phase phase; + + /** + * Processing-time wall clock at which the operator transitioned from {@link Phase#LOAD} to + * {@link Phase#JOIN}. {@code null} while still in LOAD. Used by the TTL handler to reschedule + * state TTL timers that fire too early. + */ + @Nullable private transient Long flipProcTime; + + /** Highest build-side watermark observed. */ + private transient long currentBuildSideWm; + + /** Latest probe-side watermark observed. */ + private transient long currentProbeSideWm; + + /** Non-keyed processing-time idle-flip timer. */ + @Nullable private transient ScheduledFuture idleFlipTimer; + + /** + * True if the keyed state backend iterates map entries in serialized-key order (RocksDB/ForSt), + * so the row-time-keyed {@link #buildChangeBuffer} is already scanned in event-time order. + */ + private transient boolean sortedStateBackend; + + // -------------------------- keyed state -------------------------- + + /** + * Build-side table as multi-set (row → reference count). Holds the committed snapshot, i.e. all + * build changes up to the last applied build watermark; probes join against this. + */ + private transient MapState buildTableState; + + /** + * Build-side changes not yet visible in {@link #buildTableState}, keyed by their build row-time + * (multiple changes with the same row-time share a list value). Changes are applied to {@code + * buildTableState} once the build watermark reaches their row-time. + */ + private transient MapState> buildChangeBuffer; + + /** Build-side watermark to ensure atomic application of build changes. */ + private transient ValueState bufferedAtWmState; + + /** + * Buffer for probe-side records during LOAD (and for the rare JOIN-phase record that arrives + * while a key's buffer is still draining). Keyed by a synthetic, monotonically increasing + * timestamp so the per-record flip timers fire in insertion order. Each buffered row has one + * event-time timer registered on its key; the timer drains exactly that row. + */ + private transient MapState probeBuffer; + + /** + * Next sequence number for the current key, used to derive distinct, ordered synthetic + * timestamps for {@link #probeBuffer}. {@code null} iff the buffer is empty, so it doubles as a + * cheap "buffer non-empty" flag on the JOIN-phase fast path. + */ + private transient ValueState probeBufferSeq; + + /** Most recently registered TTL timer deadline; used to advance TTL timer. */ + private transient ValueState ttlExpiryState; + + // -------------------------- operator state -------------------------- + + private transient ListState operatorPhaseState; + + // -------------------------- metrics -------------------------- + + private transient Counter numStateTtlEvictions; + private transient Counter numUnmatchedProbeRecords; + private transient Counter numUnmatchedBuildRetractions; + + private transient SimpleGauge probeBufferedGauge; + private transient SimpleGauge buildWmGauge; + private transient SimpleGauge probeWmGauge; + private transient SimpleGauge maxFanOutGauge; + private transient SimpleGauge avgFanOutGauge; + private transient Gauge phaseGauge; + + /** Backing accumulators for the push-model gauges (in-memory, not persisted, best-effort). */ + private transient long probeBufferedCount; + + private transient long maxJoinFanOut; + private transient long totalJoinFanOut; + private transient long totalProbeJoins; + + public LateralSnapshotJoinOperator( + boolean isLeftOuterJoin, + InternalTypeInfo leftType, + InternalTypeInfo rightType, + int buildRowtimeIndex, + GeneratedJoinCondition generatedJoinCondition, + boolean[] filterNullKeys, + Long loadCompletedTime, + @Nullable Long loadCompletedIdleTimeoutMs, + @Nullable Long stateTtlMs) { + this.isLeftOuterJoin = isLeftOuterJoin; + this.leftType = Preconditions.checkNotNull(leftType); + this.rightType = Preconditions.checkNotNull(rightType); + this.buildRowtimeIndex = buildRowtimeIndex; + if (buildRowtimeIndex < 0) { + throw new IllegalArgumentException("buildRowtimeIndex must be non-negative"); + } + if (buildRowtimeIndex >= rightType.toRowType().getFieldCount()) { + throw new IllegalArgumentException("buildRowtimeIndex out of bounds for build row"); + } + this.generatedJoinCondition = Preconditions.checkNotNull(generatedJoinCondition); + this.filterNullKeys = Preconditions.checkNotNull(filterNullKeys); + this.loadCompletedTime = Preconditions.checkNotNull(loadCompletedTime); + if (this.loadCompletedTime < 0) { + throw new IllegalArgumentException("loadCompletedTime must be non-negative"); + } + this.loadCompletedIdleTimeoutMs = loadCompletedIdleTimeoutMs; + if (this.loadCompletedIdleTimeoutMs != null && this.loadCompletedIdleTimeoutMs < 0) { + throw new IllegalArgumentException("loadCompletedIdleTimeoutMs must be non-negative"); + } + this.minStateTtlMs = stateTtlMs == null ? 0 : stateTtlMs; + if (this.minStateTtlMs < 0) { + throw new IllegalArgumentException("stateTtlMs must be non-negative"); + } + // maxStateTtlMs is 1.5x of minStateTtlMs + this.maxStateTtlMs = this.minStateTtlMs + this.minStateTtlMs / 2; + } + + // -------------------------- lifecycle -------------------------- + + @Override + public boolean useInterruptibleTimers(ReadableConfig config) { + return true; + } + + @Override + public void initializeState(StateInitializationContext context) throws Exception { + super.initializeState(context); + + // Operator state only — keyed state and timer services are initialized in open() + operatorPhaseState = + context.getOperatorStateStore() + .getUnionListState( + new ListStateDescriptor<>( + OPERATOR_PHASE_STATE_NAME, + new EnumSerializer<>(Phase.class))); + + // any LOAD entry → LOAD; empty (fresh start) → LOAD; else JOIN + boolean phaseStateExists = false; + boolean anyTaskInLoad = false; + for (Phase persistedPhase : operatorPhaseState.get()) { + phaseStateExists = true; + if (persistedPhase == Phase.LOAD) { + anyTaskInLoad = true; + break; + } + } + // we start in LOAD phase if no phaseState exists (no savepoint/checkpoint) or any task was + // still in LOAD phase (not all tasks transitioned to JOIN phase). + phase = (!phaseStateExists || anyTaskInLoad) ? Phase.LOAD : Phase.JOIN; + + // When restored into JOIN, anchor flipProcTime on the current wall clock so the TTL + // handler's post-flip grace window restarts from now. + flipProcTime = + phase == Phase.JOIN ? getProcessingTimeService().getCurrentProcessingTime() : null; + + currentBuildSideWm = Long.MIN_VALUE; + currentProbeSideWm = Long.MIN_VALUE; + + probeBufferedCount = 0L; + maxJoinFanOut = 0L; + totalJoinFanOut = 0L; + totalProbeJoins = 0L; + } + + @Override + public void open() throws Exception { + super.open(); + + // Setup keyed states + buildTableState = + getRuntimeContext() + .getMapState( + new MapStateDescriptor<>( + BUILD_TABLE_STATE_NAME, rightType, Types.LONG)); + buildChangeBuffer = + getRuntimeContext() + .getMapState( + new MapStateDescriptor<>( + BUILD_CHANGE_BUFFER_STATE_NAME, + Types.LONG, + Types.LIST(rightType))); + bufferedAtWmState = + getRuntimeContext() + .getState( + new ValueStateDescriptor<>(BUFFERED_AT_WM_STATE_NAME, Types.LONG)); + probeBuffer = + getRuntimeContext() + .getMapState( + new MapStateDescriptor<>( + PROBE_BUFFER_STATE_NAME, Types.LONG, leftType)); + probeBufferSeq = + getRuntimeContext() + .getState( + new ValueStateDescriptor<>( + PROBE_BUFFER_SEQ_STATE_NAME, Types.LONG)); + ttlExpiryState = + getRuntimeContext() + .getState(new ValueStateDescriptor<>(TTL_EXPIRY_STATE_NAME, Types.LONG)); + + // Setup timerservice. Both timer kinds use VoidNamespace; they are told apart by timer type + // (event-time -> flip drain, processing-time -> TTL eviction), not by namespace. + timerService = + getInternalTimerService(TIMER_SERVICE_NAME, VoidNamespaceSerializer.INSTANCE, this); + + // RocksDB/ForSt iterate map entries in serialized-key order, so the row-time-keyed build + // change buffer is scanned in event-time order; unordered backends (heap) need a sort. + final String backendId = getKeyedStateBackend().getBackendTypeIdentifier(); + sortedStateBackend = + StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME.equals(backendId) + || StateBackendLoader.FORST_STATE_BACKEND_NAME.equals(backendId); + + // Wrap the codegen'd condition with a null-key filter so SQL semantics are honored for + // equi-keys whose values may be NULL. + final JoinCondition rawCondition = + generatedJoinCondition.newInstance(getRuntimeContext().getUserCodeClassLoader()); + joinCondition = new JoinConditionWithNullFilters(rawCondition, filterNullKeys, this); + joinCondition.setRuntimeContext(getRuntimeContext()); + joinCondition.open(DefaultOpenContext.INSTANCE); + + // Construct null-padded record for left-outer join + nullPaddedBuild = + isLeftOuterJoin ? new GenericRowData(rightType.toRowType().getFieldCount()) : null; + + // Create output collector + collector = new TimestampedCollector<>(output); + + // Register metrics + final MetricGroup metricGroup = getRuntimeContext().getMetricGroup(); + numStateTtlEvictions = metricGroup.counter(NUM_STATE_TTL_EVICTIONS_METRIC_NAME); + numUnmatchedProbeRecords = metricGroup.counter(NUM_UNMATCHED_PROBE_METRIC_NAME); + numUnmatchedBuildRetractions = + metricGroup.counter(NUM_UNMATCHED_BUILD_RETRACTIONS_METRIC_NAME); + + // Best-effort in-memory tally; not restored, so it starts at 0 after a restore/rescale. + probeBufferedGauge = new SimpleGauge<>(probeBufferedCount); + buildWmGauge = new SimpleGauge<>(currentBuildSideWm); + probeWmGauge = new SimpleGauge<>(currentProbeSideWm); + maxFanOutGauge = new SimpleGauge<>(maxJoinFanOut); + avgFanOutGauge = new SimpleGauge<>(0.0d); + metricGroup.gauge(NUM_PROBE_BUFFERED_METRIC_NAME, probeBufferedGauge); + metricGroup.gauge(CURRENT_BUILD_WM_METRIC_NAME, buildWmGauge); + metricGroup.gauge(CURRENT_PROBE_WM_METRIC_NAME, probeWmGauge); + metricGroup.gauge(MAX_JOIN_FAN_OUT_METRIC_NAME, maxFanOutGauge); + metricGroup.gauge(AVG_JOIN_FAN_OUT_METRIC_NAME, avgFanOutGauge); + phaseGauge = () -> phase == null ? -1 : phase.ordinal(); + metricGroup.gauge(CURRENT_PHASE_METRIC_NAME, phaseGauge); + + // Pin the build-side input idle: the operator absorbs build-side watermarks and status. + combinedWatermark.updateStatus(1, true); + + if (phase == Phase.LOAD && loadCompletedIdleTimeoutMs != null) { + scheduleIdleFlipTimer(); + } + + LOG.info( + "Opened LateralSnapshotJoinOperator: phase={}, leftOuter={}, loadCompletedTime={}, " + + "idleTimeoutMs={}, stateTtlMs=[{}, {}], buildRowtimeIndex={}", + phase, + isLeftOuterJoin, + loadCompletedTime, + loadCompletedIdleTimeoutMs, + minStateTtlMs, + maxStateTtlMs, + buildRowtimeIndex); + } + + @Override + public void snapshotState(StateSnapshotContext context) throws Exception { + super.snapshotState(context); + operatorPhaseState.update(List.of(phase)); + } + + @Override + public void close() throws Exception { + if (idleFlipTimer != null) { + idleFlipTimer.cancel(false); + idleFlipTimer = null; + } + if (joinCondition != null) { + joinCondition.close(); + } + super.close(); + } + + // -------------------------- input row processing -------------------------- + + @Override + public void processElement1(StreamRecord element) throws Exception { + RowData probe = element.getValue(); + // Apply any buffered build-side changes if the build-side watermark has advanced. + applyBufferedChangesIfReady(); + // Buffer during LOAD; in JOIN, buffer too if this key's flip drain has not finished yet + // (probeBufferSeq != null), so the new probe is joined in order after the buffered ones by + // its own timer instead of draining the whole buffer inline. Otherwise join immediately. + if (phase == Phase.LOAD || probeBufferSeq.value() != null) { + bufferProbe(probe); + } else { + joinProbeRow(probe); + } + refreshStateTtl(); + } + + /** + * Buffers a probe row for the current key and registers a per-record event-time flip timer. The + * synthetic timestamp {@code (seq + 1) - Long.MAX_VALUE} is negative (so any non-negative + * watermark fires it) and increases with the sequence number (so timers drain in insertion + * order on every state backend). + */ + private void bufferProbe(RowData probe) throws Exception { + long seq = probeBufferSeq.value() == null ? 0L : probeBufferSeq.value(); + long ts = (seq + 1) - Long.MAX_VALUE; + probeBuffer.put(ts, probe); + probeBufferSeq.update(seq + 1); + timerService.registerEventTimeTimer(VoidNamespace.INSTANCE, ts); + probeBufferedGauge.update(++probeBufferedCount); + } + + @Override + public void processElement2(StreamRecord element) throws Exception { + RowData build = element.getValue(); + // Apply any buffered build-side changes if the build-side watermark has advanced. + Long bufferedAt = applyBufferedChangesIfReady(); + // Buffer the change under its build row-time (grouping changes that share a row-time). + long rowtime = build.getLong(buildRowtimeIndex); + List atRowtime = buildChangeBuffer.get(rowtime); + if (atRowtime == null) { + atRowtime = new ArrayList<>(); + } + atRowtime.add(build); + buildChangeBuffer.put(rowtime, atRowtime); + // Tag the buffer with the current build watermark so it drains once the watermark passes. + // The tag may move backwards on restore (currentBuildSideWm resets to MIN_VALUE), which + // re-anchors a recovered buffer to this subtask's watermark. + if (bufferedAt == null || bufferedAt != currentBuildSideWm) { + bufferedAtWmState.update(currentBuildSideWm); + } + refreshStateTtl(); + } + + // -------------------------- watermark processing -------------------------- + + @Override + public void processWatermark1(Watermark mark) throws Exception { + // Probe-side watermark: held back during LOAD (neither advances the timer service nor is + // forwarded), forwarded during JOIN. + if (phase == Phase.JOIN) { + super.processWatermark1(mark); + } + currentProbeSideWm = Math.max(currentProbeSideWm, mark.getTimestamp()); + probeWmGauge.update(currentProbeSideWm); + } + + @Override + public void processWatermark2(Watermark mark) throws Exception { + // Build-side watermark: NEVER forwarded; never advances the timer service. + long ts = mark.getTimestamp(); + currentBuildSideWm = Math.max(currentBuildSideWm, ts); + buildWmGauge.update(currentBuildSideWm); + if (phase == Phase.LOAD) { + if (currentBuildSideWm >= loadCompletedTime) { + // we reached the flip point. Transition to JOIN phase. + transitionToJoinPhase(FlipTrigger.BUILD_WATERMARK); + } else if (loadCompletedIdleTimeoutMs != null) { + // we got a new build-side wm. Reschedule the idle timer (if it was configured) + rescheduleIdleFlipTimer(); + } + } + } + + @Override + protected void processWatermarkStatus(WatermarkStatus watermarkStatus, int index) + throws Exception { + if (index == 1) { + // Build-side idle status is absorbed; combined watermark is probe-driven. + return; + } + if (phase == Phase.LOAD) { + // LOAD emits nothing downstream; track partial[0]'s idle bit for use after the flip. + combinedWatermark.updateStatus(0, watermarkStatus.isIdle()); + return; + } + super.processWatermarkStatus(watermarkStatus, index); + } + + // -------------------------- timers processing -------------------------- + + @Override + public void onEventTime(InternalTimer timer) throws Exception { + // Only event-time (flip) timers are registered: each drains exactly one buffered probe for + // its key. All of a key's timers become due at the flip; interruptible timers fire them one + // at a time, yielding to checkpoints between firings. + long ts = timer.getTimestamp(); + // Apply this key's due build-side changes before joining, so probes see the materialized + // build snapshot. Idempotent across the key's firings within one watermark sweep. + applyDueBufferedChanges(); + RowData probe = probeBuffer.get(ts); + if (probe == null) { + // Already drained or evicted (e.g. by TTL) before this timer fired. + return; + } + joinProbeRow(probe); + probeBuffer.remove(ts); + probeBufferedCount = Math.max(0, probeBufferedCount - 1); + probeBufferedGauge.update(probeBufferedCount); + // This is the last buffered row iff it carries the highest sequence number; clearing the + // sequence here marks the buffer empty (cheap point lookup, no isEmpty() scan). + Long seq = probeBufferSeq.value(); + if (seq != null && ts + Long.MAX_VALUE == seq) { + probeBufferSeq.clear(); + } + } + + /** Removes all probe-buffer state for the current key. */ + private void clearProbeBuffer() { + probeBuffer.clear(); + probeBufferSeq.clear(); + } + + @Override + public void onProcessingTime(InternalTimer timer) throws Exception { + // Only processing-time (TTL) timers are registered. Semantics match Flink's standard + // StateTtlConfig. + if (minStateTtlMs == 0) { + // TTL wasn't configured and shouldn't have registered any timers. + return; + } + Long deadline = ttlExpiryState.value(); + if (deadline == null || timer.getTimestamp() != deadline) { + return; // stale timer fire + } + long now = getProcessingTimeService().getCurrentProcessingTime(); + // Reschedule if still in LOAD, or in JOIN but within stateTtlMs of the flip (grace window). + if (phase == Phase.LOAD || (flipProcTime != null && now < flipProcTime + minStateTtlMs)) { + // set the new TTL timer maxStateTtlMs ahead + long newDeadline = + phase == Phase.LOAD ? now + maxStateTtlMs : flipProcTime + maxStateTtlMs; + timerService.registerProcessingTimeTimer(VoidNamespace.INSTANCE, newDeadline); + ttlExpiryState.update(newDeadline); + return; + } + clearAllPerKeyState(); + numStateTtlEvictions.inc(); + } + + /** + * Registers the load-completion idle-timeout timer. No-op when the timeout is not configured. + */ + private void scheduleIdleFlipTimer() { + if (loadCompletedIdleTimeoutMs == null) { + return; + } + long deadline = + getProcessingTimeService().getCurrentProcessingTime() + loadCompletedIdleTimeoutMs; + idleFlipTimer = + getProcessingTimeService() + .registerTimer( + deadline, t -> transitionToJoinPhase(FlipTrigger.IDLE_TIMEOUT)); + } + + /** Updates the idle flip timer. */ + private void rescheduleIdleFlipTimer() { + cancelIdleFlipTimer(); + scheduleIdleFlipTimer(); + } + + /** Deactivates the currently registered idle flip timer. */ + private void cancelIdleFlipTimer() { + if (idleFlipTimer != null) { + idleFlipTimer.cancel(false); + idleFlipTimer = null; + } + } + + // -------------------------- core logic -------------------------- + + /** + * Transition from LOAD to JOIN. Runs in a NON-KEYED context (callers {@link #processWatermark2} + * and {@link #idleFlipTimer}), so it must not touch keyed state directly; the per-key probe + * drain happens in the per-record event-time timers fired below. + */ + private void transitionToJoinPhase(FlipTrigger trigger) throws Exception { + if (phase == Phase.JOIN) { + return; + } + LOG.info( + "Flipping phase LOAD -> JOIN (trigger={}): buildWm={}, loadCompletedTime={}. " + + "Joining buffered probe records...", + trigger, + currentBuildSideWm, + loadCompletedTime); + phase = Phase.JOIN; + // Anchor the TTL grace window at the flip so keys loaded before it aren't evicted early. + flipProcTime = getProcessingTimeService().getCurrentProcessingTime(); + cancelIdleFlipTimer(); + + // If we have seen a (non-negative) probe-side WM, forward it so the per-record flip timers + // fire (their synthetic timestamps are negative) and drain the buffered probes. Firing is + // interruptible and the watermark is forwarded after the buffered probes are joined. + // If we haven't seen a probe-side WM yet, draining is triggered by the next probe-side WM. + if (currentProbeSideWm >= 0) { + super.processWatermark1(new Watermark(currentProbeSideWm)); + } + LOG.info( + "Completed flip to JOIN phase (trigger={}): emittedProbeWm={}", + trigger, + currentProbeSideWm >= 0 ? currentProbeSideWm : "none"); + } + + /** + * Joins a probe-side row against the current build-side table and applies the join predicate. + * Returns a null-padded result if the row doesn't match any build-side row and this is a LEFT + * OUTER join. + */ + private void joinProbeRow(RowData probe) throws Exception { + boolean matched = false; + // Number of result rows emitted for this probe row (the join fan-out). + long fanOut = 0; + for (Map.Entry entry : buildTableState.entries()) { + RowData buildRow = entry.getKey(); + long count = entry.getValue(); + if (joinCondition.apply(probe, buildRow)) { + matched = true; + // Each emitted record uses a fresh JoinedRowData wrapper. + // Reusing a row object here is unsafe when subsequent collects mutate it. + for (long i = 0; i < count; i++) { + JoinedRowData out = new JoinedRowData(); + out.replace(probe, buildRow); + out.setRowKind(RowKind.INSERT); + collector.collect(out); + fanOut++; + } + } + } + if (!matched && isLeftOuterJoin) { + // No join match, emit a null-padded LEFT OUTER join result + JoinedRowData out = new JoinedRowData(); + out.replace(probe, nullPaddedBuild); + out.setRowKind(RowKind.INSERT); + collector.collect(out); + fanOut++; + } + if (!matched && !isLeftOuterJoin) { + numUnmatchedProbeRecords.inc(); + } + // Update fan-out statistics (fan-out = number of result rows emitted for this probe). + totalProbeJoins++; + totalJoinFanOut += fanOut; + if (fanOut > maxJoinFanOut) { + maxJoinFanOut = fanOut; + maxFanOutGauge.update(maxJoinFanOut); + } + avgFanOutGauge.update(((double) totalJoinFanOut) / totalProbeJoins); + } + + /** + * Applies the buffered build-side changes if the build-side watermark advanced since last + * buffer application. This ensures that we apply buffered changes atomically once their + * corresponding build-side WM is passed. + * + * @return the watermark tag of the buffer that is still in effect for this key after this call, + * or {@code null} if nothing is buffered. + */ + @Nullable + private Long applyBufferedChangesIfReady() throws Exception { + Long bufferedAt = bufferedAtWmState.value(); + if (bufferedAt == null) { + // Nothing buffered for this key. + return null; + } + if (currentBuildSideWm > bufferedAt) { + // The build-side wm advanced: apply the changes that are now due (their row-time has + // been reached by the build-side watermark) and keep any not-yet-due changes buffered. + applyDueBufferedChanges(); + return bufferedAtWmState.value(); + } else if (phase == Phase.JOIN && currentBuildSideWm == Long.MIN_VALUE) { + // JOIN-phase fallback: no build-side watermark seen by this subtask (recovery/rescale, + // or idle-timeout flip). Force-apply everything now, since there is no watermark to + // gate on and the next build watermark may not arrive for a long time. + applyBufferedChanges(); + return null; + } + return bufferedAt; + } + + /** + * Applies the buffered build-side changes that are due — i.e. whose row-time has been reached + * by the build-side watermark — in event-time order, leaving any not-yet-due changes untouched + * in state. + */ + private void applyDueBufferedChanges() throws Exception { + if (currentBuildSideWm == Long.MIN_VALUE) { + applyBufferedChanges(); + return; + } + // Scan keys (row-times) only; the values (rows) of not-yet-due buckets stay serialized. + List due = new ArrayList<>(); + boolean hasKept = false; + for (Long rowtime : buildChangeBuffer.keys()) { + if (rowtime <= currentBuildSideWm) { + due.add(rowtime); + } else { + hasKept = true; + if (sortedStateBackend) { + break; + } + } + } + // Ordered backends already yield due row-times ascending; otherwise sort them. + if (!sortedStateBackend) { + due.sort(Long::compareTo); + } + for (long rowtime : due) { + applyBufferedChangesAt(rowtime); + } + if (hasKept) { + bufferedAtWmState.update(currentBuildSideWm); + } else { + clearBuildChangeBuffer(); + } + } + + /** + * Force-applies all buffered build-side changes for the current key to {@code buildTableState} + * in event-time order, regardless of row-time, then clears the buffer. Used only when there is + * no build watermark to gate on (none seen yet). + */ + private void applyBufferedChanges() throws Exception { + List rowtimes = new ArrayList<>(); + for (Long rowtime : buildChangeBuffer.keys()) { + rowtimes.add(rowtime); + } + if (!sortedStateBackend) { + rowtimes.sort(Long::compareTo); + } + for (long rowtime : rowtimes) { + applyBufferedChangesAt(rowtime); + } + clearBuildChangeBuffer(); + } + + /** + * Applies all buffered build-side changes at a single row-time to {@code buildTableState}, then + * removes the row-time bucket. + */ + private void applyBufferedChangesAt(long rowtime) throws Exception { + for (RowData c : buildChangeBuffer.get(rowtime)) { + applyBuildChange(c); + } + buildChangeBuffer.remove(rowtime); + } + + /** Clears the per-key build-side change buffer and its watermark tag. */ + private void clearBuildChangeBuffer() { + buildChangeBuffer.clear(); + bufferedAtWmState.clear(); + } + + /** + * Clears all keyed state for the current key. Used by the TTL-eviction path. The {@code + * probeBuffer} is expected to be empty in JOIN phase (where eviction runs) but is cleared + * defensively; its best-effort gauge is intentionally not adjusted here. + */ + private void clearAllPerKeyState() { + buildTableState.clear(); + clearBuildChangeBuffer(); + clearProbeBuffer(); + ttlExpiryState.clear(); + } + + /** + * Apply a build-side change record directly to the build-table multi-set. + * + *

MUTATES the input row's {@link RowKind} to {@link RowKind#INSERT} to normalize the key + * used for {@code buildTableState} lookups. The caller must not rely on the original kind after + * this call returns. The mutation avoids a per-record copy on a hot path. + */ + private void applyBuildChange(RowData change) throws Exception { + RowKind changeType = change.getRowKind(); + change.setRowKind(RowKind.INSERT); + Long currentCnt = buildTableState.get(change); + if (changeType == RowKind.INSERT || changeType == RowKind.UPDATE_AFTER) { + buildTableState.put(change, currentCnt == null ? 1L : currentCnt + 1L); + } else { + if (currentCnt == null || currentCnt <= 0L) { + numUnmatchedBuildRetractions.inc(); + return; + } + if (currentCnt == 1L) { + buildTableState.remove(change); + } else { + buildTableState.put(change, currentCnt - 1L); + } + } + } + + /** If state TTL is configured, refreshes the state TTL timer if needed. */ + private void refreshStateTtl() throws Exception { + if (minStateTtlMs == 0) { + // Nothing to do when state TTL is not configured. + return; + } + // We register it at maxStateTtlMs to avoid rearming the timer on every access. + long now = getProcessingTimeService().getCurrentProcessingTime(); + long refreshThreshold = now + minStateTtlMs; + Long currentTtlTimer = ttlExpiryState.value(); + if (currentTtlTimer != null && currentTtlTimer >= refreshThreshold) { + // Existing timer still covers at least one full stateTtlMs — leave it in place. + return; + } + if (currentTtlTimer != null) { + timerService.deleteProcessingTimeTimer(VoidNamespace.INSTANCE, currentTtlTimer); + } + long newDeadline = now + maxStateTtlMs; + timerService.registerProcessingTimeTimer(VoidNamespace.INSTANCE, newDeadline); + ttlExpiryState.update(newDeadline); + } + + // -------------------------- accessors (testing) -------------------------- + + @VisibleForTesting + Phase getPhase() { + return phase; + } + + @VisibleForTesting + long getCurrentBuildSideWm() { + return currentBuildSideWm; + } + + @VisibleForTesting + long getCurrentProbeSideWm() { + return currentProbeSideWm; + } + + @Nullable + @VisibleForTesting + Long getFlipProcTime() { + return flipProcTime; + } + + @VisibleForTesting + boolean isIdleFlipTimerActive() { + return idleFlipTimer != null; + } + + @VisibleForTesting + MapState getBuildTableState() { + return buildTableState; + } + + @VisibleForTesting + MapState> getBuildChangeBuffer() { + return buildChangeBuffer; + } + + @VisibleForTesting + boolean isSortedStateBackend() { + return sortedStateBackend; + } + + @VisibleForTesting + ValueState getBufferedAtWmState() { + return bufferedAtWmState; + } + + @VisibleForTesting + MapState getProbeBuffer() { + return probeBuffer; + } + + @VisibleForTesting + ValueState getProbeBufferSeq() { + return probeBufferSeq; + } + + @VisibleForTesting + ValueState getTtlExpiryState() { + return ttlExpiryState; + } + + // -------------------------- accessors (metrics, testing) -------------------------- + + @VisibleForTesting + Counter getNumStateTtlEvictions() { + return numStateTtlEvictions; + } + + @VisibleForTesting + Counter getNumUnmatchedProbeRecords() { + return numUnmatchedProbeRecords; + } + + @VisibleForTesting + Counter getNumUnmatchedBuildRetractions() { + return numUnmatchedBuildRetractions; + } + + @VisibleForTesting + SimpleGauge getNumProbeSideRecordsBufferedGauge() { + return probeBufferedGauge; + } + + @VisibleForTesting + SimpleGauge getCurrentBuildSideWatermarkGauge() { + return buildWmGauge; + } + + @VisibleForTesting + SimpleGauge getCurrentProbeSideWatermarkGauge() { + return probeWmGauge; + } + + @VisibleForTesting + SimpleGauge getMaxJoinFanOutGauge() { + return maxFanOutGauge; + } + + @VisibleForTesting + SimpleGauge getAvgJoinFanOutGauge() { + return avgFanOutGauge; + } + + @VisibleForTesting + Gauge getCurrentPhaseGauge() { + return phaseGauge; + } +} diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/snapshot/LateralSnapshotJoinOperatorTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/snapshot/LateralSnapshotJoinOperatorTest.java new file mode 100644 index 00000000000000..cf4b8f6d93b3b7 --- /dev/null +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/join/snapshot/LateralSnapshotJoinOperatorTest.java @@ -0,0 +1,1963 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.join.snapshot; + +import org.apache.flink.api.java.functions.KeySelector; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.runtime.state.StateBackend; +import org.apache.flink.runtime.state.VoidNamespace; +import org.apache.flink.runtime.state.hashmap.HashMapStateBackend; +import org.apache.flink.state.rocksdb.EmbeddedRocksDBStateBackend; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus; +import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; +import org.apache.flink.streaming.util.KeyedTwoInputStreamOperatorTestHarness; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.data.writer.BinaryRowWriter; +import org.apache.flink.table.runtime.generated.GeneratedJoinCondition; +import org.apache.flink.table.runtime.operators.join.snapshot.LateralSnapshotJoinOperator.Phase; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.runtime.util.RowDataHarnessAssertor; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.VarCharType; +import org.apache.flink.types.RowKind; + +import org.junit.jupiter.api.Named; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Stream; + +import static org.apache.flink.table.runtime.operators.join.snapshot.LateralSnapshotJoinOperator.BUILD_CHANGE_BUFFER_STATE_NAME; +import static org.apache.flink.table.runtime.operators.join.snapshot.LateralSnapshotJoinOperator.BUILD_TABLE_STATE_NAME; +import static org.apache.flink.table.runtime.operators.join.snapshot.LateralSnapshotJoinOperator.PROBE_BUFFER_STATE_NAME; +import static org.apache.flink.table.runtime.util.StreamRecordUtils.deleteRecord; +import static org.apache.flink.table.runtime.util.StreamRecordUtils.insertRecord; +import static org.apache.flink.table.runtime.util.StreamRecordUtils.updateAfterRecord; +import static org.apache.flink.table.runtime.util.StreamRecordUtils.updateBeforeRecord; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; + +/** Harness tests for {@link LateralSnapshotJoinOperator}. */ +class LateralSnapshotJoinOperatorTest { + + // ----------------------------------------------------------------- Schema + + /** Probe row schema: (id BIGINT, key VARCHAR, val VARCHAR). */ + private static final InternalTypeInfo PROBE_TYPE = + InternalTypeInfo.ofFields( + new BigIntType(), VarCharType.STRING_TYPE, VarCharType.STRING_TYPE); + + /** Build row schema: (key VARCHAR, val VARCHAR, rt BIGINT). */ + private static final InternalTypeInfo BUILD_TYPE = + InternalTypeInfo.ofFields( + VarCharType.STRING_TYPE, VarCharType.STRING_TYPE, new BigIntType()); + + /** Joined output schema: probe ++ build = (id, pKey, pVal, bKey, bVal, bRt). */ + private static final LogicalType[] OUTPUT_TYPES = { + new BigIntType(), + VarCharType.STRING_TYPE, + VarCharType.STRING_TYPE, + VarCharType.STRING_TYPE, + VarCharType.STRING_TYPE, + new BigIntType() + }; + + /** Probe key column index (key VARCHAR is at field 1). */ + private static final int PROBE_KEY_IDX = 1; + + /** Build key column index (key VARCHAR is at field 0). */ + private static final int BUILD_KEY_IDX = 0; + + /** Build row-time column index (rt BIGINT is at field 2). */ + private static final int BUILD_RT_IDX = 2; + + private static final InternalTypeInfo KEY_TYPE = + InternalTypeInfo.ofFields(VarCharType.STRING_TYPE); + + private static final KeySelector PROBE_KEY_SELECTOR = + nullSafeStringKeySelector(PROBE_KEY_IDX); + private static final KeySelector BUILD_KEY_SELECTOR = + nullSafeStringKeySelector(BUILD_KEY_IDX); + + private static final RowDataHarnessAssertor JOINED_ASSERTOR = + new RowDataHarnessAssertor(OUTPUT_TYPES); + + // ----------------------------------------------------------------- Join conditions + + /** Trivial join condition that always matches (equality is enforced by partitioning). */ + private static final String ALWAYS_TRUE_JOIN_FUNC_CODE = + "public class LateralSnapshotJoinConditionStub extends " + + "org.apache.flink.api.common.functions.AbstractRichFunction " + + "implements org.apache.flink.table.runtime.generated.JoinCondition {\n" + + " public LateralSnapshotJoinConditionStub(Object[] reference) {}\n" + + " @Override public boolean apply(" + + " org.apache.flink.table.data.RowData in1," + + " org.apache.flink.table.data.RowData in2) { return true; }\n" + + "}\n"; + + /** + * Join condition that only matches when the probe value (field 2) equals {@code "match"}. Used + * to verify that the codegen'd condition is actually invoked at join time. + */ + private static final String MATCH_VAL_JOIN_FUNC_CODE = + "public class LateralSnapshotJoinConditionMatchVal extends " + + "org.apache.flink.api.common.functions.AbstractRichFunction " + + "implements org.apache.flink.table.runtime.generated.JoinCondition {\n" + + " public LateralSnapshotJoinConditionMatchVal(Object[] reference) {}\n" + + " @Override public boolean apply(" + + " org.apache.flink.table.data.RowData in1," + + " org.apache.flink.table.data.RowData in2) {\n" + + " if (in1.isNullAt(2)) { return false; }\n" + + " return \"match\".equals(in1.getString(2).toString());\n" + + " }\n" + + "}\n"; + + private static GeneratedJoinCondition newTrueCondition() { + return new GeneratedJoinCondition( + "LateralSnapshotJoinConditionStub", ALWAYS_TRUE_JOIN_FUNC_CODE, new Object[0]); + } + + private static GeneratedJoinCondition newMatchValCondition() { + return new GeneratedJoinCondition( + "LateralSnapshotJoinConditionMatchVal", MATCH_VAL_JOIN_FUNC_CODE, new Object[0]); + } + + // ----------------------------------------------------------------- Operator / harness + // factories + + private static LateralSnapshotJoinOperator newOperator( + boolean isLeftOuterJoin, + GeneratedJoinCondition joinCondition, + boolean[] filterNullKeys, + Long loadCompletedTime, + Long loadCompletedIdleTimeoutMs, + Long stateTtlMs) { + + return new LateralSnapshotJoinOperator( + isLeftOuterJoin, + PROBE_TYPE, + BUILD_TYPE, + BUILD_RT_IDX, + joinCondition, + filterNullKeys, + loadCompletedTime, + loadCompletedIdleTimeoutMs, + stateTtlMs); + } + + private static LateralSnapshotJoinOperator newOperator( + boolean isLeftOuterJoin, + Long loadCompletedTime, + Long loadCompletedIdleTimeoutMs, + Long stateTtlMs) { + + return newOperator( + isLeftOuterJoin, + newTrueCondition(), + new boolean[] {true}, + loadCompletedTime, + loadCompletedIdleTimeoutMs, + stateTtlMs); + } + + private static KeyedTwoInputStreamOperatorTestHarness + newHarness(LateralSnapshotJoinOperator op) throws Exception { + return new KeyedTwoInputStreamOperatorTestHarness<>( + op, PROBE_KEY_SELECTOR, BUILD_KEY_SELECTOR, KEY_TYPE); + } + + private static KeySelector nullSafeStringKeySelector(final int keyIdx) { + return value -> { + BinaryRowData ret = new BinaryRowData(1); + BinaryRowWriter writer = new BinaryRowWriter(ret); + if (value.isNullAt(keyIdx)) { + writer.setNullAt(0); + } else { + writer.writeString(0, value.getString(keyIdx)); + } + writer.complete(); + return ret; + }; + } + + // ----------------------------------------------------------------- LOAD phase + + @Test + void loadPhaseBuildSideChangeProcessing() throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + // During LOAD, build-side changes are buffered and later applied in event-time order. + // -D for a never-inserted (key, value) pair is defensively ignored. + addBuildChange(h, deleteRecord("k1", "ghost", 5L)); + // Two identical records (same key/val/row-time) → count(k1, v1, 20) = 2. + addBuildChange(h, insertRecord("k1", "v1", 20L)); + addBuildChange(h, insertRecord("k1", "v1", 20L)); + // Earlier row-time than v1, but arrives later. + addBuildChange(h, insertRecord("k1", "v2", 10L)); + + // Still LOAD: changes are buffered, nothing applied or emitted yet. + assertPhase(op, Phase.LOAD); + assertThat(h.getOutput()).isEmpty(); + assertThat(bufferedChangesForKey(h, op, "k1")).hasSize(4); + assertThat(buildTableKeys(h)).isEmpty(); + + // Advance the build watermark (still below the flip point) and access k1 again. The + // access drains the buffered batch in event-time order before buffering the new change. + addBuildWm(h, 50L); + addBuildChange(h, insertRecord("k1", "v3", 30L)); + + assertPhase(op, Phase.LOAD); + // First batch applied in row-time order: -D(ghost)@5 (ignored), +I(v2)@10, +I(v1)@20 + // ×2. + assertThat(buildTableKeys(h)).containsExactly("k1"); + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("v1", 2L, "v2", 1L)); + assertThat(bufferedChangesForKey(h, op, "k1")).hasSize(1); // v3 + + // Buffer an update pair with inverted after/before order, drained on the next access: + // -U removes one v1, +U adds v4. + addBuildChange(h, updateAfterRecord("k1", "v4", 20L)); + addBuildChange(h, updateBeforeRecord("k1", "v1", 20L)); + addBuildWm(h, 60L); + addBuildChange(h, insertRecord("k1", "v5", 70L)); // access drains v3, -U(v1), +U(v4) + + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf( + Map.of("v1", 1L, "v2", 1L, "v3", 1L, "v4", 1L)); + assertThat(bufferedChangesForKey(h, op, "k1")).hasSize(1); // v5 + } + } + + // --------------------------------------------- row-time gated build-change application + + @Test + void buildChangeDeferredUntilWatermarkReachesItsRowtime() throws Exception { + // Atomicity guarantee: a -U/+U pair sharing a row-time above the watermark must not be + // split when the build watermark advances between the two records (e.g. another build input + // channel raising the combined watermark). With row-time gating the retraction stays + // buffered until the watermark reaches the pair's row-time, so it is never applied alone. + LateralSnapshotJoinOperator op = newOperator(false, 1L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + addBuildWm(h, 50L); // flip to JOIN; build wm = 50 + assertPhase(op, Phase.JOIN); + + // -U buffered at tag 50, then the watermark advances to 150 (past the tag, below the + // pair's row-time 200), then +U arrives. The retraction must NOT be flushed alone. + addBuildChange(h, updateBeforeRecord("k1", "x", 200L)); + addBuildWm(h, 150L); + addBuildChange(h, updateAfterRecord("k1", "y", 200L)); + // additional insert to be kept in the buffer and applied later + addBuildChange(h, insertRecord("k1", "z", 300L)); + + // Both halves still buffered together; the retraction was not applied (it would have + // counted as an unmatched retraction against the empty build table). + assertThat(bufferedChangesForKey(h, op, "k1")).hasSize(3); + assertThat(buildTableForKey(h, op, "k1")).isEmpty(); + + // Once the watermark reaches the pair's row-time, both halves apply together. + addBuildWm(h, 200L); + addProbeRecord(h, 1L, "k1", "p"); + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("y", 1L)); + assertThat(bufferedChangesForKey(h, op, "k1")).hasSize(1); + assertThat(bufferedAtWmFor(h, op, "k1")).isEqualTo(200L); + + // apply the final insert. + addBuildWm(h, 400L); + addProbeRecord(h, 1L, "k1", "p2"); + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("y", 1L, "z", 1L)); + assertThat(bufferedChangesForKey(h, op, "k1")).isEmpty(); + assertThat(bufferedAtWmFor(h, op, "k1")).isNull(); + } + } + + @Test + void wmFlipAppliesOnlyDueBuildChanges() throws Exception { + // A watermark flip materializes build changes only up to loadCompletedTime; changes with a + // row-time above it stay buffered and are applied later as the watermark advances. + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + addBuildChange(h, insertRecord("k1", "due", 50L)); // row-time <= loadCompletedTime + addBuildChange(h, insertRecord("k1", "future", 200L)); // row-time > loadCompletedTime + addProbeRecord(h, 1L, "k1", "p"); // buffered during LOAD + addProbeWm(h, 80L); // lets the flip fire the per-key drain + + addBuildWm(h, 100L); // flip to JOIN at loadCompletedTime + assertPhase(op, Phase.JOIN); + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("due", 1L)); + assertThat(bufferedChangesForKey(h, op, "k1")).hasSize(1); // future still buffered + + // The future change is applied once the watermark reaches its row-time. + addBuildWm(h, 200L); + addProbeRecord(h, 2L, "k1", "p2"); + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("due", 1L, "future", 1L)); + assertThat(bufferedChangesForKey(h, op, "k1")).isEmpty(); + } + } + + @Test + void idleFlipWithoutWatermarkForceAppliesBufferedChanges() throws Exception { + // When the flip is triggered by the idle timeout and no build watermark was ever seen + // (currentBuildSideWm == MIN_VALUE), there is nothing to gate on, so buffered changes are + // force-applied regardless of row-time. Otherwise the build table would stay empty forever. + LateralSnapshotJoinOperator op = newOperator(false, 1000L, 100L, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.setProcessingTime(0); + h.open(); + addBuildChange(h, insertRecord("k1", "r", 200L)); // buffered during LOAD, no build wm + + h.setProcessingTime(100); // idle timeout fires the flip + assertPhase(op, Phase.JOIN); + + // No watermark to gate on: the change is force-applied on the next access despite its + // row-time being ahead of the (absent) watermark. + addProbeRecord(h, 1L, "k1", "p"); + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("r", 1L)); + assertThat(bufferedChangesForKey(h, op, "k1")).isEmpty(); + } + } + + @Test + void loadPhaseProbeSideInputProcessing() throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + + addBuildChange(h, insertRecord("k1", "build1", 1L)); + addBuildWm(h, 20L); + addProbeRecord(h, 1L, "k1", "probe-load-1"); + addProbeRecord(h, 2L, "k1", "probe-load-2"); + addProbeWm(h, 50L); + + assertPhase(op, Phase.LOAD); + // No output (records buffered, watermarks held back). + assertThat(h.getOutput()).isEmpty(); + + assertThat(op.getCurrentProbeSideWm()).isEqualTo(50L); + assertThat(probeBufferKeys(h)).containsExactly("k1"); + assertThat(probeBufferForKey(h, op, "k1")).hasSize(2); + // Build-side change was applied to the build table. + assertThat(bufferedChangesForKey(h, op, "k1")).isEmpty(); + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("build1", 1L)); + } + } + + // ----------------------------------------------------------------- Flip / transition + + @ParameterizedTest(name = "leftOuter={0}, wmFlip={1}") + @CsvSource({"true, true", "true, false", "false, true", "false, false"}) + void flipDrainsProbeBufferAndJoins(boolean leftOuter, boolean wmFlip) throws Exception { + LateralSnapshotJoinOperator op = newOperator(leftOuter, 100L, 200L, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.setProcessingTime(0); + h.open(); + // LOAD: build state and buffered probes (one without a matching build row). + // k1 is a multi-set: count(v1)=2, count(v2)=1; k2 is a single row. + addBuildChange(h, insertRecord("k1", "build-k1-v1", 1L)); + addBuildChange(h, insertRecord("k1", "build-k1-v1", 1L)); + addBuildChange(h, insertRecord("k1", "build-k1-v2", 1L)); + addBuildChange(h, insertRecord("k2", "build-k2", 1L)); + // LOAD: add probe-side records + addProbeRecord(h, 1L, "k1", "probe-1"); + addProbeRecord(h, 2L, "k2", "probe-2"); + addProbeRecord(h, 3L, "k2", "probe-3"); + addProbeRecord(h, 4L, "k3", "probe-no-match"); + addProbeWm(h, 80L); + assertPhase(op, Phase.LOAD); + + // assert that probe-side buffer is filled + assertThat(probeBufferForKey(h, op, "k1")).hasSize(1); + assertThat(probeBufferForKey(h, op, "k2")).hasSize(2); + assertThat(probeBufferForKey(h, op, "k3")).hasSize(1); + // idle-timeout timer is armed while in LOAD + assertThat(op.isIdleFlipTimerActive()).isTrue(); + + // trigger flip from LOAD to JOIN + if (wmFlip) { + // build WM crosses loadCompletedTime. + addBuildWm(h, 100L); + } else { + // proc-time exceeds idle timeout + h.setProcessingTime(200); + } + assertPhase(op, Phase.JOIN); + // idle-timeout timer is removed on flip (canceled by a WM flip, fired by an idle flip) + assertThat(op.isIdleFlipTimerActive()).isFalse(); + + // probe k1 joins k1's multi-set (count-respecting: 2x v1 + 1x v2 = three rows); + // probe k2 (2 rows) joins a single k2 build row; + // probe k3 doesn't have a matching build row. INNER: no output, LEFT OUTER: null-padded + assertWatermarkForwardedAfterRecords(h.getOutput(), 80L); + stripWatermarksAndStatusesFromOutput(h); + if (leftOuter) { + JOINED_ASSERTOR.shouldEmitAll( + h, + row(1L, "k1", "probe-1", "k1", "build-k1-v1", 1L), + row(1L, "k1", "probe-1", "k1", "build-k1-v1", 1L), + row(1L, "k1", "probe-1", "k1", "build-k1-v2", 1L), + row(2L, "k2", "probe-2", "k2", "build-k2", 1L), + row(3L, "k2", "probe-3", "k2", "build-k2", 1L), + row(4L, "k3", "probe-no-match", null, null, null)); + } else { + JOINED_ASSERTOR.shouldEmitAll( + h, + row(1L, "k1", "probe-1", "k1", "build-k1-v1", 1L), + row(1L, "k1", "probe-1", "k1", "build-k1-v1", 1L), + row(1L, "k1", "probe-1", "k1", "build-k1-v2", 1L), + row(2L, "k2", "probe-2", "k2", "build-k2", 1L), + row(3L, "k2", "probe-3", "k2", "build-k2", 1L)); + } + // Probe buffer drained on flip; build table preserved (k1 keeps multi-set counts). + assertThat(probeBufferKeys(h)).isEmpty(); + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf( + Map.of("build-k1-v1", 2L, "build-k1-v2", 1L)); + assertThat(buildTableForKey(h, op, "k2")) + .containsExactlyInAnyOrderEntriesOf(Map.of("build-k2", 1L)); + } + } + + @Test + void flipDrainsBufferedProbesInOrderViaPerRecordTimers() throws Exception { + // Each buffered probe gets its own event-time timer; at the flip they fire one per record + // (interruptibly) and join in insertion order. Afterwards the buffer and its timers are + // gone. + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + for (long id = 1; id <= 5; id++) { + addProbeRecord(h, id, "k1", "p" + id); + } + // One event-time timer registered per buffered probe (not one per key). + assertThat(h.numEventTimeTimers()).isEqualTo(5); + addProbeWm(h, 80L); // advances the probe wm so the flip fires the timers + assertPhase(op, Phase.LOAD); + assertThat(probeBufferForKey(h, op, "k1")).hasSize(5); + + addBuildWm(h, 100L); // flip -> per-record timers drain the buffered probes + assertPhase(op, Phase.JOIN); + + assertWatermarkForwardedAfterRecords(h.getOutput(), 80L); + stripWatermarksAndStatusesFromOutput(h); + List emittedProbeIds = + h.getOutput().stream() + .map(o -> ((RowData) ((StreamRecord) o).getValue()).getLong(0)) + .toList(); + assertThat(emittedProbeIds).containsExactly(1L, 2L, 3L, 4L, 5L); + JOINED_ASSERTOR.shouldEmitAll( + h, + row(1L, "k1", "p1", "k1", "v1", 1L), + row(2L, "k1", "p2", "k1", "v1", 1L), + row(3L, "k1", "p3", "k1", "v1", 1L), + row(4L, "k1", "p4", "k1", "v1", 1L), + row(5L, "k1", "p5", "k1", "v1", 1L)); + // Buffer fully drained: state cleared and no timers left. + assertThat(probeBufferKeys(h)).isEmpty(); + assertThat(h.numEventTimeTimers()).isZero(); + h.getOperator().setCurrentKey(stringKey("k1")); + assertThat(op.getProbeBufferSeq().value()).isNull(); + } + } + + @Test + void joinPhaseProbeIsBufferedWhileKeyStillDraining() throws Exception { + // Flip without a probe watermark leaves the LOAD-buffered probe in place. A new JOIN-phase + // probe for the same key must be appended behind it (not joined inline, not blocking), and + // the next probe watermark drains both in insertion order. + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addProbeRecord(h, 1L, "k1", "p1"); // buffered during LOAD + addBuildWm(h, 100L); // flip, no probe watermark seen + assertPhase(op, Phase.JOIN); + assertThat(probeBufferForKey(h, op, "k1")).hasSize(1); + assertThat(h.getOutput()).isEmpty(); + + // New JOIN probe for k1: buffer non-empty -> appended behind p1, nothing emitted yet. + addProbeRecord(h, 2L, "k1", "p2"); + assertThat(probeBufferForKey(h, op, "k1")).hasSize(2); + assertThat(h.getOutput()).isEmpty(); + assertThat(h.numEventTimeTimers()).isEqualTo(2); + + // The next probe watermark fires both timers; they drain in insertion order. + addProbeWm(h, 90L); + stripWatermarksAndStatusesFromOutput(h); + List emittedProbeIds = + h.getOutput().stream() + .map(o -> ((RowData) ((StreamRecord) o).getValue()).getLong(0)) + .toList(); + assertThat(emittedProbeIds).containsExactly(1L, 2L); + JOINED_ASSERTOR.shouldEmitAll( + h, row(1L, "k1", "p1", "k1", "v1", 1L), row(2L, "k1", "p2", "k1", "v1", 1L)); + assertThat(probeBufferForKey(h, op, "k1")).isEmpty(); + h.getOperator().setCurrentKey(stringKey("k1")); + assertThat(op.getProbeBufferSeq().value()).isNull(); + } + } + + @Test + void flipWithoutProbeWatermarkDrainsBufferedProbesOnNextProbeWatermark() throws Exception { + // No probe-side watermark before the flip (currentProbeSideWm == MIN_VALUE): the FLIP + // timers cannot fire at the flip, so the buffered probes stay buffered. The next + // probe-side watermark advances event time past the FLIP timer and drains them. + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addProbeRecord(h, 1L, "k1", "p1"); // buffered during LOAD, no probe watermark + assertPhase(op, Phase.LOAD); + assertThat(probeBufferForKey(h, op, "k1")).hasSize(1); + + addBuildWm(h, 100L); // flip, but no probe watermark yet + assertPhase(op, Phase.JOIN); + // The buffered probe is NOT drained at the flip and nothing is emitted yet. + assertThat(probeBufferForKey(h, op, "k1")).hasSize(1); + assertThat(h.getOutput()).isEmpty(); + + // The next probe-side watermark fires the FLIP timer and drains the buffered probe. + addProbeWm(h, 100L); + assertThat(probeBufferKeys(h)).isEmpty(); + stripWatermarksAndStatusesFromOutput(h); + JOINED_ASSERTOR.shouldEmitAll(h, row(1L, "k1", "p1", "k1", "v1", 1L)); + } + } + + @Test + void idleTimerRearmsOnBuildWatermark() throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 1000L, 100L, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.setProcessingTime(10); + h.open(); + h.setProcessingTime(60); + // Build WM advances → re-arm. + addBuildWm(h, 10L); + // Original idle deadline was 10+100=110. Re-armed to 60+100=160. + h.setProcessingTime(159); + assertPhase(op, Phase.LOAD); + h.setProcessingTime(160); + assertPhase(op, Phase.JOIN); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void flipJoiningInvokesCodeGeneratedJoinCondition(boolean leftOuter) throws Exception { + LateralSnapshotJoinOperator op = + newOperator( + leftOuter, newMatchValCondition(), new boolean[] {true}, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addProbeRecord(h, 1L, "k1", "match"); + addProbeRecord(h, 2L, "k1", "skip"); + addProbeWm(h, 120L); + addBuildWm(h, 100L); + + assertWatermarkForwardedAfterRecords(h.getOutput(), 120L); + stripWatermarksAndStatusesFromOutput(h); + if (leftOuter) { + JOINED_ASSERTOR.shouldEmitAll( + h, + row(1L, "k1", "match", "k1", "v1", 1L), + row(2L, "k1", "skip", null, null, null)); + } else { + JOINED_ASSERTOR.shouldEmitAll(h, row(1L, "k1", "match", "k1", "v1", 1L)); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void flipJoiningCompositeEquiKeys(boolean leftOuter) throws Exception { + // Probe schema (kA VARCHAR, kB VARCHAR, val VARCHAR); build schema additionally carries a + // row-time attribute (kA VARCHAR, kB VARCHAR, val VARCHAR, rt BIGINT) at index 3. + InternalTypeInfo probeType = + InternalTypeInfo.ofFields( + VarCharType.STRING_TYPE, VarCharType.STRING_TYPE, VarCharType.STRING_TYPE); + InternalTypeInfo buildType = + InternalTypeInfo.ofFields( + VarCharType.STRING_TYPE, + VarCharType.STRING_TYPE, + VarCharType.STRING_TYPE, + new BigIntType()); + final int buildRowtimeIndex = 3; + // Compose the composite key as a BinaryRowData with both key fields. + KeySelector selector = + value -> { + BinaryRowData ret = new BinaryRowData(2); + BinaryRowWriter writer = new BinaryRowWriter(ret); + if (value.isNullAt(0)) { + writer.setNullAt(0); + } else { + writer.writeString(0, value.getString(0)); + } + if (value.isNullAt(1)) { + writer.setNullAt(1); + } else { + writer.writeString(1, value.getString(1)); + } + writer.complete(); + return ret; + }; + InternalTypeInfo compositeKeyType = + InternalTypeInfo.ofFields(VarCharType.STRING_TYPE, VarCharType.STRING_TYPE); + + LateralSnapshotJoinOperator op = + new LateralSnapshotJoinOperator( + leftOuter, + probeType, + buildType, + buildRowtimeIndex, + newTrueCondition(), + new boolean[] {true, true}, + 100L, + null, + null); + + try (KeyedTwoInputStreamOperatorTestHarness h = + new KeyedTwoInputStreamOperatorTestHarness<>( + op, selector, selector, compositeKeyType)) { + h.open(); + // build rows: two identical rows for key a-1, a different row for key a-1, one row for + // key a-2. + addBuildChange(h, insertRecord("a", "1", "b-a-1-1", 1L)); + addBuildChange(h, insertRecord("a", "1", "b-a-1-1", 1L)); + addBuildChange(h, insertRecord("a", "1", "b-a-1-2", 1L)); + addBuildChange(h, insertRecord("a", "2", "b-a-2-1", 1L)); + // probes: matching composite, non-matching composite. + h.processElement1(insertRecord("a", "1", "p-a-1")); + h.processElement1(insertRecord("a", "9", "p-a-9")); + h.processElement1(insertRecord("b", "1", "p-b-1")); + h.processElement1(insertRecord("b", "9", "p-b-9")); + addProbeWm(h, 100L); + // flip to JOIN phase + addBuildWm(h, 100L); + + LogicalType[] outTypes = { + VarCharType.STRING_TYPE, + VarCharType.STRING_TYPE, + VarCharType.STRING_TYPE, + VarCharType.STRING_TYPE, + VarCharType.STRING_TYPE, + VarCharType.STRING_TYPE, + new BigIntType() + }; + RowDataHarnessAssertor compositeAssertor = new RowDataHarnessAssertor(outTypes); + + stripWatermarksAndStatusesFromOutput(h); + if (leftOuter) { + compositeAssertor.shouldEmitAll( + h, + compKeyRow("a", "1", "p-a-1", "a", "1", "b-a-1-1", 1L), + compKeyRow("a", "1", "p-a-1", "a", "1", "b-a-1-1", 1L), + compKeyRow("a", "1", "p-a-1", "a", "1", "b-a-1-2", 1L), + compKeyRow("a", "9", "p-a-9", null, null, null, null), + compKeyRow("b", "1", "p-b-1", null, null, null, null), + compKeyRow("b", "9", "p-b-9", null, null, null, null)); + } else { + compositeAssertor.shouldEmitAll( + h, + compKeyRow("a", "1", "p-a-1", "a", "1", "b-a-1-1", 1L), + compKeyRow("a", "1", "p-a-1", "a", "1", "b-a-1-1", 1L), + compKeyRow("a", "1", "p-a-1", "a", "1", "b-a-1-2", 1L)); + } + } + } + + // ----------------------------------------------------------------- JOIN phase + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void joinPhaseImmediateInnerJoin(boolean leftOuter) throws Exception { + LateralSnapshotJoinOperator op = newOperator(leftOuter, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildWm(h, 100L); + assertPhase(op, Phase.JOIN); + + // First probe WM + addProbeWm(h, 10L); + assertWatermarkForwardedAfterRecords(h.getOutput(), 10L); + + // First probe — joined immediately. + addProbeRecord(h, 1L, "k1", "probe-immediate-1"); + stripWatermarksAndStatusesFromOutput(h); + JOINED_ASSERTOR.shouldEmitAll(h, row(1L, "k1", "probe-immediate-1", "k1", "v1", 1L)); + + // Another probe WM + addProbeWm(h, 20L); + assertWatermarkForwardedAfterRecords(h.getOutput(), 20L); + + // Second probe for same key — joined immediately. + addProbeRecord(h, 2L, "k1", "probe-immediate-2"); + stripWatermarksAndStatusesFromOutput(h); + JOINED_ASSERTOR.shouldEmitAll(h, row(2L, "k1", "probe-immediate-2", "k1", "v1", 1L)); + + // Probe for non-existent key — no output (INNER). + addProbeRecord(h, 3L, "k2", "probe-no-match"); + stripWatermarksAndStatusesFromOutput(h); + if (leftOuter) { + JOINED_ASSERTOR.shouldEmitAll(h, row(3L, "k2", "probe-no-match", null, null, null)); + } else { + assertThat(h.extractOutputStreamRecords()).isEmpty(); + } + + // one more probe WM + addProbeWm(h, 30L); + assertWatermarkForwardedAfterRecords(h.getOutput(), 30L); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void joinPhaseBuildSideChangeApplication(boolean appliedByBuild) throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + // Two identical records for k1 (count 2) at row-time 1, buffered during LOAD. + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildWm(h, 100L); + assertPhase(op, Phase.JOIN); + + // Buffer four changes for k1 + addBuildChange(h, deleteRecord("k1", "v1", 1L)); + addBuildChange(h, insertRecord("k1", "v2", 102L)); + addBuildChange(h, updateBeforeRecord("k1", "v1", 1L)); + addBuildChange(h, updateAfterRecord("k1", "v3", 103L)); + // Buffer one change for k2. + addBuildChange(h, insertRecord("k2", "v1", 101L)); + + // assert number of buffered changes + assertThat(op.getCurrentBuildSideWm()).isEqualTo(100L); + assertThat(bufferedAtWmFor(h, op, "k1")).isEqualTo(100L); + assertThat(bufferedChangesForKey(h, op, "k1")).hasSize(4); + assertThat(bufferedAtWmFor(h, op, "k2")).isEqualTo(100L); + assertThat(bufferedChangesForKey(h, op, "k2")).hasSize(1); + assertThat(buildTableForKey(h, op, "k1")).isEqualTo(Map.of("v1", 2L)); + + // Probe record with no build WM advance - changes are not applied yet + addProbeRecord(h, 1L, "k1", "p-1"); + JOINED_ASSERTOR.shouldEmitAll( + h, row(1L, "k1", "p-1", "k1", "v1", 1L), row(1L, "k1", "p-1", "k1", "v1", 1L)); + + // increment build-side WM + addBuildWm(h, 110L); + + // assert that all changes are still buffered + assertThat(op.getCurrentBuildSideWm()).isEqualTo(110L); + assertThat(bufferedAtWmFor(h, op, "k1")).isEqualTo(100L); + assertThat(bufferedChangesForKey(h, op, "k1")).hasSize(4); + assertThat(bufferedAtWmFor(h, op, "k2")).isEqualTo(100L); + assertThat(bufferedChangesForKey(h, op, "k2")).hasSize(1); + + // trigger application of k1 changes by build or probe-side input + if (!appliedByBuild) { + addBuildChange(h, insertRecord("k1", "v4", 111L)); + // assert that changes have been applied and removed from buffer + // the triggering change is appended to the buffer + assertThat(bufferedAtWmFor(h, op, "k1")).isEqualTo(110L); + assertThat(bufferedChangesForKey(h, op, "k1")).hasSize(1); + assertThat(buildTableForKey(h, op, "k1")).isEqualTo(Map.of("v2", 1L, "v3", 1L)); + // assert empty output + assertThat(h.getOutput()).isEmpty(); + } else { + addProbeRecord(h, 2L, "k1", "p-2"); + // assert that changes have been applied and removed from buffer + assertThat(bufferedAtWmFor(h, op, "k1")).isNull(); + assertThat(bufferedChangesForKey(h, op, "k1")).isEmpty(); + assertThat(buildTableForKey(h, op, "k1")) + .isEqualTo(Map.of("v2", 1L, "v3", 1L)); // / + // assert join results (v2 carries row-time 2, v3 carries row-time 1) + JOINED_ASSERTOR.shouldEmitAll( + h, + row(2L, "k1", "p-2", "k1", "v2", 102L), + row(2L, "k1", "p-2", "k1", "v3", 103L)); + } + + // assert that k2 change is still buffered + assertThat(bufferedAtWmFor(h, op, "k2")).isEqualTo(100L); + assertThat(bufferedChangesForKey(h, op, "k2")).hasSize(1); + // apply k2 change and join + addProbeRecord(h, 3L, "k2", "p-3"); + assertThat(bufferedAtWmFor(h, op, "k2")).isNull(); + assertThat(bufferedChangesForKey(h, op, "k2")).isEmpty(); + assertThat(buildTableForKey(h, op, "k2")).isEqualTo(Map.of("v1", 1L)); + // assert join results + JOINED_ASSERTOR.shouldEmitAll(h, row(3L, "k2", "p-3", "k2", "v1", 101L)); + } + } + + @Test + void joinPhaseWmForwardingLogic() throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildWm(h, 100L); // flip + assertPhase(op, Phase.JOIN); + + // Build-side WMs after flip are not forwarded. + addBuildWm(h, 200L); + assertThat(extractWatermarks(h.getOutput())).isEmpty(); + + // Probe-side WMs in JOIN are forwarded. + addProbeWm(h, 150L); + assertThat(extractWatermarks(h.getOutput())).containsExactly(new Watermark(150)); + h.getOutput().clear(); + + // another build-side WM + addBuildWm(h, 300L); + assertThat(extractWatermarks(h.getOutput())).isEmpty(); + + addProbeWm(h, 250L); + assertThat(extractWatermarks(h.getOutput())).containsExactly(new Watermark(250)); + } + } + + /** + * Build-side changes sharing a row-time are applied in insertion order, which mirrors the + * upstream emit order (retraction before its accumulation). + */ + @Test + void joinPhaseAppliesEqualRowTimeChangesInInsertionOrder() throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + // Establish {vOld:1} for k1 at row-time 10. + addBuildChange(h, insertRecord("k1", "vOld", 10L)); + addBuildWm(h, 100L); + assertPhase(op, Phase.JOIN); + addProbeRecord(h, 0L, "k1", "warmup"); // applies +I(vOld) + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("vOld", 1L)); + + // Add two update change pairs. The second pair updates the result of the first update + addBuildChange(h, updateBeforeRecord("k1", "vOld", 10L)); + addBuildChange(h, updateAfterRecord("k1", "vNew1", 10L)); + addBuildChange(h, updateBeforeRecord("k1", "vNew1", 10L)); + addBuildChange(h, updateAfterRecord("k1", "vNew2", 10L)); + addBuildWm(h, 200L); + addProbeRecord(h, 1L, "k1", "p1"); + + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("vNew2", 1L)); + stripWatermarksAndStatusesFromOutput(h); + JOINED_ASSERTOR.shouldEmitAll( + h, + row(0L, "k1", "warmup", "k1", "vOld", 10L), + row(1L, "k1", "p1", "k1", "vNew2", 10L)); + } + } + + private static Stream> stateBackends() { + return Stream.of( + Named.of("heap", new HashMapStateBackend()), + Named.of("rocksdb", new EmbeddedRocksDBStateBackend())); + } + + /** + * Exercises the state-backend-dependent build-change drain on both an unordered (heap) and a + * sorted (RocksDB) backend. Build changes are buffered out of row-time order; when the + * watermark advances only partway, the due changes must be applied and the not-yet-due ones + * left buffered. On heap this uses the collect-and-sort path; on RocksDB the sorted-iteration + + * early-break path — neither may skip a due row-time nor apply a not-yet-due one. + */ + @ParameterizedTest(name = "backend={0}") + @MethodSource("stateBackends") + void drainsDueBuildChangesByRowTimeOrder(StateBackend backend) throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 1L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.setStateBackend(backend); + h.open(); + // Only RocksDB is a sorted backend, which selects the skip-sort + early-break path. + assertThat(op.isSortedStateBackend()) + .isEqualTo(backend instanceof EmbeddedRocksDBStateBackend); + + addBuildWm(h, 5L); // flip to JOIN (loadCompletedTime = 1); build wm = 5 + assertPhase(op, Phase.JOIN); + + // Buffer inserts for k1 out of row-time order (all above the current wm of 5). + addBuildChange(h, insertRecord("k1", "v50", 50L)); + addBuildChange(h, insertRecord("k1", "v10", 10L)); + addBuildChange(h, insertRecord("k1", "v40", 40L)); + addBuildChange(h, insertRecord("k1", "v20", 20L)); + addBuildChange(h, insertRecord("k1", "v30", 30L)); + assertThat(bufferedChangesForKey(h, op, "k1")).hasSize(5); + assertThat(buildTableForKey(h, op, "k1")).isEmpty(); + + // Advance the build wm to 30 and touch k1: row-times 10/20/30 are due, 40/50 are not. + addBuildWm(h, 30L); + addProbeRecord(h, 1L, "k1", "p1"); + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("v10", 1L, "v20", 1L, "v30", 1L)); + assertThat(bufferedChangesForKey(h, op, "k1")).hasSize(2); // v40@40, v50@50 + assertThat(bufferedAtWmFor(h, op, "k1")).isEqualTo(30); + + // Advancing past their row-times applies the remaining changes. + addBuildWm(h, 100L); + addProbeRecord(h, 2L, "k1", "p2"); + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf( + Map.of("v10", 1L, "v20", 1L, "v30", 1L, "v40", 1L, "v50", 1L)); + assertThat(bufferedChangesForKey(h, op, "k1")).isEmpty(); + assertThat(bufferedAtWmFor(h, op, "k1")).isNull(); + } + } + + @Test + void joinPhaseBufferedUpdatePairIsVisibleAtomicallyAfterWatermark() throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + addBuildChange(h, insertRecord("k1", "vOld", 10L)); + addBuildWm(h, 100L); // flip; build WM = 100 + addProbeRecord(h, 1L, "k1", "p-init"); // drains +I(vOld); joins vOld + + // Buffer a -U/+U pair (tagged at WM 100). A probe joining before the WM advances must + // still see the old value (the pair is not yet applied). + addBuildChange(h, updateBeforeRecord("k1", "vOld", 10L)); + addBuildChange(h, updateAfterRecord("k1", "vNew", 20L)); + addProbeRecord(h, 2L, "k1", "p-mid"); // sees vOld + + // After the build WM advances past the tag, the pair is applied atomically. + addBuildWm(h, 110L); + addProbeRecord(h, 3L, "k1", "p-after"); // sees vNew + + stripWatermarksAndStatusesFromOutput(h); + JOINED_ASSERTOR.shouldEmitAll( + h, + row(1L, "k1", "p-init", "k1", "vOld", 10L), + row(2L, "k1", "p-mid", "k1", "vOld", 10L), + row(3L, "k1", "p-after", "k1", "vNew", 20L)); + } + } + + // ----------------------------------------------------------------- NULL keys + + @ParameterizedTest + @CsvSource({"true, true", "true, false", "false, true", "false, false"}) + void joinRespectsNullKeysFilter(boolean leftOuter, boolean filterNullKey) throws Exception { + LateralSnapshotJoinOperator op = + newOperator( + leftOuter, + newTrueCondition(), + new boolean[] {filterNullKey}, + 100L, + null, + null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + addBuildChange(h, insertRecord(null, "v_null", 1L)); + addProbeRecord(h, 1L, null, "p_null"); + addProbeWm(h, 100L); + addBuildWm(h, 100L); + stripWatermarksAndStatusesFromOutput(h); + + // test joining during LOAD -> JOIN transition + if (filterNullKey) { + if (leftOuter) { + JOINED_ASSERTOR.shouldEmitAll(h, row(1L, null, "p_null", null, null, null)); + } else { + assertThat(h.getOutput()).isEmpty(); + } + } else { + JOINED_ASSERTOR.shouldEmitAll(h, row(1L, null, "p_null", null, "v_null", 1L)); + } + + // test joining in JOIN phase + addProbeRecord(h, 2L, null, "p_null"); + if (filterNullKey) { + if (leftOuter) { + JOINED_ASSERTOR.shouldEmitAll(h, row(2L, null, "p_null", null, null, null)); + } else { + assertThat(h.getOutput()).isEmpty(); + } + } else { + JOINED_ASSERTOR.shouldEmitAll(h, row(2L, null, "p_null", null, "v_null", 1L)); + } + } + } + + // ----------------------------------------------------------------- Watermark status + + @Test + void buildSideWmAndWmStatusForwarding() throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + // probe-side WM + addProbeWm(h, 70L); + + // LOAD phase: build-side becomes idle then active again. WM is advanced. + h.processWatermarkStatus2(WatermarkStatus.IDLE); + h.processWatermarkStatus2(WatermarkStatus.ACTIVE); + addBuildWm(h, 50L); + // assert that no WMs or WM statuses are emitted in LOAD. + assertThat(h.getOutput()).isEmpty(); + + // Drive into JOIN + addBuildWm(h, 100L); + assertPhase(op, Phase.JOIN); + + // assert that only the probe-side WM was forwarded after the transition + assertThat(h.getOutput()).containsExactly(new Watermark(70L)); + h.getOutput().clear(); + + // JOIN phase: build-side becomes idle then active again. WM is advanced + h.processWatermarkStatus2(WatermarkStatus.IDLE); + h.processWatermarkStatus2(WatermarkStatus.ACTIVE); + addBuildWm(h, 200L); + + // No records, WMs, or WM statuses emitted in JOIN. + assertThat(h.getOutput()).isEmpty(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void probeSideWmAndWmStatusForwarding(boolean probeIdleDuringLoad) throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + addProbeWm(h, 25L); + addProbeWm(h, 50L); + // Probe-side WM statuses received during LOAD. + h.processWatermarkStatus1(WatermarkStatus.IDLE); + if (!probeIdleDuringLoad) { + h.processWatermarkStatus1(WatermarkStatus.ACTIVE); + } + // Absorbed during LOAD — nothing emitted. + assertThat(extractWatermarks(h.getOutput())).isEmpty(); + assertThat(extractWatermarkStatuses(h.getOutput())).isEmpty(); + + // Flip to JOIN phase — last probe WM emitted. Probe-side idleness is intentionally not + // propagated at the flip (all parallel instances are uniformly idle), so regardless of + // whether the probe was idle during LOAD, only the watermark is emitted here. + addBuildWm(h, 100L); + assertPhase(op, Phase.JOIN); + + assertThat(extractWatermarkStatuses(h.getOutput())).isEmpty(); + assertThat(extractWatermarks(h.getOutput())).containsExactly(new Watermark(50)); + h.getOutput().clear(); + + // WMs and WM status updates received during JOIN are forwarded. + addProbeWm(h, 150L); + assertThat(h.getOutput()).containsExactly(new Watermark(150)); + h.getOutput().clear(); + // set probe-side to idle + h.processWatermarkStatus1(WatermarkStatus.IDLE); + assertThat(h.getOutput()).containsExactly(WatermarkStatus.IDLE); + h.getOutput().clear(); + // set probe-side to active + h.processWatermarkStatus1(WatermarkStatus.ACTIVE); + assertThat(h.getOutput()).containsExactly(WatermarkStatus.ACTIVE); + h.getOutput().clear(); + // emit another watermark + addProbeWm(h, 200L); + assertThat(h.getOutput()).containsExactly(new Watermark(200)); + h.getOutput().clear(); + } + } + + // ----------------------------------------------------------------- State TTL + + @Test + void stateTtlRefreshesOnAccessAndEvictsInactiveKeys() throws Exception { + // stateTtlMs = 100. Timers are registered at 1.5 × stateTtlMs, so the deadline for access + // at t=0 is 150. + LateralSnapshotJoinOperator op = newOperator(false, 50L, null, 100L); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.setProcessingTime(0); + h.open(); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildChange(h, insertRecord("k2", "v1", 1L)); + addBuildChange(h, insertRecord("k3", "v1", 1L)); + addBuildChange(h, insertRecord("k4", "v1", 1L)); + + // State is NOT evicted during LOAD even after the deadline passes (TTL fires are + // rescheduled past the LOAD phase). + h.setProcessingTime(200); + assertThat(buildStateKeys(h)).containsExactly("k1", "k2", "k3", "k4"); + + // flip to JOIN + addBuildWm(h, 50L); + assertPhase(op, Phase.JOIN); + + // Touch k1, k2, k3 to reset their TTL in JOIN; leave k4 alone. + h.setProcessingTime(275); + assertThat(buildStateKeys(h)).containsExactly("k1", "k2", "k3", "k4"); + addBuildChange(h, insertRecord("k1", "v2", 2L)); + addBuildChange(h, insertRecord("k2", "v2", 2L)); + addBuildChange(h, insertRecord("k3", "v2", 2L)); + + // k4 evicted: it wasn't accessed since proc-time (0) and we flipped to JOIN at (200) + h.setProcessingTime(350); + assertThat(buildStateKeys(h)).containsExactly("k1", "k2", "k3"); + + // Update build state for k1 and k2; leave k3 alone + addBuildWm(h, 60L); + h.setProcessingTime(400); + assertThat(buildStateKeys(h)).containsExactly("k1", "k2", "k3"); + addBuildChange(h, insertRecord("k1", "v3", 3L)); + addBuildChange(h, insertRecord("k2", "v3", 3L)); + + // k3 evicted, not accessed since (275) + addBuildWm(h, 70L); + h.setProcessingTime(475); + assertThat(buildStateKeys(h)).containsExactly("k1", "k2"); + // Access k1 from probe side to reset its TTL again; leave k2 alone + addProbeRecord(h, 1L, "k1", "p1"); + + // k2 evicted, not accessed since (400) + h.setProcessingTime(550); + assertThat(buildStateKeys(h)).containsExactly("k1"); + + // k1 finally evicted. + h.setProcessingTime(700); + assertThat(buildStateKeys(h)).isEmpty(); + + // The probe joined against the (k1) multi-set state at proc-time 475 — entries v1, + // v2, v3. + stripWatermarksAndStatusesFromOutput(h); + JOINED_ASSERTOR.shouldEmitAll( + h, + row(1L, "k1", "p1", "k1", "v1", 1L), + row(1L, "k1", "p1", "k1", "v2", 2L), + row(1L, "k1", "p1", "k1", "v3", 3L)); + } + } + + @Test + void stateTtlClearsAllPerKeyState() throws Exception { + // stateTtlMs = 100. Timers are registered at 1.5 × stateTtlMs. + LateralSnapshotJoinOperator op = newOperator(false, 50L, null, 100L); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.setProcessingTime(0); + h.open(); + // Buffered during LOAD + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildWm(h, 50L); // flip to JOIN + assertPhase(op, Phase.JOIN); + + // Buffer a change in JOIN that does NOT drain (no further WM advance / access). This + // populates the change buffer and the buffered-at tag while the build table keeps {v1}. + addBuildChange(h, insertRecord("k1", "v2", 2L)); + assertThat(buildTableForKey(h, op, "k1")).isEqualTo(Map.of("v1", 1L)); + assertThat(bufferedChangesForKey(h, op, "k1")).hasSize(1); + assertThat(bufferedAtWmFor(h, op, "k1")).isEqualTo(50L); + assertThat(ttlExpiryFor(h, op, "k1")).isEqualTo(150L); + + // Fire the TTL timer (well past the eviction deadline). + h.setProcessingTime(1000); + + // Every per-key state object is cleared, not just the build table. + assertThat(buildTableForKey(h, op, "k1")).isEmpty(); + assertThat(bufferedChangesForKey(h, op, "k1")).isEmpty(); + assertThat(bufferedAtWmFor(h, op, "k1")).isNull(); + assertThat(probeBufferForKey(h, op, "k1")).isEmpty(); + assertThat(ttlExpiryFor(h, op, "k1")).isNull(); + } + } + + @Test + void stateTtlRestoreResetsFlipProcTime() throws Exception { + // stateTtlMs = 50. Build write at t=0 arms TTL at 75. Flip at t=0 → flipProcTime=0. + // Original grace ends at 0+50=50. + LateralSnapshotJoinOperator op1 = newOperator(false, 100L, null, 50L); + OperatorSubtaskState state; + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op1)) { + h.setProcessingTime(0); + h.open(); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildWm(h, 100L); + assertThat(op1.getFlipProcTime()).isEqualTo(0L); + state = h.snapshot(0L, 0L); + } + + // Restart at t=30 — flipProcTime is re-anchored to 30; new grace ends at 30+50=80. + LateralSnapshotJoinOperator op2 = newOperator(false, 100L, null, 50L); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op2)) { + h.setProcessingTime(30); + h.initializeState(state); + h.open(); + assertThat(op2.getFlipProcTime()).isEqualTo(30L); + + // At t=75 the recovered TTL timer fires. Grace check: now=75 < flipProcTime+stateTtlMs + // = 80 → reschedule rather than evict. + h.setProcessingTime(75); + + // k1 still present because the grace window was re-anchored. + assertThat(buildStateKeys(h)).containsExactly("k1"); + + // advance proc-time to 105 to evict state + h.setProcessingTime(105); + assertThat(buildStateKeys(h)).isEmpty(); + } + } + + // ----------------------------------------------------------------- Snapshot / restore + + @Test + void restoreFromLoadPhaseSnapshot() throws Exception { + LateralSnapshotJoinOperator op1 = newOperator(false, 100L, null, null); + OperatorSubtaskState state; + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op1)) { + h.open(); + // Build-side multi-set with a duplicate count, plus a buffered + // probe. + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildChange(h, insertRecord("k1", "v2", 2L)); + addBuildWm(h, 10L); + addBuildChange(h, insertRecord("k1", "v3", 3L)); + addProbeRecord(h, 1L, "k1", "p1"); + assertPhase(op1, Phase.LOAD); + assertThat(probeBufferKeys(h)).containsExactly("k1"); + state = h.snapshot(0L, 0L); + } + + LateralSnapshotJoinOperator op2 = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op2)) { + h.initializeState(state); + h.open(); + assertPhase(op2, Phase.LOAD); + // Buffered probe and build-table multi-set (with counts) preserved across restore. + assertThat(probeBufferKeys(h)).containsExactly("k1"); + assertThat(buildTableForKey(h, op2, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("v1", 2L, "v2", 1L)); + assertThat(bufferedChangesForKey(h, op2, "k1")).hasSize(1); + // The buffer-count gauge is a best-effort in-memory tally that is not restored from + // state, so it reads 0 after restore even though one probe is buffered for k1. + assertThat(op2.getNumProbeSideRecordsBufferedGauge().getValue()).isEqualTo(0L); + + // Trigger flip; the buffered probe is joined post-restore, count-respecting against the + // restored multi-set (2× v1 + 1× v2 = three rows). + addProbeWm(h, 50L); + addBuildWm(h, 100L); + assertPhase(op2, Phase.JOIN); + assertThat(probeBufferKeys(h)).isEmpty(); + // The probe-buffer gauge stays 0 (best-effort, not restored from state). + assertThat(op2.getNumProbeSideRecordsBufferedGauge().getValue()).isEqualTo(0L); + assertThat(buildTableForKey(h, op2, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("v1", 2L, "v2", 1L, "v3", 1L)); + + stripWatermarksAndStatusesFromOutput(h); + JOINED_ASSERTOR.shouldEmitAll( + h, + row(1L, "k1", "p1", "k1", "v1", 1L), + row(1L, "k1", "p1", "k1", "v1", 1L), + row(1L, "k1", "p1", "k1", "v2", 2L), + row(1L, "k1", "p1", "k1", "v3", 3L)); + } + } + + @Test + void restoreFromLoadPhaseWithMultipleBufferedProbesPerKey() throws Exception { + // Recovery with several per-record flip timers for one key: the buffered probes, their + // sequence counter, and one event-time timer per probe must all survive the snapshot and, + // after restore, drain in insertion order when the flip fires them. + LateralSnapshotJoinOperator op1 = newOperator(false, 100L, null, null); + OperatorSubtaskState state; + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op1)) { + h.open(); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addProbeRecord(h, 1L, "k1", "p1"); + addProbeRecord(h, 2L, "k1", "p2"); + addProbeRecord(h, 3L, "k1", "p3"); + assertPhase(op1, Phase.LOAD); + assertThat(probeBufferForKey(h, op1, "k1")).hasSize(3); + assertThat(h.numEventTimeTimers()).isEqualTo(3); // one timer per buffered probe + state = h.snapshot(0L, 0L); + } + + LateralSnapshotJoinOperator op2 = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op2)) { + h.initializeState(state); + h.open(); + assertPhase(op2, Phase.LOAD); + // Buffer and all three per-record timers restored. + assertThat(probeBufferForKey(h, op2, "k1")).hasSize(3); + assertThat(h.numEventTimeTimers()).isEqualTo(3); + + // Flip: the restored timers fire and drain the buffered probes in insertion order. + addProbeWm(h, 50L); + addBuildWm(h, 100L); + assertPhase(op2, Phase.JOIN); + + stripWatermarksAndStatusesFromOutput(h); + List emittedProbeIds = + h.getOutput().stream() + .map(o -> ((RowData) ((StreamRecord) o).getValue()).getLong(0)) + .toList(); + assertThat(emittedProbeIds).containsExactly(1L, 2L, 3L); + JOINED_ASSERTOR.shouldEmitAll( + h, + row(1L, "k1", "p1", "k1", "v1", 1L), + row(2L, "k1", "p2", "k1", "v1", 1L), + row(3L, "k1", "p3", "k1", "v1", 1L)); + // Buffer, sequence counter, and timers all cleared after the drain. + assertThat(probeBufferKeys(h)).isEmpty(); + assertThat(h.numEventTimeTimers()).isZero(); + h.getOperator().setCurrentKey(stringKey("k1")); + assertThat(op2.getProbeBufferSeq().value()).isNull(); + } + } + + @Test + void restoreFromMixedPhaseSnapshot() throws Exception { + // Subtask A: drive into JOIN with a buffered change for k1. + LateralSnapshotJoinOperator opA = newOperator(false, 100L, null, null); + OperatorSubtaskState stateA; + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(opA)) { + h.open(); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildWm(h, 100L); + assertPhase(opA, Phase.JOIN); + // Buffer a build-side change + addBuildChange(h, insertRecord("k1", "v1", 1L)); + stateA = h.snapshot(0L, 0L); + } + + // Subtask B: stay in LOAD (no flip-triggering build WM). + LateralSnapshotJoinOperator opB = newOperator(false, 100L, null, null); + OperatorSubtaskState stateB; + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(opB)) { + h.open(); + addProbeRecord(h, 1L, "k2", "p1"); + assertPhase(opB, Phase.LOAD); + stateB = h.snapshot(0L, 0L); + } + + OperatorSubtaskState combined = + AbstractStreamOperatorTestHarness.repackageState(stateA, stateB); + + // Restore the combined state — phase must be LOAD because some subtask was LOAD. + LateralSnapshotJoinOperator opC = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(opC)) { + h.initializeState(combined); + h.open(); + assertPhase(opC, Phase.LOAD); + + // assert state: A's build table ({v1:1}) plus A's buffered change and B's probe. + assertThat(buildTableKeys(h)).containsExactly("k1"); + assertThat(buildTableForKey(h, opC, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("v1", 1L)); + assertThat(bufferedChangesForKey(h, opC, "k1")).hasSize(1); + assertThat(probeBufferKeys(h)).containsExactly("k2"); + + // During LOAD the current build WM is still MIN_VALUE, so the recovered buffer (tagged + // at the pre-restore WM) is not drained; the new change is appended to it. + addBuildChange(h, insertRecord("k1", "v2", 2L)); + assertThat(bufferedChangesForKey(h, opC, "k1")).hasSize(2); + assertThat(buildTableForKey(h, opC, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("v1", 1L)); + + // Trigger flip + addBuildWm(h, 100L); + assertPhase(opC, Phase.JOIN); + + // Access k1: the buffered changes drain in event-time order, then the probe joins. + addProbeRecord(h, 1L, "k1", "p1"); + assertThat(bufferedChangesForKey(h, opC, "k1")).isEmpty(); + assertThat(buildTableForKey(h, opC, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("v1", 2L, "v2", 1L)); + stripWatermarksAndStatusesFromOutput(h); + JOINED_ASSERTOR.shouldEmitAll( + h, + row(1L, "k1", "p1", "k1", "v1", 1L), + row(1L, "k1", "p1", "k1", "v1", 1L), + row(1L, "k1", "p1", "k1", "v2", 2L)); + } + } + + @Test + void restoreFromJoinPhaseSnapshot() throws Exception { + LateralSnapshotJoinOperator op1 = newOperator(false, 100L, null, null); + OperatorSubtaskState state; + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op1)) { + h.open(); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildChange(h, insertRecord("k2", "v1", 1L)); + addBuildWm(h, 100L); + assertPhase(op1, Phase.JOIN); + // Buffer a -U/+U pair for each key (tagged at bufferedAt = 100). + addBuildChange(h, updateBeforeRecord("k1", "v1", 1L)); + addBuildChange(h, updateAfterRecord("k1", "v2", 101L)); + addBuildChange(h, updateBeforeRecord("k2", "v1", 1L)); + addBuildChange(h, updateAfterRecord("k2", "v2", 101L)); + state = h.snapshot(0L, 0L); + } + + LateralSnapshotJoinOperator op2 = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op2)) { + h.initializeState(state); + h.open(); + assertPhase(op2, Phase.JOIN); + assertThat(op2.getCurrentBuildSideWm()).isEqualTo(Long.MIN_VALUE); + assertThat(bufferedChangesForKey(h, op2, "k1")).hasSize(2); + assertThat(bufferedChangesForKey(h, op2, "k2")).hasSize(2); + + // k2: accessed while no build WM has arrived since restore → eager drain. + addProbeRecord(h, 1L, "k2", "p-k2"); + // Draining k2's two restored changes leaves k1's two still buffered. + assertThat(bufferedChangesForKey(h, op2, "k2")).isEmpty(); + assertThat(buildTableForKey(h, op2, "k2")) + .containsExactlyInAnyOrderEntriesOf(Map.of("v2", 1L)); + + // k1: still buffered (eager drain of k2 left latestBuildSideWm at MIN_VALUE); advance + // the build WM, then access → normal WM-gated drain. + assertThat(bufferedChangesForKey(h, op2, "k1")).hasSize(2); + addBuildWm(h, 200L); + addProbeRecord(h, 2L, "k1", "p-k1"); + // All restored changes drained. + assertThat(bufferedChangesForKey(h, op2, "k1")).isEmpty(); + assertThat(buildTableForKey(h, op2, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("v2", 1L)); + + stripWatermarksAndStatusesFromOutput(h); + JOINED_ASSERTOR.shouldEmitAll( + h, + row(1L, "k2", "p-k2", "k2", "v2", 101L), + row(2L, "k1", "p-k1", "k1", "v2", 101L)); + } + } + + @Test + void restoreRearmsIdleFlipTimer() throws Exception { + LateralSnapshotJoinOperator op1 = newOperator(false, 1000L, 100L, null); + OperatorSubtaskState state; + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op1)) { + h.setProcessingTime(50); + h.open(); + assertPhase(op1, Phase.LOAD); + state = h.snapshot(0L, 0L); + } + + LateralSnapshotJoinOperator op2 = newOperator(false, 1000L, 100L, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op2)) { + h.setProcessingTime(150); + h.initializeState(state); + h.open(); + assertPhase(op2, Phase.LOAD); + // Re-armed at open()'s proc-time + idleTimeout = 150 + 100 = 250. + h.setProcessingTime(249); + assertPhase(op2, Phase.LOAD); + h.setProcessingTime(250); + assertPhase(op2, Phase.JOIN); + } + } + + // ----------------------------------------------------------------- Metrics + + @Test + void currentPhaseGaugeReflectsPhase() throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + // LOAD = ordinal 0. + assertThat(op.getCurrentPhaseGauge().getValue()).isEqualTo(0); + + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildWm(h, 100L); + assertPhase(op, Phase.JOIN); + // JOIN = ordinal 1. + assertThat(op.getCurrentPhaseGauge().getValue()).isEqualTo(1); + } + } + + @Test + void probeBufferedGaugeTracksLoadBufferingAndDrainsOnProbeWatermark() throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + assertThat(op.getNumProbeSideRecordsBufferedGauge().getValue()).isEqualTo(0L); + + addProbeRecord(h, 1L, "k1", "p1"); + assertThat(op.getNumProbeSideRecordsBufferedGauge().getValue()).isEqualTo(1L); + addProbeRecord(h, 2L, "k2", "p2"); + assertThat(op.getNumProbeSideRecordsBufferedGauge().getValue()).isEqualTo(2L); + addProbeRecord(h, 3L, "k1", "p3"); + assertThat(op.getNumProbeSideRecordsBufferedGauge().getValue()).isEqualTo(3L); + + // The flip alone does not drain the probes: no probe-side watermark has arrived yet. + addBuildWm(h, 100L); + assertPhase(op, Phase.JOIN); + assertThat(op.getNumProbeSideRecordsBufferedGauge().getValue()).isEqualTo(3L); + + // A probe-side watermark fires the FLIP timers and drains all buffered probes. + addProbeWm(h, 100L); + assertThat(op.getNumProbeSideRecordsBufferedGauge().getValue()).isEqualTo(0L); + } + } + + @Test + void watermarkGaugesTrackBuildAndProbeWatermarks() throws Exception { + // High loadCompletedTime so the operator stays in LOAD for the first build WMs. + LateralSnapshotJoinOperator op = newOperator(false, 1000L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + assertThat(op.getCurrentBuildSideWatermarkGauge().getValue()).isEqualTo(Long.MIN_VALUE); + assertThat(op.getCurrentProbeSideWatermarkGauge().getValue()).isEqualTo(Long.MIN_VALUE); + + // LOAD phase: both watermarks are tracked even though nothing is forwarded. + addBuildWm(h, 50L); + assertThat(op.getCurrentBuildSideWatermarkGauge().getValue()).isEqualTo(50L); + addProbeWm(h, 70L); + assertThat(op.getCurrentProbeSideWatermarkGauge().getValue()).isEqualTo(70L); + assertPhase(op, Phase.LOAD); + + // Flip to JOIN. + addBuildWm(h, 1000L); + assertPhase(op, Phase.JOIN); + assertThat(op.getCurrentBuildSideWatermarkGauge().getValue()).isEqualTo(1000L); + + // JOIN phase: probe WM gauge keeps tracking (guards the dedicated currentProbeWm + // field). + addProbeWm(h, 1500L); + assertThat(op.getCurrentProbeSideWatermarkGauge().getValue()).isEqualTo(1500L); + addBuildWm(h, 1100L); + assertThat(op.getCurrentBuildSideWatermarkGauge().getValue()).isEqualTo(1100L); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void fanOutGaugesTrackMaxAndAverage(boolean leftOuter) throws Exception { + LateralSnapshotJoinOperator op = newOperator(leftOuter, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + // k1 multi-set: v1 x2, v2 x1 → fan-out 3; k2 single row → fan-out 1. + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildChange(h, insertRecord("k1", "v2", 1L)); + addBuildChange(h, insertRecord("k2", "v1", 1L)); + // a probe record during load + addProbeRecord(h, 1L, "k2", "p1"); // fan-out 1 + // a probe record that does not match + addProbeRecord(h, 2L, "k3", "p2"); // fan-out INNER: 0, LEFT OUTER: 1 + addProbeWm(h, 100L); + addBuildWm(h, 100L); + assertPhase(op, Phase.JOIN); + + // check merge stats after transition + assertThat(op.getMaxJoinFanOutGauge().getValue()).isEqualTo(1L); + if (leftOuter) { + assertThat(op.getAvgJoinFanOutGauge().getValue()).isEqualTo(2d / 2); + assertThat(op.getNumUnmatchedProbeRecords().getCount()).isEqualTo(0L); + } else { + assertThat(op.getAvgJoinFanOutGauge().getValue()).isCloseTo(1d / 2, within(1e-9)); + assertThat(op.getNumUnmatchedProbeRecords().getCount()).isEqualTo(1L); + } + + // join probe in JOIN phase + addProbeRecord(h, 3L, "k1", "p3"); // fan-out 3 + assertThat(op.getMaxJoinFanOutGauge().getValue()).isEqualTo(3L); + if (leftOuter) { + assertThat(op.getAvgJoinFanOutGauge().getValue()).isEqualTo(5d / 3, within(1e-9)); + assertThat(op.getNumUnmatchedProbeRecords().getCount()).isEqualTo(0L); + } else { + assertThat(op.getAvgJoinFanOutGauge().getValue()).isCloseTo(4d / 3, within(1e-9)); + assertThat(op.getNumUnmatchedProbeRecords().getCount()).isEqualTo(1L); + } + + addProbeRecord(h, 3L, "k3", "no-match"); + assertThat(op.getMaxJoinFanOutGauge().getValue()).isEqualTo(3L); + if (leftOuter) { + // fan-out 1 (LEFT OUTER, null-padded) + assertThat(op.getAvgJoinFanOutGauge().getValue()).isCloseTo(6.0d / 4, within(1e-9)); + assertThat(op.getNumUnmatchedProbeRecords().getCount()).isEqualTo(0L); + } else { + // fan-out 0 (INNER, unmatched) + assertThat(op.getAvgJoinFanOutGauge().getValue()).isCloseTo(4.0d / 4, within(1e-9)); + assertThat(op.getNumUnmatchedProbeRecords().getCount()).isEqualTo(2L); + } + } + } + + @Test + void numUnmatchedBuildRetractionsCountsAbsentRowRetractions() throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + // LOAD: a -D for a never-inserted row is counted. + addBuildChange(h, deleteRecord("k1", "ghost", 1L)); + addBuildChange(h, insertRecord("k1", "v1", 2L)); + assertThat(op.getNumUnmatchedBuildRetractions().getCount()).isEqualTo(0L); + + // Add a real row for k1 + addBuildChange(h, insertRecord("k1", "v1", 3L)); + // Flip to JOIN + addBuildWm(h, 100L); + assertPhase(op, Phase.JOIN); + + // Access k1 → drains [-D(ghost)@1, +I(v1)@2]. The -D hits an absent row and is counted. + addBuildChange(h, deleteRecord("k1", "ghost2", 101L)); + assertThat(op.getNumUnmatchedBuildRetractions().getCount()).isEqualTo(1L); + + // Advance WM + access → drains -D(ghost2)@3 on an absent row. + addBuildWm(h, 110L); + addProbeRecord(h, 1L, "k1", "p1"); + assertThat(op.getNumUnmatchedBuildRetractions().getCount()).isEqualTo(2L); + + // Deleting an existing row (matching row-time) does not change the metric. + addBuildChange(h, deleteRecord("k1", "v1", 2L)); + addBuildWm(h, 120L); + addProbeRecord(h, 2L, "k1", "p2"); + assertThat(op.getNumUnmatchedBuildRetractions().getCount()).isEqualTo(2L); + } + } + + @Test + void buildRetractionAtCountOneRemovesRowWithoutGoingNegative() throws Exception { + LateralSnapshotJoinOperator op = newOperator(false, 100L, null, null); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.open(); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildWm(h, 100L); // flip + // Access drains +I(v1) → count 1; then buffer a -D for the same row. + addBuildChange(h, deleteRecord("k1", "v1", 1L)); + assertThat(buildTableForKey(h, op, "k1")) + .containsExactlyInAnyOrderEntriesOf(Map.of("v1", 1L)); + + // Drain the -D: count 1 → row removed from the multi-set (not a negative/0 entry), and + // it is not counted as an unmatched retraction. + addBuildWm(h, 110L); + addProbeRecord(h, 1L, "k1", "p1"); + assertThat(buildTableForKey(h, op, "k1")).isEmpty(); + assertThat(op.getNumUnmatchedBuildRetractions().getCount()).isEqualTo(0L); + } + } + + @Test + void numStateTtlEvictionsCountsEvictedKeys() throws Exception { + // stateTtlMs = 100 → timers registered at 1.5 × = 150. + LateralSnapshotJoinOperator op = newOperator(false, 50L, null, 100L); + try (KeyedTwoInputStreamOperatorTestHarness h = + newHarness(op)) { + h.setProcessingTime(0); + h.open(); + addBuildChange(h, insertRecord("k1", "v1", 1L)); + addBuildChange(h, insertRecord("k2", "v1", 1L)); + addBuildWm(h, 50L); // flip to JOIN at proc-time 0 + assertPhase(op, Phase.JOIN); + assertThat(op.getNumStateTtlEvictions().getCount()).isEqualTo(0L); + + // Neither key is touched again; both evict once their TTL timer (150) fires. + h.setProcessingTime(150); + assertThat(buildTableKeys(h)).isEmpty(); + assertThat(op.getNumStateTtlEvictions().getCount()).isEqualTo(2L); + } + } + + // ----------------------------------------------------------------- Helpers + + /** Sends a build-side change record (any {@link RowKind}) to the harness. */ + private static void addBuildChange( + KeyedTwoInputStreamOperatorTestHarness h, + StreamRecord record) + throws Exception { + h.processElement2(record); + } + + /** Sends a probe-side {@link RowKind#INSERT} record with the default probe schema. */ + private static void addProbeRecord( + KeyedTwoInputStreamOperatorTestHarness h, + Long id, + String key, + String val) + throws Exception { + h.processElement1(insertRecord(id, key, val)); + } + + /** Sends a build-side watermark to the harness (input 2). */ + private static void addBuildWm( + KeyedTwoInputStreamOperatorTestHarness h, long ts) + throws Exception { + h.processWatermark2(new Watermark(ts)); + } + + /** Sends a probe-side watermark to the harness (input 1). */ + private static void addProbeWm( + KeyedTwoInputStreamOperatorTestHarness h, long ts) + throws Exception { + h.processWatermark1(new Watermark(ts)); + } + + /** + * Builds an expected joined row. The operator only ever emits {@link RowKind#INSERT}, so we + * don't accept a rowkind parameter. + */ + private static GenericRowData row( + Long id, String pKey, String pVal, String bKey, String bVal, Long bRt) { + GenericRowData r = new GenericRowData(OUTPUT_TYPES.length); + r.setField(0, id); + r.setField(1, pKey == null ? null : StringData.fromString(pKey)); + r.setField(2, pVal == null ? null : StringData.fromString(pVal)); + r.setField(3, bKey == null ? null : StringData.fromString(bKey)); + r.setField(4, bVal == null ? null : StringData.fromString(bVal)); + r.setField(5, bRt); + r.setRowKind(RowKind.INSERT); + return r; + } + + /** + * Builds an expected joined row for a composite (two-field) equi-key on both sides: probe + * {@code (kA, kB, val)} concatenated with build {@code (kA, kB, val)}. + */ + private static GenericRowData compKeyRow( + String pKa, String pKb, String pVal, String bKa, String bKb, String bVal, Long bRt) { + GenericRowData r = new GenericRowData(7); + r.setField(0, pKa == null ? null : StringData.fromString(pKa)); + r.setField(1, pKb == null ? null : StringData.fromString(pKb)); + r.setField(2, pVal == null ? null : StringData.fromString(pVal)); + r.setField(3, bKa == null ? null : StringData.fromString(bKa)); + r.setField(4, bKb == null ? null : StringData.fromString(bKb)); + r.setField(5, bVal == null ? null : StringData.fromString(bVal)); + r.setField(6, bRt); + r.setRowKind(RowKind.INSERT); + return r; + } + + private static void assertPhase(LateralSnapshotJoinOperator op, Phase expected) { + assertThat(op.getPhase()).isEqualTo(expected); + } + + private static BinaryRowData stringKey(String key) { + BinaryRowData k = new BinaryRowData(1); + BinaryRowWriter w = new BinaryRowWriter(k); + if (key == null) { + w.setNullAt(0); + } else { + w.writeString(0, StringData.fromString(key)); + } + w.complete(); + return k; + } + + private static List buildTableKeys( + KeyedTwoInputStreamOperatorTestHarness h) + throws Exception { + return h.getOperator() + .getKeyedStateBackend() + .getKeys(BUILD_TABLE_STATE_NAME, VoidNamespace.INSTANCE) + .map(r -> ((BinaryRowData) r).getString(0).toString()) + .sorted() + .toList(); + } + + private static List probeBufferKeys( + KeyedTwoInputStreamOperatorTestHarness h) + throws Exception { + return h.getOperator() + .getKeyedStateBackend() + .getKeys(PROBE_BUFFER_STATE_NAME, VoidNamespace.INSTANCE) + .map(r -> ((BinaryRowData) r).getString(0).toString()) + .sorted() + .toList(); + } + + /** + * Returns the keys that currently hold any build-side state — either a materialized build-table + * entry or a buffered (not-yet-applied) build-side change. TTL eviction clears both, so this + * reflects whether a key is still "alive", regardless of whether its buffered changes have been + * drained into the build table yet. + */ + private static List buildStateKeys( + KeyedTwoInputStreamOperatorTestHarness h) + throws Exception { + java.util.SortedSet keys = new java.util.TreeSet<>(); + h.getOperator() + .getKeyedStateBackend() + .getKeys(BUILD_TABLE_STATE_NAME, VoidNamespace.INSTANCE) + .forEach(r -> keys.add(((BinaryRowData) r).getString(0).toString())); + h.getOperator() + .getKeyedStateBackend() + .getKeys(BUILD_CHANGE_BUFFER_STATE_NAME, VoidNamespace.INSTANCE) + .forEach(r -> keys.add(((BinaryRowData) r).getString(0).toString())); + return new ArrayList<>(keys); + } + + /** + * Returns the build-table multi-set for the given key as {@code build-val → count}. Assumes the + * schema's value field is at index 1. + */ + private static Map buildTableForKey( + KeyedTwoInputStreamOperatorTestHarness h, + LateralSnapshotJoinOperator op, + String key) + throws Exception { + h.getOperator().setCurrentKey(stringKey(key)); + Map result = new LinkedHashMap<>(); + for (Map.Entry e : op.getBuildTableState().entries()) { + result.put(e.getKey().getString(1).toString(), e.getValue()); + } + return result; + } + + private static List probeBufferForKey( + KeyedTwoInputStreamOperatorTestHarness h, + LateralSnapshotJoinOperator op, + String key) + throws Exception { + h.getOperator().setCurrentKey(stringKey(key)); + // Buffer is keyed by a synthetic, increasing timestamp; sort by key to recover insertion + // order regardless of state-backend iteration order. + TreeMap ordered = new TreeMap<>(); + for (Map.Entry e : op.getProbeBuffer().entries()) { + ordered.put(e.getKey(), e.getValue()); + } + return new ArrayList<>(ordered.values()); + } + + private static List bufferedChangesForKey( + KeyedTwoInputStreamOperatorTestHarness h, + LateralSnapshotJoinOperator op, + String key) + throws Exception { + h.getOperator().setCurrentKey(stringKey(key)); + // Buffer is keyed by row-time; flatten the per-row-time lists in ascending row-time order. + TreeMap> ordered = new TreeMap<>(); + for (Map.Entry> e : op.getBuildChangeBuffer().entries()) { + ordered.put(e.getKey(), e.getValue()); + } + List result = new ArrayList<>(); + ordered.values().forEach(result::addAll); + return result; + } + + private static Long bufferedAtWmFor( + KeyedTwoInputStreamOperatorTestHarness h, + LateralSnapshotJoinOperator op, + String key) + throws Exception { + h.getOperator().setCurrentKey(stringKey(key)); + return op.getBufferedAtWmState().value(); + } + + private static Long ttlExpiryFor( + KeyedTwoInputStreamOperatorTestHarness h, + LateralSnapshotJoinOperator op, + String key) + throws Exception { + h.getOperator().setCurrentKey(stringKey(key)); + return op.getTtlExpiryState().value(); + } + + /** Drops watermarks and watermark statuses from the harness output queue, in place. */ + private static void stripWatermarksAndStatusesFromOutput( + KeyedTwoInputStreamOperatorTestHarness h) { + h.getOutput().removeIf(o -> o instanceof Watermark || o instanceof WatermarkStatus); + } + + private static List extractWatermarks(ConcurrentLinkedQueue output) { + return output.stream().filter(o -> o instanceof Watermark).map(w -> (Watermark) w).toList(); + } + + /** + * Asserts that exactly one watermark equal to {@code expected} was emitted and that no {@link + * StreamRecord} follows it. A forwarded watermark must be released only after the records that + * logically precede it (e.g. probes drained on flip) have been emitted. + */ + private static void assertWatermarkForwardedAfterRecords( + ConcurrentLinkedQueue output, long expectedTs) { + Watermark expectedWatermark = new Watermark(expectedTs); + List elements = List.copyOf(output); + assertThat(extractWatermarks(output)).containsExactly(expectedWatermark); + int wmIndex = elements.indexOf(expectedWatermark); + assertThat(elements.subList(wmIndex + 1, elements.size())) + .as("no records may be emitted after watermark %s", expectedWatermark) + .noneMatch(o -> o instanceof StreamRecord); + } + + private static List extractWatermarkStatuses( + ConcurrentLinkedQueue output) { + return output.stream() + .filter(o -> o instanceof WatermarkStatus) + .map(w -> (WatermarkStatus) w) + .toList(); + } +} From 0e8e31c471e5e20ee820e5007da3a93aa5518e44 Mon Sep 17 00:00:00 2001 From: Fabian Hueske Date: Fri, 19 Jun 2026 18:11:51 +0200 Subject: [PATCH 2/5] [FLINK-39783][table] Translate LATERAL SNAPSHOT into a dedicated join operator Rewrites a join over a SNAPSHOT table function in a LATERAL clause into a dedicated FlinkLogicalLateralSnapshotJoin and converts it to a StreamPhysicalLateralSnapshotJoin / StreamExecLateralSnapshotJoin backed by the LateralSnapshotJoinOperator. The nodes derive their own output row type (materializing the build-side rowtime) and carry the SNAPSHOT arguments as fields. Generated-By: Claude Opus 4.8 (1M context) --- .../functions/BuiltInFunctionDefinitions.java | 3 +- .../LateralSnapshotTypeStrategy.java | 32 +- .../calcite/RelTimeIndicatorConverter.java | 40 ++ .../stream/StreamExecLateralSnapshotJoin.java | 242 ++++++++ .../LogicalJoinToLateralSnapshotJoinRule.java | 507 ++++++++++++++++ ...StreamPhysicalLateralSnapshotJoinRule.java | 93 +++ .../plan/utils/ExecNodeMetadataUtil.java | 2 + .../plan/utils/LateralSnapshotJoinUtil.java | 97 ++++ .../FlinkLogicalLateralSnapshotJoin.scala | 140 +++++ .../StreamPhysicalLateralSnapshotJoin.scala | 125 ++++ .../FlinkChangelogModeInferenceProgram.scala | 53 +- .../plan/rules/FlinkStreamRuleSets.scala | 4 + .../testutils/RestoreTestCompleteness.java | 6 + .../stream/sql/SnapshotTableFunctionTest.java | 110 ++-- .../sql/join/LateralSnapshotJoinTest.java | 540 ++++++++++++++++++ .../stream/sql/SnapshotTableFunctionTest.xml | 54 -- .../sql/join/LateralSnapshotJoinTest.xml | 310 ++++++++++ .../testInnerJoinJsonPlan.out | 396 +++++++++++++ ...testInnerJoinWithCompositeKeysJsonPlan.out | 396 +++++++++++++ ...JoinWithIdleTimeoutAndStateTtlJsonPlan.out | 398 +++++++++++++ ...tInnerJoinWithNonEquiConditionJsonPlan.out | 410 +++++++++++++ .../testLeftJoinJsonPlan.out | 396 +++++++++++++ 22 files changed, 4207 insertions(+), 147 deletions(-) create mode 100644 flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecLateralSnapshotJoin.java create mode 100644 flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/LogicalJoinToLateralSnapshotJoinRule.java create mode 100644 flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalLateralSnapshotJoinRule.java create mode 100644 flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/LateralSnapshotJoinUtil.java create mode 100644 flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalLateralSnapshotJoin.scala create mode 100644 flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalLateralSnapshotJoin.scala create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest.java create mode 100644 flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest.xml create mode 100644 flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinJsonPlan.out create mode 100644 flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinWithCompositeKeysJsonPlan.out create mode 100644 flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinWithIdleTimeoutAndStateTtlJsonPlan.out create mode 100644 flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinWithNonEquiConditionJsonPlan.out create mode 100644 flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testLeftJoinJsonPlan.out diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java index 0828f2bb8b4e92..180777f6561f62 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java @@ -946,7 +946,8 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL) .inputTypeStrategy(LATERAL_SNAPSHOT_INPUT_TYPE_STRATEGY) .outputTypeStrategy(LATERAL_SNAPSHOT_OUTPUT_TYPE_STRATEGY) .runtimeProvided() - // TODO: disableSystemArguments(true), once we have a dedicated translation rule + // SNAPSHOT does not support the implicit PTF system arguments (on_time, uid) + .disableSystemArguments(true) .notDeterministic() .build(); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/LateralSnapshotTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/LateralSnapshotTypeStrategy.java index 8ec2d4899c975d..49bead4730d099 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/LateralSnapshotTypeStrategy.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/LateralSnapshotTypeStrategy.java @@ -69,21 +69,31 @@ @Internal public final class LateralSnapshotTypeStrategy { - /** Argument index of the {@code input} TABLE. */ + /** The {@code input} TABLE argument. */ public static final int INPUT_ARG_INDEX = 0; - /** Argument index of the {@code load_completed_condition} STRING. */ + public static final String INPUT_ARG_NAME = "input"; + + /** The {@code load_completed_condition} STRING argument. */ public static final int LOAD_COMPLETED_CONDITION_ARG_INDEX = 1; - /** Argument index of the {@code load_completed_time} TIMESTAMP_LTZ. */ + public static final String LOAD_COMPLETED_CONDITION_ARG_NAME = "load_completed_condition"; + + /** The {@code load_completed_time} TIMESTAMP_LTZ argument. */ public static final int LOAD_COMPLETED_TIME_ARG_INDEX = 2; - /** Argument index of the {@code load_completed_idle_timeout} INTERVAL. */ + public static final String LOAD_COMPLETED_TIME_ARG_NAME = "load_completed_time"; + + /** The {@code load_completed_idle_timeout} INTERVAL argument. */ public static final int LOAD_COMPLETED_IDLE_TIMEOUT_ARG_INDEX = 3; - /** Argument index of the {@code state_ttl} INTERVAL. */ + public static final String LOAD_COMPLETED_IDLE_TIMEOUT_ARG_NAME = "load_completed_idle_timeout"; + + /** The {@code state_ttl} INTERVAL argument. */ public static final int STATE_TTL_ARG_INDEX = 4; + public static final String STATE_TTL_ARG_NAME = "state_ttl"; + /** Default value for {@code load_completed_condition}. */ public static final String LOAD_COMPLETED_CONDITION_COMPILE_TIME = "compile_time"; @@ -122,11 +132,13 @@ public Optional> inferInputTypes( public List getExpectedSignatures(final FunctionDefinition definition) { return List.of( Signature.of( - Argument.of("input", "TABLE"), - Argument.of("load_completed_condition", "STRING"), - Argument.of("load_completed_time", "TIMESTAMP_LTZ(3)"), - Argument.of("load_completed_idle_timeout", "INTERVAL SECOND"), - Argument.of("state_ttl", "INTERVAL SECOND"))); + Argument.of(INPUT_ARG_NAME, "TABLE"), + Argument.of(LOAD_COMPLETED_CONDITION_ARG_NAME, "STRING"), + Argument.of(LOAD_COMPLETED_TIME_ARG_NAME, "TIMESTAMP_LTZ(3)"), + Argument.of( + LOAD_COMPLETED_IDLE_TIMEOUT_ARG_NAME, + "INTERVAL SECOND"), + Argument.of(STATE_TTL_ARG_NAME, "INTERVAL SECOND"))); } }; diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/RelTimeIndicatorConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/RelTimeIndicatorConverter.java index 262665155789d5..6b2d229efb950d 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/RelTimeIndicatorConverter.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/RelTimeIndicatorConverter.java @@ -30,6 +30,7 @@ import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalExpand; import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalIntersect; import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalJoin; +import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalLateralSnapshotJoin; import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalLegacySink; import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalMatch; import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalMinus; @@ -164,6 +165,8 @@ public RelNode visit(RelNode node) { return visitCalc((FlinkLogicalCalc) node); } else if (node instanceof FlinkLogicalCorrelate) { return visitCorrelate((FlinkLogicalCorrelate) node); + } else if (node instanceof FlinkLogicalLateralSnapshotJoin) { + return visitLateralSnapshotJoin((FlinkLogicalLateralSnapshotJoin) node); } else if (node instanceof FlinkLogicalJoin) { return visitJoin((FlinkLogicalJoin) node); } else if (node instanceof FlinkLogicalMultiJoin) { @@ -367,6 +370,43 @@ public RexNode visitInputRef(RexInputRef inputRef) { } } + private RelNode visitLateralSnapshotJoin(FlinkLogicalLateralSnapshotJoin join) { + RelNode newLeft = join.getLeft().accept(this); + RelNode newRight = join.getRight().accept(this); + + // Materialize the build-side (right) proc-time attributes + newRight = materializeProcTime(newRight); + + List leftRightFields = new ArrayList<>(); + leftRightFields.addAll(newLeft.getRowType().getFieldList()); + leftRightFields.addAll(newRight.getRowType().getFieldList()); + + RexNode newCondition = + join.getCondition() + .accept( + new RexShuttle() { + @Override + public RexNode visitInputRef(RexInputRef inputRef) { + if (isTimeIndicatorType(inputRef.getType())) { + return RexInputRef.of( + inputRef.getIndex(), leftRightFields); + } else { + return super.visitInputRef(inputRef); + } + } + }); + + return FlinkLogicalLateralSnapshotJoin.create( + newLeft, + newRight, + newCondition, + join.getJoinType(), + join.loadCompletedCondition(), + join.loadCompletedTime(), + join.loadCompletedIdleTimeoutMs(), + join.stateTtlMs()); + } + private RelNode visitCorrelate(FlinkLogicalCorrelate correlate) { // visit children and update inputs RelNode newLeft = correlate.getLeft().accept(this); diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecLateralSnapshotJoin.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecLateralSnapshotJoin.java new file mode 100644 index 00000000000000..fe7c2de1fd4517 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecLateralSnapshotJoin.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.FlinkVersion; +import org.apache.flink.api.dag.Transformation; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.streaming.api.transformations.TwoInputTransformation; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.planner.delegation.PlannerBase; +import org.apache.flink.table.planner.plan.nodes.exec.ExecEdge; +import org.apache.flink.table.planner.plan.nodes.exec.ExecNodeBase; +import org.apache.flink.table.planner.plan.nodes.exec.ExecNodeConfig; +import org.apache.flink.table.planner.plan.nodes.exec.ExecNodeContext; +import org.apache.flink.table.planner.plan.nodes.exec.ExecNodeMetadata; +import org.apache.flink.table.planner.plan.nodes.exec.InputProperty; +import org.apache.flink.table.planner.plan.nodes.exec.SingleTransformationTranslator; +import org.apache.flink.table.planner.plan.nodes.exec.spec.JoinSpec; +import org.apache.flink.table.planner.plan.nodes.exec.utils.ExecNodeUtil; +import org.apache.flink.table.planner.plan.utils.JoinUtil; +import org.apache.flink.table.planner.plan.utils.KeySelectorUtil; +import org.apache.flink.table.runtime.generated.GeneratedJoinCondition; +import org.apache.flink.table.runtime.keyselector.RowDataKeySelector; +import org.apache.flink.table.runtime.operators.join.FlinkJoinType; +import org.apache.flink.table.runtime.operators.join.snapshot.LateralSnapshotJoinOperator; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.util.Preconditions; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.util.Arrays; +import java.util.List; + +/** + * {@link StreamExecNode} for the LATERAL SNAPSHOT processing-time temporal table join. The + * underlying {@link LateralSnapshotJoinOperator} runs in two phases (LOAD then JOIN) gated by a + * flip point on the build-side watermark. + */ +@ExecNodeMetadata( + name = "stream-exec-lateral-snapshot-join", + version = 1, + producedTransformations = + StreamExecLateralSnapshotJoin.LATERAL_SNAPSHOT_JOIN_TRANSFORMATION, + minPlanVersion = FlinkVersion.v2_4, + minStateVersion = FlinkVersion.v2_4) +public class StreamExecLateralSnapshotJoin extends ExecNodeBase + implements StreamExecNode, SingleTransformationTranslator { + + public static final String LATERAL_SNAPSHOT_JOIN_TRANSFORMATION = "lateral-snapshot-join"; + + public static final String FIELD_NAME_JOIN_SPEC = "joinSpec"; + public static final String FIELD_NAME_RIGHT_TIME_ATTRIBUTE_INDEX = "rightTimeAttributeIndex"; + public static final String FIELD_NAME_LOAD_COMPLETED_CONDITION = "loadCompletedCondition"; + public static final String FIELD_NAME_LOAD_COMPLETED_TIME = "loadCompletedTime"; + public static final String FIELD_NAME_LOAD_COMPLETED_IDLE_TIMEOUT_MS = + "loadCompletedIdleTimeoutMs"; + public static final String FIELD_NAME_STATE_TTL_MS = "stateTtlMs"; + + @JsonProperty(FIELD_NAME_JOIN_SPEC) + private final JoinSpec joinSpec; + + /** Field index of the build-side (right) row-time attribute. */ + @JsonProperty(FIELD_NAME_RIGHT_TIME_ATTRIBUTE_INDEX) + private final int rightTimeAttributeIndex; + + // Carried for explain output and plan serialization only; the operator uses loadCompletedTime, + // which the planner already resolved from this condition. + @JsonProperty(FIELD_NAME_LOAD_COMPLETED_CONDITION) + private final String loadCompletedCondition; + + @JsonProperty(FIELD_NAME_LOAD_COMPLETED_TIME) + private final Long loadCompletedTime; + + @JsonProperty(FIELD_NAME_LOAD_COMPLETED_IDLE_TIMEOUT_MS) + @JsonInclude(JsonInclude.Include.NON_NULL) + @Nullable + private final Long loadCompletedIdleTimeoutMs; + + @JsonProperty(FIELD_NAME_STATE_TTL_MS) + @JsonInclude(JsonInclude.Include.NON_NULL) + @Nullable + private final Long stateTtlMs; + + public StreamExecLateralSnapshotJoin( + ReadableConfig tableConfig, + JoinSpec joinSpec, + int rightTimeAttributeIndex, + String loadCompletedCondition, + Long loadCompletedTime, + @Nullable Long loadCompletedIdleTimeoutMs, + @Nullable Long stateTtlMs, + InputProperty leftInputProperty, + InputProperty rightInputProperty, + RowType outputType, + String description) { + this( + ExecNodeContext.newNodeId(), + ExecNodeContext.newContext(StreamExecLateralSnapshotJoin.class), + ExecNodeContext.newPersistedConfig( + StreamExecLateralSnapshotJoin.class, tableConfig), + joinSpec, + rightTimeAttributeIndex, + loadCompletedCondition, + loadCompletedTime, + loadCompletedIdleTimeoutMs, + stateTtlMs, + Arrays.asList(leftInputProperty, rightInputProperty), + outputType, + description); + } + + @JsonCreator + public StreamExecLateralSnapshotJoin( + @JsonProperty(FIELD_NAME_ID) int id, + @JsonProperty(FIELD_NAME_TYPE) ExecNodeContext context, + @JsonProperty(FIELD_NAME_CONFIGURATION) ReadableConfig persistedConfig, + @JsonProperty(FIELD_NAME_JOIN_SPEC) JoinSpec joinSpec, + @JsonProperty(FIELD_NAME_RIGHT_TIME_ATTRIBUTE_INDEX) int rightTimeAttributeIndex, + @JsonProperty(FIELD_NAME_LOAD_COMPLETED_CONDITION) String loadCompletedCondition, + @JsonProperty(FIELD_NAME_LOAD_COMPLETED_TIME) Long loadCompletedTime, + @JsonProperty(FIELD_NAME_LOAD_COMPLETED_IDLE_TIMEOUT_MS) @Nullable + Long loadCompletedIdleTimeoutMs, + @JsonProperty(FIELD_NAME_STATE_TTL_MS) @Nullable Long stateTtlMs, + @JsonProperty(FIELD_NAME_INPUT_PROPERTIES) List inputProperties, + @JsonProperty(FIELD_NAME_OUTPUT_TYPE) RowType outputType, + @JsonProperty(FIELD_NAME_DESCRIPTION) String description) { + super(id, context, persistedConfig, inputProperties, outputType, description); + Preconditions.checkArgument(inputProperties.size() == 2); + this.joinSpec = Preconditions.checkNotNull(joinSpec); + Preconditions.checkArgument( + rightTimeAttributeIndex >= 0, + "rightTimeAttributeIndex must be non-negative, but was %s", + rightTimeAttributeIndex); + this.rightTimeAttributeIndex = rightTimeAttributeIndex; + this.loadCompletedCondition = Preconditions.checkNotNull(loadCompletedCondition); + this.loadCompletedTime = Preconditions.checkNotNull(loadCompletedTime); + // the idle timeout and state TTL are optional and non-negative when set + Preconditions.checkArgument( + loadCompletedIdleTimeoutMs == null || loadCompletedIdleTimeoutMs >= 0, + "loadCompletedIdleTimeoutMs must be non-negative, but was %s", + loadCompletedIdleTimeoutMs); + Preconditions.checkArgument( + stateTtlMs == null || stateTtlMs >= 0, + "stateTtlMs must be non-negative, but was %s", + stateTtlMs); + this.loadCompletedIdleTimeoutMs = loadCompletedIdleTimeoutMs; + this.stateTtlMs = stateTtlMs; + } + + @Override + @SuppressWarnings("unchecked") + protected Transformation translateToPlanInternal( + PlannerBase planner, ExecNodeConfig config) { + final ExecEdge leftInputEdge = getInputEdges().get(0); + final ExecEdge rightInputEdge = getInputEdges().get(1); + final RowType leftInputType = (RowType) leftInputEdge.getOutputType(); + final RowType rightInputType = (RowType) rightInputEdge.getOutputType(); + + JoinUtil.validateJoinSpec(joinSpec, leftInputType, rightInputType, true); + + // Defensive: the SQL grammar and the rewrite rule already restrict LATERAL joins to + // INNER/LEFT, so this branch is not reachable from SQL. + final FlinkJoinType joinType = joinSpec.getJoinType(); + if (joinType != FlinkJoinType.INNER && joinType != FlinkJoinType.LEFT) { + throw new ValidationException( + "LATERAL SNAPSHOT join only supports INNER JOIN and LEFT OUTER JOIN, but was " + + joinType + + " JOIN."); + } + + final boolean isLeftOuterJoin = joinType == FlinkJoinType.LEFT; + final RowType returnType = (RowType) getOutputType(); + + final GeneratedJoinCondition generatedJoinCondition = + JoinUtil.generateConditionFunction( + config, + planner.getFlinkContext().getClassLoader(), + joinSpec, + leftInputType, + rightInputType); + + final LateralSnapshotJoinOperator operator = + new LateralSnapshotJoinOperator( + isLeftOuterJoin, + InternalTypeInfo.of(leftInputType), + InternalTypeInfo.of(rightInputType), + rightTimeAttributeIndex, + generatedJoinCondition, + joinSpec.getFilterNulls(), + loadCompletedTime, + loadCompletedIdleTimeoutMs, + stateTtlMs); + + final Transformation leftTransform = + (Transformation) leftInputEdge.translateToPlan(planner); + final Transformation rightTransform = + (Transformation) rightInputEdge.translateToPlan(planner); + + final TwoInputTransformation transform = + ExecNodeUtil.createTwoInputTransformation( + leftTransform, + rightTransform, + createTransformationMeta(LATERAL_SNAPSHOT_JOIN_TRANSFORMATION, config), + operator, + InternalTypeInfo.of(returnType), + leftTransform.getParallelism(), + false); + + final ClassLoader classLoader = planner.getFlinkContext().getClassLoader(); + final RowDataKeySelector leftKeySelector = + KeySelectorUtil.getRowDataSelector( + classLoader, joinSpec.getLeftKeys(), InternalTypeInfo.of(leftInputType)); + final RowDataKeySelector rightKeySelector = + KeySelectorUtil.getRowDataSelector( + classLoader, joinSpec.getRightKeys(), InternalTypeInfo.of(rightInputType)); + transform.setStateKeySelectors(leftKeySelector, rightKeySelector); + transform.setStateKeyType(leftKeySelector.getProducedType()); + return transform; + } +} diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/LogicalJoinToLateralSnapshotJoinRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/LogicalJoinToLateralSnapshotJoinRule.java new file mode 100644 index 00000000000000..a7c5fd770bf5a2 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/LogicalJoinToLateralSnapshotJoinRule.java @@ -0,0 +1,507 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.rules.logical; + +import org.apache.flink.table.api.TableException; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.planner.calcite.FlinkTypeFactory; +import org.apache.flink.table.planner.calcite.RexTableArgCall; +import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalCalc; +import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalJoin; +import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalLateralSnapshotJoin; +import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalTableFunctionScan; +import org.apache.flink.table.planner.plan.utils.FlinkRexUtil; +import org.apache.flink.table.planner.plan.utils.LateralSnapshotJoinUtil; +import org.apache.flink.table.types.inference.strategies.LateralSnapshotTypeStrategy; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.plan.hep.HepRelVertex; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.JoinInfo; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexExecutor; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexProgram; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.sql.SqlKind; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.immutables.value.Value; + +import java.util.ArrayList; +import java.util.List; + +/** + * Rewrites a {@link FlinkLogicalJoin} whose right side is a {@link FlinkLogicalTableFunctionScan} + * backed by the built-in {@code SNAPSHOT} function into a dedicated {@link + * FlinkLogicalLateralSnapshotJoin}. The right-side input becomes the actual TABLE argument of the + * SNAPSHOT call. The SNAPSHOT-specific arguments (load_completed_time, load_completed_idle_timeout, + * state_ttl) are carried as fields on the new node. + * + *

By the time this rule fires, Calcite's decorrelator has already converted the original {@code + * LogicalCorrelate} into a {@code LogicalJoin} (because SNAPSHOT does not actually reference any + * field of the outer input). The rule therefore matches the join shape directly. + */ +@Value.Enclosing +public class LogicalJoinToLateralSnapshotJoinRule + extends RelRule< + LogicalJoinToLateralSnapshotJoinRule.LogicalJoinToLateralSnapshotJoinRuleConfig> { + + public static final LogicalJoinToLateralSnapshotJoinRule INSTANCE = + LogicalJoinToLateralSnapshotJoinRule.LogicalJoinToLateralSnapshotJoinRuleConfig.DEFAULT + .toRule(); + + private LogicalJoinToLateralSnapshotJoinRule( + LogicalJoinToLateralSnapshotJoinRuleConfig config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + final FlinkLogicalJoin join = call.rel(0); + // the rule replaces FlinkLogicalJoin, so it won't fire on its output. + return findSnapshotScan(join.getRight()) != null; + } + + @Override + public void onMatch(RelOptRuleCall call) { + final FlinkLogicalJoin join = call.rel(0); + final RelNode leftNode = join.getLeft(); + final FlinkLogicalTableFunctionScan scan = findSnapshotScan(join.getRight()); + if (scan == null) { + // matches() guarantees a SNAPSHOT scan on the right, so this cannot happen. + throw new TableException( + "Could not find the SNAPSHOT scan on the build side of a LATERAL SNAPSHOT " + + "join. This is a bug, please file an issue."); + } + + // SQL syntax already restricts LATERAL-side joins to INNER/LEFT, this is a defensive check. + final JoinRelType joinType = join.getJoinType(); + if (joinType != JoinRelType.INNER && joinType != JoinRelType.LEFT) { + throw new ValidationException( + String.format( + "LATERAL SNAPSHOT join only supports INNER JOIN and LEFT OUTER JOIN, but was %s JOIN.", + joinType)); + } + + // Require at least one equality predicate so the operator can hash-partition both inputs. + final JoinInfo joinInfo = join.analyzeCondition(); + if (joinInfo.leftKeys.isEmpty()) { + throw new ValidationException( + "LATERAL SNAPSHOT join requires at least one equality predicate."); + } + + final RexCall snapshotCall = (RexCall) scan.getCall(); + + // Resolve the raw build-side TABLE input the operator reads. A null result means the + // SNAPSHOT call is malformed, which cannot happen for a plan that reached this rule. + final RelNode rawTableInput = getSnapshotInputTable(scan); + if (rawTableInput == null) { + throw new TableException( + "Could not resolve the TABLE input of the SNAPSHOT scan on the build side of " + + "a LATERAL SNAPSHOT join. This is a bug, please file an issue."); + } + // The build-side input must declare exactly one watermark, otherwise the operator cannot + // determine when the LOAD phase is complete. + final long rowtimeCount = + rawTableInput.getRowType().getFieldList().stream() + .filter(f -> FlinkTypeFactory.isRowtimeIndicatorType(f.getType())) + .count(); + if (rowtimeCount == 0) { + throw new ValidationException( + "LATERAL SNAPSHOT requires a watermark on the build-side input."); + } + if (rowtimeCount > 1) { + throw new ValidationException( + String.format( + "The build-side input of a LATERAL SNAPSHOT join must not have more than one " + + "row-time attribute, but found %d.", + rowtimeCount)); + } + + // Replace the SNAPSHOT TableFunctionScan with its input, preserving any FlinkLogicalCalc + // nodes that the optimizer placed above the scan. + final RelNode rightNode = replaceSnapshotScan(join.getRight()); + if (rightNode == null) { + throw new TableException( + "Could not rewrite the build side of a LATERAL SNAPSHOT join by replacing the " + + "SNAPSHOT scan with its TABLE input. This is a bug, please file an " + + "issue."); + } + + final List operands = snapshotCall.getOperands(); + final RexBuilder rexBuilder = join.getCluster().getRexBuilder(); + final RexExecutor executor = join.getCluster().getPlanner().getExecutor(); + + // All scalar SNAPSHOT arguments must be constant expressions, so we constant-fold each one + // and reject anything that does not reduce to a literal. The 'input' TABLE argument + // (index 0) is exempt. + final RexLiteral conditionLiteral = + foldToLiteral( + rexBuilder, + executor, + operands, + LateralSnapshotTypeStrategy.LOAD_COMPLETED_CONDITION_ARG_INDEX, + LateralSnapshotTypeStrategy.LOAD_COMPLETED_CONDITION_ARG_NAME); + final RexLiteral loadCompletedTimeLiteral = + foldToLiteral( + rexBuilder, + executor, + operands, + LateralSnapshotTypeStrategy.LOAD_COMPLETED_TIME_ARG_INDEX, + LateralSnapshotTypeStrategy.LOAD_COMPLETED_TIME_ARG_NAME); + final RexLiteral idleTimeoutLiteral = + foldToLiteral( + rexBuilder, + executor, + operands, + LateralSnapshotTypeStrategy.LOAD_COMPLETED_IDLE_TIMEOUT_ARG_INDEX, + LateralSnapshotTypeStrategy.LOAD_COMPLETED_IDLE_TIMEOUT_ARG_NAME); + final RexLiteral stateTtlLiteral = + foldToLiteral( + rexBuilder, + executor, + operands, + LateralSnapshotTypeStrategy.STATE_TTL_ARG_INDEX, + LateralSnapshotTypeStrategy.STATE_TTL_ARG_NAME); + + // Resolve load_completed_time according to load_completed_condition. The default + // 'compile_time' uses the wall-clock time at planning; 'user_time' uses the user-provided + // load_completed_time (which the type strategy guarantees is present for 'user_time'). + final String condition = + conditionLiteral == null ? null : conditionLiteral.getValueAs(String.class); + final Long loadCompletedTime; + if (condition == null + || LateralSnapshotTypeStrategy.LOAD_COMPLETED_CONDITION_COMPILE_TIME.equals( + condition)) { + loadCompletedTime = System.currentTimeMillis(); + } else if (LateralSnapshotTypeStrategy.LOAD_COMPLETED_CONDITION_USER_TIME.equals( + condition)) { + loadCompletedTime = + loadCompletedTimeLiteral == null + ? null + : loadCompletedTimeLiteral.getValueAs(Long.class); + if (loadCompletedTime == null) { + throw new ValidationException( + "SNAPSHOT requires 'load_completed_time' when " + + "'load_completed_condition' is 'user_time'."); + } + } else { + throw new ValidationException( + String.format("Unknown SNAPSHOT 'load_completed_condition': '%s'.", condition)); + } + + // The effective condition (defaulting to 'compile_time') is carried for explain output. + final String loadCompletedCondition = + condition == null + ? LateralSnapshotTypeStrategy.LOAD_COMPLETED_CONDITION_COMPILE_TIME + : condition; + final Long loadCompletedIdleTimeoutMs = + intervalMillis( + idleTimeoutLiteral, + LateralSnapshotTypeStrategy.LOAD_COMPLETED_IDLE_TIMEOUT_ARG_NAME); + final Long stateTtlMs = + intervalMillis(stateTtlLiteral, LateralSnapshotTypeStrategy.STATE_TTL_ARG_NAME); + + // The original join condition's field types were resolved against the SNAPSHOT scan's + // materialized output, but rightNode (its raw TABLE input) still exposes the build-side + // row-time attribute as an indicator (see replaceSnapshotScan). Retype the condition to + // the actual left+right input types. + final List leftRightFields = new ArrayList<>(); + leftRightFields.addAll(leftNode.getRowType().getFieldList()); + leftRightFields.addAll(rightNode.getRowType().getFieldList()); + final RexNode rebasedCondition = + join.getCondition() + .accept( + new RexShuttle() { + @Override + public RexNode visitInputRef(RexInputRef inputRef) { + return RexInputRef.of(inputRef.getIndex(), leftRightFields); + } + }); + + // Replace the join (over the materialized SNAPSHOT scan) with a dedicated + // FlinkLogicalLateralSnapshotJoin taking the rewritten SNAPSHOT function input as + // build-side input. + final RelNode node = + FlinkLogicalLateralSnapshotJoin.create( + leftNode, + rightNode, + rebasedCondition, + joinType, + loadCompletedCondition, + loadCompletedTime, + loadCompletedIdleTimeoutMs, + stateTtlMs); + + final int origRightCount = unwrap(join.getRight()).getRowType().getFieldCount(); + final int newRightCount = rightNode.getRowType().getFieldCount(); + final boolean isRowtimeFieldAdded = newRightCount > origRightCount; + if (isRowtimeFieldAdded) { + // If the build-side projection stripped the row-time attribute, replaceSnapshotScan + // re-appended it as a trailing column so it reaches the operator. In that case the node + // has extra trailing column(s) that a wrapper Calc projects away to restore the + // original join's output type. Otherwise, the node's output type already matches the + // original join. + final RelDataType originalOutputType = join.getRowType(); + final List wrapperProjects = new ArrayList<>(); + for (int i = 0; i < originalOutputType.getFieldCount(); i++) { + wrapperProjects.add(rexBuilder.makeInputRef(node, i)); + } + final RexProgram wrapperProgram = + RexProgram.create( + node.getRowType(), + wrapperProjects, + null, + originalOutputType.getFieldNames(), + rexBuilder); + call.transformTo(FlinkLogicalCalc.create(node, wrapperProgram)); + } else { + call.transformTo(node); + } + } + + /** + * Walks down a join's right input looking for a {@link FlinkLogicalTableFunctionScan} whose + * call is the {@code SNAPSHOT} built-in. Walks past {@link FlinkLogicalCalc} nodes and breaks + * on any other node type. Returns the {@link FlinkLogicalTableFunctionScan} if found, or null + * if an unexpected node was observed, the subtree splits up (more than one input), or the tree + * ends. + */ + @Nullable + private static FlinkLogicalTableFunctionScan findSnapshotScan(RelNode root) { + RelNode current = unwrap(root); + while (current != null) { + if (current instanceof FlinkLogicalTableFunctionScan) { + final FlinkLogicalTableFunctionScan scan = (FlinkLogicalTableFunctionScan) current; + if (scan.getCall() instanceof RexCall + && LateralSnapshotJoinUtil.isSnapshotCall((RexCall) scan.getCall())) { + return scan; + } + return null; + } + // Walk through pass-through nodes (e.g. FlinkLogicalCalc inserted by the optimizer). A + // Calc always has a single input; the size check is defensive. + if (current instanceof FlinkLogicalCalc) { + current = unwrap(current.getInput(0)); + } else { + return null; + } + } + return null; + } + + private static RelNode unwrap(RelNode node) { + return node instanceof HepRelVertex ? ((HepRelVertex) node).getCurrentRel() : node; + } + + /** + * Returns the raw (unwrapped) TABLE input of a {@code SNAPSHOT} scan, i.e. the build-side input + * the operator reads. Returns {@code null} if the scan does not carry a SNAPSHOT call or its + * TABLE argument cannot be resolved. + */ + @Nullable + private static RelNode getSnapshotInputTable(FlinkLogicalTableFunctionScan scan) { + if (!(scan.getCall() instanceof RexCall) + || !LateralSnapshotJoinUtil.isSnapshotCall((RexCall) scan.getCall())) { + return null; + } + final RexCall snapshotCall = (RexCall) scan.getCall(); + final RexNode inputArg = + snapshotCall.getOperands().get(LateralSnapshotTypeStrategy.INPUT_ARG_INDEX); + if (!(inputArg instanceof RexTableArgCall)) { + return null; + } + final RexTableArgCall tableArg = (RexTableArgCall) inputArg; + if (tableArg.getInputIndex() < 0 || tableArg.getInputIndex() >= scan.getInputs().size()) { + return null; + } + return unwrap(scan.getInputs().get(tableArg.getInputIndex())); + } + + /** + * Walks the right subtree replacing the {@link FlinkLogicalTableFunctionScan} (the SNAPSHOT + * scan) with the scan's TABLE input, while preserving any {@link FlinkLogicalCalc} nodes + * stacked above the scan. The SNAPSHOT type strategy materializes the build-side time + * attributes, so the scan's output type differs from its input's (the build-side row-time + * attribute is a plain timestamp on the scan output but a row-time indicator on the raw input). + * Each preserved Calc was built against the materialized scan output, so its {@link RexProgram} + * is rebased onto the raw (row-time-bearing) input type, which lets the row-time attribute flow + * through to the operator. + */ + @Nullable + private static RelNode replaceSnapshotScan(RelNode node) { + final RelNode current = unwrap(node); + if (current instanceof FlinkLogicalTableFunctionScan) { + // the top node is the TableFunctionScan, return its table input argument + return getSnapshotInputTable((FlinkLogicalTableFunctionScan) current); + } + if (current instanceof FlinkLogicalCalc) { + // the top node is a calc that needs to be rebased + final FlinkLogicalCalc calc = (FlinkLogicalCalc) current; + final RelNode rewrittenInput = replaceSnapshotScan(calc.getInput(0)); + if (rewrittenInput == null) { + return null; + } + return rebaseCalc(calc, rewrittenInput); + } + return null; + } + + /** + * Rebuilds {@code calc}'s {@link RexProgram} so it reads from {@code newInput} (whose + * build-side time attributes are still row-time indicators) instead of the materialized + * SNAPSHOT scan output it was originally built against. Input references are retyped to the new + * input's field types; the projection/condition expressions and output field names are + * otherwise preserved. + * + *

If the projection dropped the build-side row-time attribute, it is re-appended as a + * trailing column so it is available for the snapshot join operator. + */ + private static RelNode rebaseCalc(FlinkLogicalCalc calc, RelNode newInput) { + final RexProgram program = calc.getProgram(); + final RexBuilder rexBuilder = calc.getCluster().getRexBuilder(); + final List newInputFields = newInput.getRowType().getFieldList(); + final RexShuttle retyper = + new RexShuttle() { + @Override + public RexNode visitInputRef(RexInputRef inputRef) { + return new RexInputRef( + inputRef.getIndex(), + newInputFields.get(inputRef.getIndex()).getType()); + } + }; + final List newProjects = new ArrayList<>(); + program.getProjectList().stream() + .map(r -> program.expandLocalRef(r).accept(retyper)) + .forEach(newProjects::add); + + final RexNode newCondition = + program.getCondition() == null + ? null + : program.expandLocalRef(program.getCondition()).accept(retyper); + final List fieldNames = new ArrayList<>(program.getOutputRowType().getFieldNames()); + + // Re-append the build-side row-time attribute if this projection dropped it. + final boolean exposesRowtime = + newProjects.stream() + .anyMatch(p -> FlinkTypeFactory.isRowtimeIndicatorType(p.getType())); + if (!exposesRowtime) { + newInputFields.stream() + .filter(f -> FlinkTypeFactory.isRowtimeIndicatorType(f.getType())) + .findFirst() + .ifPresent( + f -> { + newProjects.add(new RexInputRef(f.getIndex(), f.getType())); + fieldNames.add(uniqueName(f.getName(), fieldNames)); + }); + } + + final RexProgram newProgram = + RexProgram.create( + newInput.getRowType(), newProjects, newCondition, fieldNames, rexBuilder); + return calc.copy(calc.getTraitSet(), newInput, newProgram); + } + + private static String uniqueName(String name, List existing) { + String candidate = name; + int suffix = 0; + while (existing.contains(candidate)) { + candidate = name + "_" + suffix++; + } + return candidate; + } + + /** + * Returns the SNAPSHOT argument at {@code index} as a constant {@link RexLiteral}, or {@code + * null} if the argument is absent (omitted optional arguments are carried as a {@code + * DEFAULT()} call) or explicitly NULL. The argument may still be carried as a cast or a + * deterministic function call at this point, so it is constant-folded first. Throws a {@link + * ValidationException} if the expression cannot be constant-folded. + */ + @Nullable + private static RexLiteral foldToLiteral( + RexBuilder rexBuilder, + RexExecutor executor, + List operands, + int index, + String argName) { + if (index >= operands.size()) { + return null; + } + RexNode operand = operands.get(index); + if (operand.isA(SqlKind.DEFAULT)) { + // Optional argument not provided by the user. + return null; + } + if (!(operand instanceof RexLiteral)) { + operand = FlinkRexUtil.simplify(rexBuilder, operand, executor); + } + if (!(operand instanceof RexLiteral)) { + throw new ValidationException( + String.format( + "Argument '%s' of SNAPSHOT must be a constant expression that can be evaluated at plan time.", + argName)); + } + final RexLiteral literal = (RexLiteral) operand; + return literal.isNull() ? null : literal; + } + + /** + * Returns the value of a folded {@code INTERVAL} literal in milliseconds, or {@code null} if + * the literal is absent. The function signature guarantees a day-time interval, so {@link + * RexLiteral#getValueAs} yields milliseconds; a negative duration is not meaningful for these + * arguments and is rejected. + */ + @Nullable + private static Long intervalMillis(@Nullable RexLiteral literal, String argName) { + if (literal == null) { + return null; + } + final Long millis = literal.getValueAs(Long.class); + if (millis != null && millis < 0) { + throw new ValidationException( + String.format("Argument '%s' of SNAPSHOT must not be negative.", argName)); + } + return millis; + } + + /** Rule configuration. */ + @Value.Immutable(singleton = false) + public interface LogicalJoinToLateralSnapshotJoinRuleConfig extends RelRule.Config { + + LogicalJoinToLateralSnapshotJoinRule.LogicalJoinToLateralSnapshotJoinRuleConfig DEFAULT = + ImmutableLogicalJoinToLateralSnapshotJoinRule + .LogicalJoinToLateralSnapshotJoinRuleConfig.builder() + .build() + .withOperandSupplier(b0 -> b0.operand(FlinkLogicalJoin.class).anyInputs()) + .withDescription("LogicalJoinToLateralSnapshotJoinRule"); + + @Override + default LogicalJoinToLateralSnapshotJoinRule toRule() { + return new LogicalJoinToLateralSnapshotJoinRule(this); + } + } +} diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalLateralSnapshotJoinRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalLateralSnapshotJoinRule.java new file mode 100644 index 00000000000000..628b29b9ef9e18 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalLateralSnapshotJoinRule.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.rules.physical.stream; + +import org.apache.flink.table.planner.plan.nodes.FlinkConventions; +import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalLateralSnapshotJoin; +import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalLateralSnapshotJoin; +import org.apache.flink.table.planner.plan.trait.FlinkRelDistribution; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.apache.calcite.rel.core.JoinInfo; +import org.apache.calcite.util.ImmutableIntList; + +/** + * Converts a {@link FlinkLogicalLateralSnapshotJoin} (created by {@link + * org.apache.flink.table.planner.plan.rules.logical.LogicalJoinToLateralSnapshotJoinRule}) into a + * {@link StreamPhysicalLateralSnapshotJoin}. The SNAPSHOT arguments are carried on the logical + * node, so the conversion is a straight pass-through. + */ +public class StreamPhysicalLateralSnapshotJoinRule extends ConverterRule { + + public static final StreamPhysicalLateralSnapshotJoinRule INSTANCE = + new StreamPhysicalLateralSnapshotJoinRule( + Config.INSTANCE.withConversion( + FlinkLogicalLateralSnapshotJoin.class, + FlinkConventions.LOGICAL(), + FlinkConventions.STREAM_PHYSICAL(), + "StreamPhysicalLateralSnapshotJoinRule")); + + private StreamPhysicalLateralSnapshotJoinRule(Config config) { + super(config); + } + + @Override + public RelNode convert(RelNode rel) { + final FlinkLogicalLateralSnapshotJoin join = (FlinkLogicalLateralSnapshotJoin) rel; + final RelTraitSet providedTraitSet = + rel.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL()); + + // Both inputs are hash-partitioned on their join keys. + final JoinInfo joinInfo = join.analyzeCondition(); + final RelNode newLeft = convertInput(join.getLeft(), joinInfo.leftKeys); + final RelNode newRight = convertInput(join.getRight(), joinInfo.rightKeys); + + return new StreamPhysicalLateralSnapshotJoin( + join.getCluster(), + providedTraitSet, + newLeft, + newRight, + join.getCondition(), + join.getJoinType(), + join.loadCompletedCondition(), + join.loadCompletedTime(), + join.loadCompletedIdleTimeoutMs(), + join.stateTtlMs()); + } + + /** + * Converts a join input to the stream-physical convention and requires it to be + * hash-partitioned on the given join {@code keys} (or a singleton distribution if there are + * none). + */ + private static RelNode convertInput(RelNode input, ImmutableIntList keys) { + final FlinkRelDistribution distribution = + keys.isEmpty() + ? FlinkRelDistribution.SINGLETON() + : FlinkRelDistribution.hash(keys.toIntArray(), true); + final RelTraitSet requiredTraitSet = + input.getTraitSet() + .replace(FlinkConventions.STREAM_PHYSICAL()) + .replace(distribution); + return RelOptRule.convert(input, requiredTraitSet); + } +} diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/ExecNodeMetadataUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/ExecNodeMetadataUtil.java index eff02bf2d83ceb..19fdd3b1013cea 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/ExecNodeMetadataUtil.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/ExecNodeMetadataUtil.java @@ -69,6 +69,7 @@ import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecIncrementalGroupAggregate; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecIntervalJoin; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecJoin; +import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecLateralSnapshotJoin; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecLegacySink; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecLegacyTableSourceScan; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecLimit; @@ -149,6 +150,7 @@ private ExecNodeMetadataUtil() { add(StreamExecIncrementalGroupAggregate.class); add(StreamExecIntervalJoin.class); add(StreamExecJoin.class); + add(StreamExecLateralSnapshotJoin.class); add(StreamExecLimit.class); add(StreamExecLocalGroupAggregate.class); add(StreamExecLocalWindowAggregate.class); diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/LateralSnapshotJoinUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/LateralSnapshotJoinUtil.java new file mode 100644 index 00000000000000..95bca1a1384ec6 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/utils/LateralSnapshotJoinUtil.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.utils; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.functions.BuiltInFunctionDefinition; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.FunctionDefinition; +import org.apache.flink.table.planner.functions.bridging.BridgingSqlFunction; +import org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType; + +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.validate.SqlValidatorUtil; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.List; + +/** + * Utilities for recognizing calls to the {@code SNAPSHOT} built-in used by the {@code LATERAL + * SNAPSHOT} processing-time temporal join. + */ +@Internal +public final class LateralSnapshotJoinUtil { + + /** + * {@code true} when {@code definition} is the {@link BuiltInFunctionDefinitions#SNAPSHOT} + * built-in. + */ + public static boolean isSnapshotFunction(@Nullable FunctionDefinition definition) { + return definition instanceof BuiltInFunctionDefinition + && BuiltInFunctionDefinitions.SNAPSHOT + .getName() + .equals(((BuiltInFunctionDefinition) definition).getName()); + } + + /** + * {@code true} when the operator of {@code call} is a {@link BridgingSqlFunction} whose + * function definition is the SNAPSHOT built-in. + */ + public static boolean isSnapshotCall(@Nullable RexCall call) { + if (call == null) { + return false; + } + if (!(call.getOperator() instanceof BridgingSqlFunction)) { + return false; + } + final BridgingSqlFunction bridging = (BridgingSqlFunction) call.getOperator(); + return isSnapshotFunction(bridging.getDefinition()); + } + + /** + * Derives the output row type of a {@code LATERAL SNAPSHOT} join. The {@code SNAPSHOT} function + * does not forward the build-side (right) time attributes, so they are materialized in the + * output; the probe-side (left) time attributes are forwarded unchanged. Field names are + * uniquified and the build-side nullability follows the join type, matching {@link + * org.apache.calcite.rel.core.Join#deriveRowType()}. + */ + public static RelDataType deriveRowType( + RelDataTypeFactory typeFactory, + RelDataType leftType, + RelDataType rightType, + JoinRelType joinType, + List systemFieldList) { + final RelDataTypeFactory.Builder materializedRight = typeFactory.builder(); + for (RelDataTypeField field : rightType.getFieldList()) { + final RelDataType fieldType = + field.getType() instanceof TimeIndicatorRelDataType + ? ((TimeIndicatorRelDataType) field.getType()).getOriginalType() + : field.getType(); + materializedRight.add(field.getName(), fieldType); + } + return SqlValidatorUtil.deriveJoinRowType( + leftType, materializedRight.build(), joinType, typeFactory, null, systemFieldList); + } + + private LateralSnapshotJoinUtil() {} +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalLateralSnapshotJoin.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalLateralSnapshotJoin.scala new file mode 100644 index 00000000000000..1dcaeaad6200a0 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalLateralSnapshotJoin.scala @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.table.planner.plan.nodes.logical + +import org.apache.flink.table.planner.plan.nodes.FlinkConventions +import org.apache.flink.table.planner.plan.utils.LateralSnapshotJoinUtil + +import org.apache.calcite.plan.{RelOptCluster, RelOptCost, RelOptPlanner, RelTraitSet} +import org.apache.calcite.rel.`type`.RelDataType +import org.apache.calcite.rel.{RelNode, RelWriter} +import org.apache.calcite.rel.core.{CorrelationId, Join, JoinRelType} +import org.apache.calcite.rel.hint.RelHint +import org.apache.calcite.rel.metadata.RelMetadataQuery +import org.apache.calcite.rex.RexNode + +import java.util.Collections + +/** + * Logical node for the {@code LATERAL SNAPSHOT} processing-time temporal table join. + * + *

the {@code LATERAL SNAPSHOT} join materializes the build-side row-time attribute, the + * row-time attribute of the probe-side is forwarded. The arguments of the {@code SNAPSHOT} function + * are persisted in fields of the logical node. + */ +class FlinkLogicalLateralSnapshotJoin( + cluster: RelOptCluster, + traitSet: RelTraitSet, + left: RelNode, + right: RelNode, + condition: RexNode, + joinType: JoinRelType, + val loadCompletedCondition: String, + val loadCompletedTime: java.lang.Long, + val loadCompletedIdleTimeoutMs: java.lang.Long, + val stateTtlMs: java.lang.Long) + extends Join( + cluster, + traitSet, + Collections.emptyList[RelHint](), + left, + right, + condition, + Collections.emptySet[CorrelationId](), + joinType) + with FlinkLogicalRel { + + require(loadCompletedTime != null, "loadCompletedTime must not be null.") + + override def copy( + traitSet: RelTraitSet, + conditionExpr: RexNode, + left: RelNode, + right: RelNode, + joinType: JoinRelType, + semiJoinDone: Boolean): Join = { + new FlinkLogicalLateralSnapshotJoin( + cluster, + traitSet, + left, + right, + conditionExpr, + joinType, + loadCompletedCondition, + loadCompletedTime, + loadCompletedIdleTimeoutMs, + stateTtlMs) + } + + override def deriveRowType(): RelDataType = + LateralSnapshotJoinUtil.deriveRowType( + getCluster.getTypeFactory, + left.getRowType, + right.getRowType, + joinType, + getSystemFieldList) + + override def explainTerms(pw: RelWriter): RelWriter = { + val terms = super.explainTerms(pw) + terms.item("loadCompletedCondition", loadCompletedCondition) + terms.item("loadCompletedTime", loadCompletedTime) + if (loadCompletedIdleTimeoutMs != null) { + terms.item("loadCompletedIdleTimeout", s"$loadCompletedIdleTimeoutMs ms") + } + if (stateTtlMs != null) { + terms.item("stateTtl", s"$stateTtlMs ms") + } + terms + } + + override def computeSelfCost(planner: RelOptPlanner, mq: RelMetadataQuery): RelOptCost = { + val leftRowCnt = mq.getRowCount(getLeft) + val leftRowSize = mq.getAverageRowSize(getLeft) + val rightRowCnt = mq.getRowCount(getRight) + val cpuCost = leftRowCnt + rightRowCnt + val ioCost = leftRowCnt * leftRowSize + planner.getCostFactory.makeCost(leftRowCnt, cpuCost, ioCost) + } +} + +object FlinkLogicalLateralSnapshotJoin { + + def create( + left: RelNode, + right: RelNode, + condition: RexNode, + joinType: JoinRelType, + loadCompletedCondition: String, + loadCompletedTime: java.lang.Long, + loadCompletedIdleTimeoutMs: java.lang.Long, + stateTtlMs: java.lang.Long): FlinkLogicalLateralSnapshotJoin = { + val cluster = left.getCluster + val traitSet = cluster.traitSetOf(FlinkConventions.LOGICAL).simplify() + new FlinkLogicalLateralSnapshotJoin( + cluster, + traitSet, + left, + right, + condition, + joinType, + loadCompletedCondition, + loadCompletedTime, + loadCompletedIdleTimeoutMs, + stateTtlMs) + } +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalLateralSnapshotJoin.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalLateralSnapshotJoin.scala new file mode 100644 index 00000000000000..2f1b1e8e323abc --- /dev/null +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalLateralSnapshotJoin.scala @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.table.planner.plan.nodes.physical.stream + +import org.apache.flink.table.api.TableException +import org.apache.flink.table.planner.calcite.FlinkTypeFactory +import org.apache.flink.table.planner.plan.nodes.exec.{ExecNode, InputProperty} +import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecLateralSnapshotJoin +import org.apache.flink.table.planner.plan.nodes.physical.common.CommonPhysicalJoin +import org.apache.flink.table.planner.plan.utils.LateralSnapshotJoinUtil +import org.apache.flink.table.planner.utils.ShortcutUtils.unwrapTableConfig + +import org.apache.calcite.plan.{RelOptCluster, RelTraitSet} +import org.apache.calcite.rel.`type`.RelDataType +import org.apache.calcite.rel.{RelNode, RelWriter} +import org.apache.calcite.rel.core.{Join, JoinRelType} +import org.apache.calcite.rex.RexNode + +import scala.collection.JavaConverters._ + +/** + * Stream physical node for the LATERAL SNAPSHOT processing-time temporal table join. The build side + * is loaded into operator state during a LOAD phase; once the build-side watermark crosses the + * configured flip point, the operator switches to a JOIN phase and processes probe-side records + * against the loaded build state. + */ +class StreamPhysicalLateralSnapshotJoin( + cluster: RelOptCluster, + traitSet: RelTraitSet, + leftRel: RelNode, + rightRel: RelNode, + condition: RexNode, + joinType: JoinRelType, + loadCompletedCondition: String, + loadCompletedTime: java.lang.Long, + loadCompletedIdleTimeoutMs: java.lang.Long, + stateTtlMs: java.lang.Long) + extends CommonPhysicalJoin(cluster, traitSet, leftRel, rightRel, condition, joinType) + with StreamPhysicalRel { + + require(loadCompletedTime != null, "loadCompletedTime must not be null.") + + override def requireWatermark: Boolean = true + + override def deriveRowType(): RelDataType = + LateralSnapshotJoinUtil.deriveRowType( + getCluster.getTypeFactory, + getLeft.getRowType, + getRight.getRowType, + joinType, + getSystemFieldList) + + override def copy( + traitSet: RelTraitSet, + conditionExpr: RexNode, + left: RelNode, + right: RelNode, + joinType: JoinRelType, + semiJoinDone: Boolean): Join = { + new StreamPhysicalLateralSnapshotJoin( + cluster, + traitSet, + left, + right, + conditionExpr, + joinType, + loadCompletedCondition, + loadCompletedTime, + loadCompletedIdleTimeoutMs, + stateTtlMs) + } + + override def explainTerms(pw: RelWriter): RelWriter = { + val terms = super.explainTerms(pw) + terms.item("loadCompletedCondition", loadCompletedCondition) + terms.item("loadCompletedTime", loadCompletedTime) + if (loadCompletedIdleTimeoutMs != null) { + terms.item("loadCompletedIdleTimeout", s"$loadCompletedIdleTimeoutMs ms") + } + if (stateTtlMs != null) { + terms.item("stateTtl", s"$stateTtlMs ms") + } + terms + } + + override def translateToExecNode(): ExecNode[_] = { + // The build (right) side carries a watermark, so it must expose a row-time attribute whose + // field index drives the event-time-ordered application of buffered build-side changes. + val rightTimeAttributeIndex = getRight.getRowType.getFieldList.asScala.indexWhere( + f => FlinkTypeFactory.isRowtimeIndicatorType(f.getType)) + if (rightTimeAttributeIndex < 0) { + throw new TableException( + "The build (right) side of a LATERAL SNAPSHOT join must have a row-time attribute. " + + "This is a bug, please file an issue.") + } + + new StreamExecLateralSnapshotJoin( + unwrapTableConfig(this), + joinSpec, + rightTimeAttributeIndex, + loadCompletedCondition, + loadCompletedTime, + loadCompletedIdleTimeoutMs, + stateTtlMs, + InputProperty.DEFAULT, + InputProperty.DEFAULT, + FlinkTypeFactory.toLogicalRowType(getRowType), + getRelDetailedDescription) + } +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkChangelogModeInferenceProgram.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkChangelogModeInferenceProgram.scala index 8313e4c1d96157..1c91e960a8a180 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkChangelogModeInferenceProgram.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkChangelogModeInferenceProgram.scala @@ -407,6 +407,27 @@ class FlinkChangelogModeInferenceProgram extends FlinkOptimizeProgram[StreamOpti val leftTrait = children.head.getTraitSet.getTrait(ModifyKindSetTraitDef.INSTANCE) createNewNode(temporalJoin, children, leftTrait, requiredTrait, requester) + case lateralSnapshotJoin: StreamPhysicalLateralSnapshotJoin => + // LATERAL SNAPSHOT requires append-only on the probe (left) side and supports all + // changelog modes on the build (right) side. Output is append-only. Visit the children + // individually so a rejected probe input names the probe side, not the whole operator. + val leftChild = visitChild( + lateralSnapshotJoin, + 0, + ModifyKindSetTrait.INSERT_ONLY, + "The probe (left) input of LATERAL SNAPSHOT join") + val rightChild = visitChild( + lateralSnapshotJoin, + 1, + ModifyKindSetTrait.ALL_CHANGES, + getNodeName(lateralSnapshotJoin)) + createNewNode( + lateralSnapshotJoin, + List(leftChild, rightChild), + ModifyKindSetTrait.INSERT_ONLY, + requiredTrait, + requester) + case multiJoin: StreamPhysicalMultiJoin => // multi-join supports all changes in input val children = visitChildren(multiJoin, ModifyKindSetTrait.ALL_CHANGES) @@ -740,6 +761,23 @@ class FlinkChangelogModeInferenceProgram extends FlinkOptimizeProgram[StreamOpti None } + case lateralSnapshotJoin: StreamPhysicalLateralSnapshotJoin => + // Probe (left) is required to be append-only. + // Build (right) side requires BEFORE_AND_AFTER for updates. + val left = lateralSnapshotJoin.getLeft.asInstanceOf[StreamPhysicalRel] + val right = lateralSnapshotJoin.getRight.asInstanceOf[StreamPhysicalRel] + val newLeftOption = this.visit(left, UpdateKindTrait.NONE) + val rightInputModifyKindSet = getModifyKindSet(right) + val newRightOption = this.visit(right, beforeAfterOrNone(rightInputModifyKindSet)) + (newLeftOption, newRightOption) match { + case (Some(newLeft), Some(newRight)) => + createNewNode( + lateralSnapshotJoin, + Some(List(newLeft, newRight)), + UpdateKindTrait.NONE) + case _ => None + } + // if the condition is applied on the upsert key, we can emit whatever the requiredTrait // is, because we will filter all records based on the condition that applies to that key case calc: StreamPhysicalCalcBase => @@ -1270,13 +1308,14 @@ class FlinkChangelogModeInferenceProgram extends FlinkOptimizeProgram[StreamOpti _: StreamPhysicalPythonGroupTableAggregate | _: StreamPhysicalGroupWindowAggregateBase | _: StreamPhysicalWindowAggregate | _: StreamPhysicalSort | _: StreamPhysicalRank | _: StreamPhysicalSortLimit | _: StreamPhysicalTemporalJoin | - _: StreamPhysicalCorrelateBase | _: StreamPhysicalLookupJoin | - _: StreamPhysicalWatermarkAssigner | _: StreamPhysicalWindowTableFunction | - _: StreamPhysicalWindowRank | _: StreamPhysicalWindowDeduplicate | - _: StreamPhysicalTemporalSort | _: StreamPhysicalMatch | - _: StreamPhysicalOverAggregate | _: StreamPhysicalIntervalJoin | - _: StreamPhysicalPythonOverAggregate | _: StreamPhysicalWindowJoin | - _: StreamPhysicalMLPredictTableFunction | _: StreamPhysicalVectorSearchTableFunction => + _: StreamPhysicalLateralSnapshotJoin | _: StreamPhysicalCorrelateBase | + _: StreamPhysicalLookupJoin | _: StreamPhysicalWatermarkAssigner | + _: StreamPhysicalWindowTableFunction | _: StreamPhysicalWindowRank | + _: StreamPhysicalWindowDeduplicate | _: StreamPhysicalTemporalSort | + _: StreamPhysicalMatch | _: StreamPhysicalOverAggregate | + _: StreamPhysicalIntervalJoin | _: StreamPhysicalPythonOverAggregate | + _: StreamPhysicalWindowJoin | _: StreamPhysicalMLPredictTableFunction | + _: StreamPhysicalVectorSearchTableFunction => // if not explicitly supported, all operators require full deletes if there are updates val children = rel.getInputs.map { case child: StreamPhysicalRel => diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala index bc06d2d688baac..65e6adbf772049 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala @@ -402,6 +402,9 @@ object FlinkStreamRuleSets { PushFilterInCalcIntoTableSourceScanRule.INSTANCE, // Rule that rewrites temporal join with extracted primary key TemporalJoinRewriteWithUniqueKeyRule.INSTANCE, + // Rewrites a join over a SNAPSHOT table function call into a dedicated + // FlinkLogicalLateralSnapshotJoin for the LATERAL SNAPSHOT operator. + LogicalJoinToLateralSnapshotJoinRule.INSTANCE, // Avoids accessing a field from the result (condition). PythonCalcSplitRule.SPLIT_CONDITION_REX_FIELD, // Avoids accessing a field from the result (projection). @@ -509,6 +512,7 @@ object FlinkStreamRuleSets { StreamPhysicalMultiJoinRule.INSTANCE, StreamPhysicalIntervalJoinRule.INSTANCE, StreamPhysicalTemporalJoinRule.INSTANCE, + StreamPhysicalLateralSnapshotJoinRule.INSTANCE, StreamPhysicalLookupJoinRule.SNAPSHOT_ON_TABLESCAN, StreamPhysicalLookupJoinRule.SNAPSHOT_ON_CALC_TABLESCAN, StreamPhysicalWindowJoinRule.INSTANCE, diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java index ea183bf3dbad39..29d9f0db3b9c0e 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java @@ -19,6 +19,7 @@ package org.apache.flink.table.planner.plan.nodes.exec.testutils; import org.apache.flink.table.planner.plan.nodes.exec.ExecNode; +import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecLateralSnapshotJoin; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonAsyncCalc; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonCalc; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonCorrelate; @@ -49,6 +50,11 @@ public class RestoreTestCompleteness { private static final Set>> SKIP_EXEC_NODES = new HashSet>>() { { + // TODO: FLINK-39781 - the LATERAL SNAPSHOT runtime operator is still a stub, + // so a restore test cannot generate a savepoint yet. Remove this entry and + // add LateralSnapshotJoinRestoreTest once the operator is implemented. + add(StreamExecLateralSnapshotJoin.class); + /** Ignoring python based exec nodes temporarily. */ add(StreamExecPythonCalc.class); add(StreamExecPythonCorrelate.class); diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.java index b40ad42be4e797..84377daa46e322 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.java @@ -27,17 +27,17 @@ import org.junit.jupiter.api.Test; import static org.apache.flink.core.testutils.FlinkAssertions.anyCauseMatches; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; /** * Plan tests for the SNAPSHOT built-in process table function used by the LATERAL SNAPSHOT temporal * join (FLIP-579). * - *

SNAPSHOT is a planner placeholder without a runtime implementation. These tests therefore only - * verify that a call parses, that its named arguments pass type inference, and that it survives - * logical optimization in both a plain {@code FROM} clause and a {@code LATERAL} context. Verifying - * the optimized rel plan (rather than the exec plan) stops short of the runtime translation that a - * future optimizer rule will provide by rewriting the call into a dedicated temporal-join operator. + *

These tests focus on the function surface: that a call parses, that its named arguments pass + * type inference, that PARTITION BY and the implicit system arguments are rejected, and that a + * {@code LATERAL} use is rewritten into the dedicated LATERAL SNAPSHOT join. End-to-end join + * translation and semantics are covered by {@code LateralSnapshotJoinTest}. */ public class SnapshotTableFunctionTest extends TableTestBase { @@ -65,60 +65,42 @@ void setup() { + " rate_time TIMESTAMP(3)," + " WATERMARK FOR rate_time AS rate_time" + ") WITH ('connector' = 'values')"); - // Sinks used by the execution tests below. - util.tableEnv() - .executeSql( - "CREATE TABLE RatesSink (" - + " currency STRING," - + " rate INT," - + " rate_time TIMESTAMP(3)" - + ") WITH ('connector' = 'blackhole')"); - util.tableEnv() - .executeSql( - "CREATE TABLE JoinSink (" - + " order_id INT," - + " amount INT," - + " rate INT" - + ") WITH ('connector' = 'blackhole')"); - } - - @Test - void testFromContext() { - // SNAPSHOT used as a standalone table function with the full set of named arguments - util.verifyRelPlan( - "SELECT * FROM SNAPSHOT(" - + "input => TABLE Rates, " - + "load_completed_condition => 'user_time', " - + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00.001' AS TIMESTAMP_LTZ(3)), " - + "load_completed_idle_timeout => INTERVAL '10' SECOND, " - + "state_ttl => INTERVAL '1' DAY)"); } @Test void testLateralContext() { - // SNAPSHOT used in a LATERAL context - util.verifyRelPlan( - "SELECT o.order_id, o.amount, r.rate " - + "FROM Orders AS o, LATERAL TABLE(SNAPSHOT(input => TABLE Rates)) AS r " - + "WHERE o.currency = r.currency"); + // SNAPSHOT used in a LATERAL context is rewritten into the dedicated LATERAL SNAPSHOT join. + // An explicit load_completed_time keeps the rewrite deterministic; we only assert that the + // dedicated join node appears rather than pinning the whole plan. + final String plan = + util.tableEnv() + .explainSql( + "SELECT o.order_id, o.amount, r.rate " + + "FROM Orders AS o, LATERAL TABLE(SNAPSHOT(" + + "input => TABLE Rates, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS r " + + "WHERE o.currency = r.currency"); + assertThat(plan).contains("LateralSnapshotJoin"); } @Test @Disabled( - "SNAPSHOT should disable the implicit system arguments (on_time, uid), but that is not " - + "wired up yet. disableSystemArguments(true) is only legal for a PTF that is " - + "rewritten by its own optimizer rule before reaching " - + "StreamPhysicalProcessTableFunctionRule (which otherwise rejects it with " - + "'Disabling system arguments is not supported for user-defined PTF').") + "SNAPSHOT sets disableSystemArguments(true), but that flag is currently not enforced. " + + "Re-enable once FLINK-40079 is fixed.") void testSystemArgumentsNotAllowed() { - // SNAPSHOT must disable the implicit system arguments (e.g. `on_time`). Passing one must be - // rejected because the argument is not allowed. + // SNAPSHOT disables the implicit system arguments (e.g. `on_time`). Passing one in a + // LATERAL context must be rejected because the argument is not part of the function + // signature. assertThatThrownBy( () -> util.verifyRelPlan( - "SELECT * FROM SNAPSHOT(" + "SELECT o.order_id " + + "FROM Orders AS o, LATERAL TABLE(SNAPSHOT(" + "input => TABLE Rates, " - + "on_time => DESCRIPTOR(rate_time))")) + + "on_time => DESCRIPTOR(rate_time))) AS r " + + "WHERE o.currency = r.currency")) .satisfies(anyCauseMatches("on_time")); } @@ -136,36 +118,14 @@ void testPartitionByNotAllowed() { } @Test - void testFromContextExecutionFails() { - // SNAPSHOT has no runtime implementation yet, so compiling the query into the job's - // transformations fails. A future optimizer rule is expected to rewrite the call before - // this stage. - assertThatThrownBy( - () -> - util.generateTransformations( - "INSERT INTO RatesSink " - + "SELECT * FROM SNAPSHOT(input => TABLE Rates)")) - .satisfies( - anyCauseMatches( - "Could not find a runtime implementation for built-in function 'SNAPSHOT'. " - + "The planner should have provided an implementation.")); - } - - @Test - void testLateralContextExecutionFails() { - // SNAPSHOT has no runtime implementation yet, so compiling the query into the job's - // transformations fails. A future optimizer rule is expected to rewrite the call before - // this stage. - assertThatThrownBy( - () -> - util.generateTransformations( - "INSERT INTO JoinSink " - + "SELECT o.order_id, o.amount, r.rate " - + "FROM Orders AS o, LATERAL TABLE(SNAPSHOT(input => TABLE Rates)) AS r " - + "WHERE o.currency = r.currency")) + void testFromContextRejectedOutsideLateral() { + // SNAPSHOT used outside a LATERAL context is not rewritten by the LATERAL SNAPSHOT rule and + // reaches the regular PTF physical rule. Because SNAPSHOT disables system arguments, that + // rule rejects it. FLINK-39784 will replace this with a clearer message + // ("SNAPSHOT can only be used inside a LATERAL clause"). + assertThatThrownBy(() -> util.verifyRelPlan("SELECT * FROM SNAPSHOT(input => TABLE Rates)")) .satisfies( anyCauseMatches( - "Could not find a runtime implementation for built-in function 'SNAPSHOT'. " - + "The planner should have provided an implementation.")); + "Disabling system arguments is not supported for user-defined PTF.")); } } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest.java new file mode 100644 index 00000000000000..4c3b592b96b6cf --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest.java @@ -0,0 +1,540 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.stream.sql.join; + +import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.planner.utils.TableTestBase; +import org.apache.flink.table.planner.utils.TableTestUtil; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.ZoneId; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Plan tests for the {@code LATERAL SNAPSHOT} processing-time temporal table join. */ +public class LateralSnapshotJoinTest extends TableTestBase { + + private TableTestUtil util; + + @BeforeEach + void setup() { + final TableConfig config = TableConfig.getDefault(); + config.setLocalTimeZone(ZoneId.of("UTC")); + util = streamTestUtil(config); + + util.tableEnv() + .executeSql( + "CREATE TABLE probe (" + + " pk STRING," + + " pv INT," + + " pts TIMESTAMP(3)," + + " WATERMARK FOR pts AS pts" + + ") WITH ('connector' = 'values', 'bounded' = 'false')"); + + util.tableEnv() + .executeSql( + "CREATE TABLE b (" + + " bk STRING," + + " bv INT," + + " bts TIMESTAMP(3)," + + " WATERMARK FOR bts AS bts" + + ") WITH (" + + " 'connector' = 'values'," + + " 'bounded' = 'false'," + + " 'changelog-mode' = 'I,UB,UA,D'" + + ")"); + + // Sink for the JSON-plan tests; nullable columns accept the null-padded LEFT-join build + // side. + util.tableEnv() + .executeSql( + "CREATE TABLE sink (" + + " pk STRING," + + " pv INT," + + " pts TIMESTAMP(3)," + + " bk STRING," + + " bv INT," + + " bts TIMESTAMP(3)" + + ") WITH ('connector' = 'blackhole')"); + } + + @Test + void testInnerJoin() { + util.verifyRelPlan( + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk"); + } + + @Test + void testLeftJoin() { + util.verifyRelPlan( + "SELECT * FROM probe LEFT JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk"); + } + + @Test + void testInnerJoinWithIdleTimeoutAndStateTtl() { + util.verifyRelPlan( + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)), " + + "load_completed_idle_timeout => INTERVAL '10' SECOND, " + + "state_ttl => INTERVAL '1' DAY" + + ")) AS s " + + "ON probe.pk = s.bk"); + } + + @Test + void testInnerJoinWithNonEquiCondition() { + util.verifyRelPlan( + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk AND probe.pv > s.bv"); + } + + @Test + void testInnerJoinWithCompositeKeys() { + util.verifyRelPlan( + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk AND probe.pv = s.bv"); + } + + @Test + void testInnerJoinWithTimeAttributeInCondition() { + util.verifyRelPlan( + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk AND probe.pts >= s.bts"); + } + + @Test + void testInnerJoinWithCteBuildSide() { + util.verifyRelPlan( + "WITH cte AS (SELECT bk, bv + 1 AS bv, bts FROM b) " + + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE cte, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk"); + } + + @Test + void testInnerJoinWithoutBuildTimeColumn() { + util.verifyRelPlan( + "SELECT probe.pk, probe.pv, s.bv FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk"); + } + + @Test + void testLeftJoinWithoutBuildTimeColumn() { + util.verifyRelPlan( + "SELECT probe.pk, probe.pv, s.bv FROM probe LEFT JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk"); + } + + @Test + void testBuildSideProctimeIsMaterialized() { + util.tableEnv() + .executeSql( + "CREATE TABLE b_proctime (" + + " bk STRING," + + " bv INT," + + " bts TIMESTAMP(3)," + + " pt AS PROCTIME()," + + " WATERMARK FOR bts AS bts" + + ") WITH ('connector' = 'values', 'bounded' = 'false')"); + util.verifyRelPlan( + "SELECT probe.pk, s.bk, s.bv, s.pt FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b_proctime, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk"); + } + + // ------------------------------------------------------------------------------------------ + // Behavior and compilation smoke tests + // ------------------------------------------------------------------------------------------ + + @Test + void testBuildRowtimeIsNotForwarded() { + // Derived table whose SELECT * exposes the probe time attribute as `pts` and the (now + // materialized) build time attribute as `bts`. + final String derived = + "(SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s ON probe.pk = s.bk)"; + // Windowing over the build-side time column is rejected: it is materialized, not a time + // attribute. + assertThatThrownBy( + () -> + util.verifyRelPlan( + "SELECT COUNT(*) FROM " + + derived + + " GROUP BY TUMBLE(bts, INTERVAL '1' MINUTE)")) + .hasMessageContaining("time attribute"); + // The probe-side time attribute is preserved and remains usable as event time. + util.tableEnv() + .explainSql( + "SELECT COUNT(*) FROM " + + derived + + " GROUP BY TUMBLE(pts, INTERVAL '1' MINUTE)"); + } + + @Test + void testInnerJoinWithUpsertBuildSourceMaterializesRetractions() { + util.tableEnv() + .executeSql( + "CREATE TABLE b_upsert (" + + " bk STRING," + + " bv INT," + + " bts TIMESTAMP(3)," + + " WATERMARK FOR bts AS bts," + + " PRIMARY KEY (bk) NOT ENFORCED" + + ") WITH (" + + " 'connector' = 'values'," + + " 'bounded' = 'false'," + + " 'changelog-mode' = 'I,UA,D'" + + ")"); + final String plan = + util.tableEnv() + .explainSql( + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b_upsert, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s ON probe.pk = s.bk"); + assertThat(plan).contains("ChangelogNormalize"); + assertThat(plan).doesNotContain("DropUpdateBefore"); + } + + @Test + void testNonEquiConditionCompilesEndToEnd() { + final String sql = + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s ON probe.pk = s.bk AND probe.pv > s.bv AND probe.pts >= s.bts"; + assertThat(util.tableEnv().explainSql(sql)).contains("LateralSnapshotJoin"); + } + + @Test + void testFoldableConstantArgs() { + final String plan = + util.tableEnv() + .explainSql( + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s ON probe.pk = s.bk"); + assertThat(plan).contains("LateralSnapshotJoin"); + } + + @Test + void testInnerJoinWithDefaultCompileTime_compilesEndToEnd() { + final String sql = + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(input => TABLE b)) AS s " + + "ON probe.pk = s.bk"; + // Compile via the table environment without verifying the plan XML (since + // load_completed_time embeds wall-clock millis at planning). + assertThat(util.tableEnv().explainSql(sql)) + .contains("LateralSnapshotJoin") + .contains("loadCompletedCondition=[compile_time]") + .contains("joinType=[InnerJoin]") + .contains("where=[=(pk, bk)]"); + } + + @Test + void testInnerJoinWithExplicitCompileTime_compilesEndToEnd() { + final String sql = + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'compile_time'" + + ")) AS s ON probe.pk = s.bk"; + assertThat(util.tableEnv().explainSql(sql)) + .contains("LateralSnapshotJoin") + .contains("loadCompletedCondition=[compile_time]") + .contains("joinType=[InnerJoin]") + .contains("where=[=(pk, bk)]"); + } + + // ------------------------------------------------------------------------------------------ + // Validation: rejection paths + // ------------------------------------------------------------------------------------------ + + @Test + void testRejectBuildSideWithoutWatermark() { + util.tableEnv() + .executeSql( + "CREATE TABLE b_no_wm (" + + " bk STRING," + + " bv INT," + + " bts TIMESTAMP(3)" + + ") WITH ('connector' = 'values', 'bounded' = 'false')"); + final String sql = + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b_no_wm, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pk = s.bk"; + assertThatThrownBy(() -> util.verifyRelPlan(sql)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "LATERAL SNAPSHOT requires a watermark on the build-side input."); + } + + @Test + void testRejectProbeSideNotAppendOnly() { + util.tableEnv() + .executeSql( + "CREATE TABLE probe_updates (" + + " pk STRING," + + " pv INT," + + " pts TIMESTAMP(3)," + + " WATERMARK FOR pts AS pts," + + " PRIMARY KEY (pk) NOT ENFORCED" + + ") WITH (" + + " 'connector' = 'values'," + + " 'bounded' = 'false'," + + " 'changelog-mode' = 'I,UB,UA,D'" + + ")"); + final String sql = + "SELECT * FROM probe_updates JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe_updates.pk = s.bk"; + assertThatThrownBy(() -> util.verifyRelPlan(sql)) + .hasMessageContaining( + "The probe (left) input of LATERAL SNAPSHOT join doesn't support " + + "consuming update and delete changes"); + } + + @Test + void testRejectMissingEqualityPredicate() { + final String sql = + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s " + + "ON probe.pv > s.bv"; + assertThatThrownBy(() -> util.verifyRelPlan(sql)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "LATERAL SNAPSHOT join requires at least one equality predicate."); + } + + @Test + void testRejectNonConstantCondition() { + final String sql = + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => CAST(CURRENT_TIMESTAMP AS STRING)" + + ")) AS s ON probe.pk = s.bk"; + assertThatThrownBy(() -> util.verifyRelPlan(sql)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("Invalid function call") + .hasMessageContaining("SNAPSHOT"); + } + + @Test + void testRejectNonConstantLoadCompletedTime() { + final String sql = + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CURRENT_TIMESTAMP" + + ")) AS s ON probe.pk = s.bk"; + assertThatThrownBy(() -> util.verifyRelPlan(sql)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "Argument 'load_completed_time' of SNAPSHOT must be a constant expression"); + } + + @Test + void testRejectNonConstantIdleTimeout() { + final String sql = + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)), " + + "load_completed_idle_timeout => " + + "CASE WHEN RAND() > 0.5 THEN INTERVAL '10' SECOND ELSE INTERVAL '20' SECOND END" + + ")) AS s ON probe.pk = s.bk"; + assertThatThrownBy(() -> util.verifyRelPlan(sql)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "Argument 'load_completed_idle_timeout' of SNAPSHOT must be a constant expression"); + } + + @Test + void testRejectNonConstantStateTtl() { + final String sql = + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)), " + + "state_ttl => " + + "CASE WHEN RAND() > 0.5 THEN INTERVAL '1' DAY ELSE INTERVAL '2' DAY END" + + ")) AS s ON probe.pk = s.bk"; + assertThatThrownBy(() -> util.verifyRelPlan(sql)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "Argument 'state_ttl' of SNAPSHOT must be a constant expression"); + } + + @Test + void testRejectYearMonthIntervalStateTtl() { + final String sql = + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)), " + + "state_ttl => INTERVAL '1' YEAR" + + ")) AS s ON probe.pk = s.bk"; + assertThatThrownBy(() -> util.verifyRelPlan(sql)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("No match found for function signature SNAPSHOT"); + } + + @Test + void testRejectNegativeIdleTimeout() { + final String sql = + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)), " + + "load_completed_idle_timeout => INTERVAL -'10' SECOND" + + ")) AS s ON probe.pk = s.bk"; + assertThatThrownBy(() -> util.verifyRelPlan(sql)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "Argument 'load_completed_idle_timeout' of SNAPSHOT must not be negative"); + } + + @Test + void testRejectNegativeStateTtl() { + final String sql = + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)), " + + "state_ttl => INTERVAL -'10' MINUTE" + + ")) AS s ON probe.pk = s.bk"; + assertThatThrownBy(() -> util.verifyRelPlan(sql)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("Argument 'state_ttl' of SNAPSHOT must not be negative"); + } + + // ------------------------------------------------------------------------------------------ + // Exec-node (JSON plan) serialization round-trips — verify StreamExecLateralSnapshotJoin + // compiles to a CompiledPlan and back, without executing the operator. + // ------------------------------------------------------------------------------------------ + + @Test + void testInnerJoinJsonPlan() { + util.verifyJsonPlan( + "INSERT INTO sink SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s ON probe.pk = s.bk"); + } + + @Test + void testLeftJoinJsonPlan() { + util.verifyJsonPlan( + "INSERT INTO sink SELECT * FROM probe LEFT JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s ON probe.pk = s.bk"); + } + + @Test + void testInnerJoinWithIdleTimeoutAndStateTtlJsonPlan() { + util.verifyJsonPlan( + "INSERT INTO sink SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)), " + + "load_completed_idle_timeout => INTERVAL '10' SECOND, " + + "state_ttl => INTERVAL '1' DAY" + + ")) AS s ON probe.pk = s.bk"); + } + + @Test + void testInnerJoinWithCompositeKeysJsonPlan() { + util.verifyJsonPlan( + "INSERT INTO sink SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s ON probe.pk = s.bk AND probe.pv = s.bv"); + } + + @Test + void testInnerJoinWithNonEquiConditionJsonPlan() { + util.verifyJsonPlan( + "INSERT INTO sink SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3))" + + ")) AS s ON probe.pk = s.bk AND probe.pv > s.bv"); + } +} diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.xml index 290410f2e41659..6108d41340ab0f 100644 --- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.xml +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.xml @@ -16,58 +16,4 @@ See the License for the specific language governing permissions and limitations under the License. --> - - - TABLE Rates, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00.001' AS TIMESTAMP_LTZ(3)), load_completed_idle_timeout => INTERVAL '10' SECOND, state_ttl => INTERVAL '1' DAY)]]> - - - - - - - - - - - TABLE Rates)) AS r WHERE o.currency = r.currency]]> - - - - - - - - diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest.xml new file mode 100644 index 00000000000000..4305878729bd43 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest.xml @@ -0,0 +1,310 @@ + + + + + + TABLE b_proctime, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk]]> + + + + + + + + + + + TABLE b, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk]]> + + + + + + + + + + + TABLE b, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk AND probe.pv = s.bv]]> + + + + + + + + + + + TABLE cte, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk]]> + + + + + + + + + + + TABLE b, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)), load_completed_idle_timeout => INTERVAL '10' SECOND, state_ttl => INTERVAL '1' DAY)) AS s ON probe.pk = s.bk]]> + + + + + + + + + + + TABLE b, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk AND probe.pv > s.bv]]> + + + ($1, $4))], joinType=[inner]) + :- LogicalWatermarkAssigner(rowtime=[pts], watermark=[$2]) + : +- LogicalTableScan(table=[[default_catalog, default_database, probe]]) + +- LogicalTableFunctionScan(invocation=[SNAPSHOT(TABLE(#0), _UTF-16LE'user_time', CAST(2026-07-01 00:00:00):TIMESTAMP_WITH_LOCAL_TIME_ZONE(3) NOT NULL, DEFAULT(), DEFAULT())], rowType=[RecordType(VARCHAR(2147483647) bk, INTEGER bv, TIMESTAMP(3) bts)]) + +- LogicalProject(bk=[$0], bv=[$1], bts=[$2]) + +- LogicalWatermarkAssigner(rowtime=[bts], watermark=[$2]) + +- LogicalTableScan(table=[[default_catalog, default_database, b]]) +]]> + + + (pv, bv))], select=[pk, pv, pts, bk, bv, bts], loadCompletedCondition=[user_time], loadCompletedTime=[1782864000000]) +:- Exchange(distribution=[hash[pk]]) +: +- WatermarkAssigner(rowtime=[pts], watermark=[pts]) +: +- TableSourceScan(table=[[default_catalog, default_database, probe]], fields=[pk, pv, pts]) ++- Exchange(distribution=[hash[bk]]) + +- WatermarkAssigner(rowtime=[bts], watermark=[bts]) + +- TableSourceScan(table=[[default_catalog, default_database, b]], fields=[bk, bv, bts]) +]]> + + + + + TABLE b, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk AND probe.pts >= s.bts]]> + + + =($2, $5))], joinType=[inner]) + :- LogicalWatermarkAssigner(rowtime=[pts], watermark=[$2]) + : +- LogicalTableScan(table=[[default_catalog, default_database, probe]]) + +- LogicalTableFunctionScan(invocation=[SNAPSHOT(TABLE(#0), _UTF-16LE'user_time', CAST(2026-07-01 00:00:00):TIMESTAMP_WITH_LOCAL_TIME_ZONE(3) NOT NULL, DEFAULT(), DEFAULT())], rowType=[RecordType(VARCHAR(2147483647) bk, INTEGER bv, TIMESTAMP(3) bts)]) + +- LogicalProject(bk=[$0], bv=[$1], bts=[$2]) + +- LogicalWatermarkAssigner(rowtime=[bts], watermark=[$2]) + +- LogicalTableScan(table=[[default_catalog, default_database, b]]) +]]> + + + =(pts, bts))], select=[pk, pv, pts, bk, bv, bts], loadCompletedCondition=[user_time], loadCompletedTime=[1782864000000]) +:- Exchange(distribution=[hash[pk]]) +: +- WatermarkAssigner(rowtime=[pts], watermark=[pts]) +: +- TableSourceScan(table=[[default_catalog, default_database, probe]], fields=[pk, pv, pts]) ++- Exchange(distribution=[hash[bk]]) + +- WatermarkAssigner(rowtime=[bts], watermark=[bts]) + +- TableSourceScan(table=[[default_catalog, default_database, b]], fields=[bk, bv, bts]) +]]> + + + + + TABLE b, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk]]> + + + + + + + + + + + TABLE b, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk]]> + + + + + + + + + + + TABLE b, load_completed_condition => 'user_time', load_completed_time => CAST(TIMESTAMP '2026-07-01 00:00:00' AS TIMESTAMP_LTZ(3)))) AS s ON probe.pk = s.bk]]> + + + + + + + + + diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinJsonPlan.out b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinJsonPlan.out new file mode 100644 index 00000000000000..66b52a344b2608 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinJsonPlan.out @@ -0,0 +1,396 @@ +{ + "flinkVersion" : "", + "nodes" : [ { + "id" : 1, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`probe`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "pts", + "dataType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "pts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`pts`" + } + } ] + }, + "options" : { + "bounded" : "false", + "connector" : "values" + } + } + } + }, + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `pts` TIMESTAMP(3)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, probe]], fields=[pk, pv, pts])" + }, { + "id" : 2, + "type" : "stream-exec-watermark-assigner_1", + "watermarkExpr" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "rowtimeFieldIndex" : 2, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "WatermarkAssigner(rowtime=[pts], watermark=[pts])" + }, { + "id" : 3, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[pk]])" + }, { + "id" : 4, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`b`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts", + "dataType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "bts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`bts`" + } + } ] + }, + "options" : { + "bounded" : "false", + "changelog-mode" : "I,UB,UA,D", + "connector" : "values" + } + } + } + }, + "outputType" : "ROW<`bk` VARCHAR(2147483647), `bv` INT, `bts` TIMESTAMP(3)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, b]], fields=[bk, bv, bts])" + }, { + "id" : 5, + "type" : "stream-exec-watermark-assigner_1", + "watermarkExpr" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "rowtimeFieldIndex" : 2, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "WatermarkAssigner(rowtime=[bts], watermark=[bts])" + }, { + "id" : 6, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[bk]])" + }, { + "id" : 7, + "type" : "stream-exec-lateral-snapshot-join_1", + "joinSpec" : { + "joinType" : "INNER", + "leftKeys" : [ 0 ], + "rightKeys" : [ 0 ], + "filterNulls" : [ true ], + "nonEquiCondition" : null + }, + "rightTimeAttributeIndex" : 2, + "loadCompletedCondition" : "user_time", + "loadCompletedTime" : 1782864000000, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + }, { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : "TIMESTAMP(3)" + } ] + }, + "description" : "LateralSnapshotJoin(joinType=[InnerJoin], where=[(pk = bk)], select=[pk, pv, pts, bk, bv, bts], loadCompletedCondition=[user_time], loadCompletedTime=[1782864000000])" + }, { + "id" : 8, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "pts", + "dataType" : "TIMESTAMP(3)" + }, { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts", + "dataType" : "TIMESTAMP(3)" + } ] + }, + "options" : { + "connector" : "blackhole" + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : "TIMESTAMP(3)" + } ] + }, + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[pk, pv, pts, bk, bv, bts])" + } ], + "edges" : [ { + "source" : 1, + "target" : 2, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 2, + "target" : 3, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 4, + "target" : 5, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 5, + "target" : 6, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 3, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 6, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 7, + "target" : 8, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinWithCompositeKeysJsonPlan.out b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinWithCompositeKeysJsonPlan.out new file mode 100644 index 00000000000000..d9413bf814a3c7 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinWithCompositeKeysJsonPlan.out @@ -0,0 +1,396 @@ +{ + "flinkVersion" : "", + "nodes" : [ { + "id" : 1, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`probe`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "pts", + "dataType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "pts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`pts`" + } + } ] + }, + "options" : { + "bounded" : "false", + "connector" : "values" + } + } + } + }, + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `pts` TIMESTAMP(3)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, probe]], fields=[pk, pv, pts])" + }, { + "id" : 2, + "type" : "stream-exec-watermark-assigner_1", + "watermarkExpr" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "rowtimeFieldIndex" : 2, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "WatermarkAssigner(rowtime=[pts], watermark=[pts])" + }, { + "id" : 3, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0, 1 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[pk, pv]])" + }, { + "id" : 4, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`b`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts", + "dataType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "bts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`bts`" + } + } ] + }, + "options" : { + "bounded" : "false", + "changelog-mode" : "I,UB,UA,D", + "connector" : "values" + } + } + } + }, + "outputType" : "ROW<`bk` VARCHAR(2147483647), `bv` INT, `bts` TIMESTAMP(3)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, b]], fields=[bk, bv, bts])" + }, { + "id" : 5, + "type" : "stream-exec-watermark-assigner_1", + "watermarkExpr" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "rowtimeFieldIndex" : 2, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "WatermarkAssigner(rowtime=[bts], watermark=[bts])" + }, { + "id" : 6, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0, 1 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[bk, bv]])" + }, { + "id" : 7, + "type" : "stream-exec-lateral-snapshot-join_1", + "joinSpec" : { + "joinType" : "INNER", + "leftKeys" : [ 0, 1 ], + "rightKeys" : [ 0, 1 ], + "filterNulls" : [ true, true ], + "nonEquiCondition" : null + }, + "rightTimeAttributeIndex" : 2, + "loadCompletedCondition" : "user_time", + "loadCompletedTime" : 1782864000000, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + }, { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : "TIMESTAMP(3)" + } ] + }, + "description" : "LateralSnapshotJoin(joinType=[InnerJoin], where=[((pk = bk) AND (pv = bv))], select=[pk, pv, pts, bk, bv, bts], loadCompletedCondition=[user_time], loadCompletedTime=[1782864000000])" + }, { + "id" : 8, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "pts", + "dataType" : "TIMESTAMP(3)" + }, { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts", + "dataType" : "TIMESTAMP(3)" + } ] + }, + "options" : { + "connector" : "blackhole" + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : "TIMESTAMP(3)" + } ] + }, + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[pk, pv, pts, bk, bv, bts])" + } ], + "edges" : [ { + "source" : 1, + "target" : 2, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 2, + "target" : 3, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 4, + "target" : 5, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 5, + "target" : 6, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 3, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 6, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 7, + "target" : 8, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinWithIdleTimeoutAndStateTtlJsonPlan.out b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinWithIdleTimeoutAndStateTtlJsonPlan.out new file mode 100644 index 00000000000000..e4481719e4c6d8 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinWithIdleTimeoutAndStateTtlJsonPlan.out @@ -0,0 +1,398 @@ +{ + "flinkVersion" : "", + "nodes" : [ { + "id" : 1, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`probe`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "pts", + "dataType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "pts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`pts`" + } + } ] + }, + "options" : { + "bounded" : "false", + "connector" : "values" + } + } + } + }, + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `pts` TIMESTAMP(3)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, probe]], fields=[pk, pv, pts])" + }, { + "id" : 2, + "type" : "stream-exec-watermark-assigner_1", + "watermarkExpr" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "rowtimeFieldIndex" : 2, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "WatermarkAssigner(rowtime=[pts], watermark=[pts])" + }, { + "id" : 3, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[pk]])" + }, { + "id" : 4, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`b`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts", + "dataType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "bts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`bts`" + } + } ] + }, + "options" : { + "bounded" : "false", + "changelog-mode" : "I,UB,UA,D", + "connector" : "values" + } + } + } + }, + "outputType" : "ROW<`bk` VARCHAR(2147483647), `bv` INT, `bts` TIMESTAMP(3)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, b]], fields=[bk, bv, bts])" + }, { + "id" : 5, + "type" : "stream-exec-watermark-assigner_1", + "watermarkExpr" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "rowtimeFieldIndex" : 2, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "WatermarkAssigner(rowtime=[bts], watermark=[bts])" + }, { + "id" : 6, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[bk]])" + }, { + "id" : 7, + "type" : "stream-exec-lateral-snapshot-join_1", + "joinSpec" : { + "joinType" : "INNER", + "leftKeys" : [ 0 ], + "rightKeys" : [ 0 ], + "filterNulls" : [ true ], + "nonEquiCondition" : null + }, + "rightTimeAttributeIndex" : 2, + "loadCompletedCondition" : "user_time", + "loadCompletedTime" : 1782864000000, + "loadCompletedIdleTimeoutMs" : 10000, + "stateTtlMs" : 86400000, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + }, { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : "TIMESTAMP(3)" + } ] + }, + "description" : "LateralSnapshotJoin(joinType=[InnerJoin], where=[(pk = bk)], select=[pk, pv, pts, bk, bv, bts], loadCompletedCondition=[user_time], loadCompletedTime=[1782864000000], loadCompletedIdleTimeout=[10000 ms], stateTtl=[86400000 ms])" + }, { + "id" : 8, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "pts", + "dataType" : "TIMESTAMP(3)" + }, { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts", + "dataType" : "TIMESTAMP(3)" + } ] + }, + "options" : { + "connector" : "blackhole" + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : "TIMESTAMP(3)" + } ] + }, + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[pk, pv, pts, bk, bv, bts])" + } ], + "edges" : [ { + "source" : 1, + "target" : 2, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 2, + "target" : 3, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 4, + "target" : 5, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 5, + "target" : 6, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 3, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 6, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 7, + "target" : 8, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinWithNonEquiConditionJsonPlan.out b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinWithNonEquiConditionJsonPlan.out new file mode 100644 index 00000000000000..412957c8b1f59e --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testInnerJoinWithNonEquiConditionJsonPlan.out @@ -0,0 +1,410 @@ +{ + "flinkVersion" : "", + "nodes" : [ { + "id" : 1, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`probe`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "pts", + "dataType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "pts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`pts`" + } + } ] + }, + "options" : { + "bounded" : "false", + "connector" : "values" + } + } + } + }, + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `pts` TIMESTAMP(3)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, probe]], fields=[pk, pv, pts])" + }, { + "id" : 2, + "type" : "stream-exec-watermark-assigner_1", + "watermarkExpr" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "rowtimeFieldIndex" : 2, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "WatermarkAssigner(rowtime=[pts], watermark=[pts])" + }, { + "id" : 3, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[pk]])" + }, { + "id" : 4, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`b`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts", + "dataType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "bts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`bts`" + } + } ] + }, + "options" : { + "bounded" : "false", + "changelog-mode" : "I,UB,UA,D", + "connector" : "values" + } + } + } + }, + "outputType" : "ROW<`bk` VARCHAR(2147483647), `bv` INT, `bts` TIMESTAMP(3)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, b]], fields=[bk, bv, bts])" + }, { + "id" : 5, + "type" : "stream-exec-watermark-assigner_1", + "watermarkExpr" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "rowtimeFieldIndex" : 2, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "WatermarkAssigner(rowtime=[bts], watermark=[bts])" + }, { + "id" : 6, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[bk]])" + }, { + "id" : 7, + "type" : "stream-exec-lateral-snapshot-join_1", + "joinSpec" : { + "joinType" : "INNER", + "leftKeys" : [ 0 ], + "rightKeys" : [ 0 ], + "filterNulls" : [ true ], + "nonEquiCondition" : { + "kind" : "CALL", + "syntax" : "BINARY", + "internalName" : "$>$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "INT" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 4, + "type" : "INT" + } ], + "type" : "BOOLEAN" + } + }, + "rightTimeAttributeIndex" : 2, + "loadCompletedCondition" : "user_time", + "loadCompletedTime" : 1782864000000, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + }, { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : "TIMESTAMP(3)" + } ] + }, + "description" : "LateralSnapshotJoin(joinType=[InnerJoin], where=[((pk = bk) AND (pv > bv))], select=[pk, pv, pts, bk, bv, bts], loadCompletedCondition=[user_time], loadCompletedTime=[1782864000000])" + }, { + "id" : 8, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "pts", + "dataType" : "TIMESTAMP(3)" + }, { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts", + "dataType" : "TIMESTAMP(3)" + } ] + }, + "options" : { + "connector" : "blackhole" + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : "TIMESTAMP(3)" + } ] + }, + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[pk, pv, pts, bk, bv, bts])" + } ], + "edges" : [ { + "source" : 1, + "target" : 2, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 2, + "target" : 3, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 4, + "target" : 5, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 5, + "target" : 6, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 3, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 6, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 7, + "target" : 8, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testLeftJoinJsonPlan.out b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testLeftJoinJsonPlan.out new file mode 100644 index 00000000000000..3213689d7b9797 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/LateralSnapshotJoinTest_jsonplan/testLeftJoinJsonPlan.out @@ -0,0 +1,396 @@ +{ + "flinkVersion" : "", + "nodes" : [ { + "id" : 1, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`probe`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "pts", + "dataType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "pts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`pts`" + } + } ] + }, + "options" : { + "bounded" : "false", + "connector" : "values" + } + } + } + }, + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `pts` TIMESTAMP(3)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, probe]], fields=[pk, pv, pts])" + }, { + "id" : 2, + "type" : "stream-exec-watermark-assigner_1", + "watermarkExpr" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "rowtimeFieldIndex" : 2, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "WatermarkAssigner(rowtime=[pts], watermark=[pts])" + }, { + "id" : 3, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[pk]])" + }, { + "id" : 4, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`b`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts", + "dataType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "bts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`bts`" + } + } ] + }, + "options" : { + "bounded" : "false", + "changelog-mode" : "I,UB,UA,D", + "connector" : "values" + } + } + } + }, + "outputType" : "ROW<`bk` VARCHAR(2147483647), `bv` INT, `bts` TIMESTAMP(3)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, b]], fields=[bk, bv, bts])" + }, { + "id" : 5, + "type" : "stream-exec-watermark-assigner_1", + "watermarkExpr" : { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "TIMESTAMP(3)" + }, + "rowtimeFieldIndex" : 2, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "WatermarkAssigner(rowtime=[bts], watermark=[bts])" + }, { + "id" : 6, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[bk]])" + }, { + "id" : 7, + "type" : "stream-exec-lateral-snapshot-join_1", + "joinSpec" : { + "joinType" : "LEFT", + "leftKeys" : [ 0 ], + "rightKeys" : [ 0 ], + "filterNulls" : [ true ], + "nonEquiCondition" : null + }, + "rightTimeAttributeIndex" : 2, + "loadCompletedCondition" : "user_time", + "loadCompletedTime" : 1782864000000, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + }, { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : "TIMESTAMP(3)" + } ] + }, + "description" : "LateralSnapshotJoin(joinType=[LeftOuterJoin], where=[(pk = bk)], select=[pk, pv, pts, bk, bv, bts], loadCompletedCondition=[user_time], loadCompletedTime=[1782864000000])" + }, { + "id" : 8, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "pts", + "dataType" : "TIMESTAMP(3)" + }, { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts", + "dataType" : "TIMESTAMP(3)" + } ] + }, + "options" : { + "connector" : "blackhole" + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "pk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "fieldType" : "INT" + }, { + "name" : "pts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : "TIMESTAMP(3)" + } ] + }, + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[pk, pv, pts, bk, bv, bts])" + } ], + "edges" : [ { + "source" : 1, + "target" : 2, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 2, + "target" : 3, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 4, + "target" : 5, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 5, + "target" : 6, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 3, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 6, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 7, + "target" : 8, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file From 14510d2a4c0d7f3b241248eae9845522ea11e39b Mon Sep 17 00:00:00 2001 From: Fabian Hueske Date: Tue, 7 Jul 2026 11:54:05 +0200 Subject: [PATCH 3/5] [FLINK-39784][table] Forbid SNAPSHOT outside a LATERAL context Adds ForbidSnapshotOutsideLateralRule, which rejects any SNAPSHOT scan that survives the LATERAL SNAPSHOT rewrite with a clear message instead of failing later in the generic PTF translation. Generated-By: Claude Opus 4.8 (1M context) --- .../ForbidSnapshotOutsideLateralRule.java | 85 +++++++++++++++++++ .../plan/rules/FlinkStreamRuleSets.scala | 3 + .../stream/sql/SnapshotTableFunctionTest.java | 12 +-- 3 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/ForbidSnapshotOutsideLateralRule.java diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/ForbidSnapshotOutsideLateralRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/ForbidSnapshotOutsideLateralRule.java new file mode 100644 index 00000000000000..95db7221528f9b --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/ForbidSnapshotOutsideLateralRule.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.rules.logical; + +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalTableFunctionScan; +import org.apache.flink.table.planner.plan.utils.LateralSnapshotJoinUtil; + +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rex.RexCall; +import org.immutables.value.Value; + +/** + * Rejects any {@link FlinkLogicalTableFunctionScan} that is still backed by the built-in {@code + * SNAPSHOT} function, with a clear error message. + * + *

{@code SNAPSHOT} is a planner placeholder that is only valid as the build side of a {@code + * LATERAL} join, where {@link LogicalJoinToLateralSnapshotJoinRule} rewrites the surrounding join + * into a dedicated {@link + * org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalLateralSnapshotJoin} and removes + * the SNAPSHOT scan. This rule must therefore run after that rewrite (see {@code + * FlinkStreamRuleSets.LOGICAL_REWRITE}). + */ +@Value.Enclosing +public class ForbidSnapshotOutsideLateralRule + extends RelRule { + + public static final ForbidSnapshotOutsideLateralRule INSTANCE = + ForbidSnapshotOutsideLateralRule.ForbidSnapshotOutsideLateralRuleConfig.DEFAULT + .toRule(); + + private ForbidSnapshotOutsideLateralRule(ForbidSnapshotOutsideLateralRuleConfig config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + final FlinkLogicalTableFunctionScan scan = call.rel(0); + return scan.getCall() instanceof RexCall + && LateralSnapshotJoinUtil.isSnapshotCall((RexCall) scan.getCall()); + } + + @Override + public void onMatch(RelOptRuleCall call) { + throw new ValidationException( + "The SNAPSHOT function can only be used as the build side (right-hand side) of a " + + "LATERAL join. It cannot be used as a standalone table function or " + + "outside of a LATERAL context."); + } + + /** Rule configuration. */ + @Value.Immutable(singleton = false) + public interface ForbidSnapshotOutsideLateralRuleConfig extends RelRule.Config { + + ForbidSnapshotOutsideLateralRule.ForbidSnapshotOutsideLateralRuleConfig DEFAULT = + ImmutableForbidSnapshotOutsideLateralRule.ForbidSnapshotOutsideLateralRuleConfig + .builder() + .build() + .withOperandSupplier( + b0 -> b0.operand(FlinkLogicalTableFunctionScan.class).anyInputs()) + .withDescription("ForbidSnapshotOutsideLateralRule"); + + @Override + default ForbidSnapshotOutsideLateralRule toRule() { + return new ForbidSnapshotOutsideLateralRule(this); + } + } +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala index 65e6adbf772049..284b1dbc056dbb 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala @@ -405,6 +405,9 @@ object FlinkStreamRuleSets { // Rewrites a join over a SNAPSHOT table function call into a dedicated // FlinkLogicalLateralSnapshotJoin for the LATERAL SNAPSHOT operator. LogicalJoinToLateralSnapshotJoinRule.INSTANCE, + // Rejects SNAPSHOT scans that survived the rewrite above, i.e. SNAPSHOT calls used outside a + // LATERAL context. Must run after LogicalJoinToLateralSnapshotJoinRule. + ForbidSnapshotOutsideLateralRule.INSTANCE, // Avoids accessing a field from the result (condition). PythonCalcSplitRule.SPLIT_CONDITION_REX_FIELD, // Avoids accessing a field from the result (projection). diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.java index 84377daa46e322..da435f28eef3c8 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/SnapshotTableFunctionTest.java @@ -19,6 +19,7 @@ package org.apache.flink.table.planner.plan.stream.sql; import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.planner.utils.TableTestBase; import org.apache.flink.table.planner.utils.TableTestUtil; @@ -119,13 +120,14 @@ void testPartitionByNotAllowed() { @Test void testFromContextRejectedOutsideLateral() { - // SNAPSHOT used outside a LATERAL context is not rewritten by the LATERAL SNAPSHOT rule and - // reaches the regular PTF physical rule. Because SNAPSHOT disables system arguments, that - // rule rejects it. FLINK-39784 will replace this with a clearer message - // ("SNAPSHOT can only be used inside a LATERAL clause"). + // SNAPSHOT used outside a LATERAL context is not rewritten by the LATERAL SNAPSHOT rule. + // ForbidSnapshotOutsideLateralRule intercepts the surviving SNAPSHOT scan and rejects it + // with a clear message before it reaches the generic PTF translation. assertThatThrownBy(() -> util.verifyRelPlan("SELECT * FROM SNAPSHOT(input => TABLE Rates)")) .satisfies( anyCauseMatches( - "Disabling system arguments is not supported for user-defined PTF.")); + ValidationException.class, + "The SNAPSHOT function can only be used as the build side " + + "(right-hand side) of a LATERAL join")); } } From 928c48cb7fd6aaa58c009a46916db1ba315e7b5e Mon Sep 17 00:00:00 2001 From: Fabian Hueske Date: Wed, 8 Jul 2026 20:14:01 +0200 Subject: [PATCH 4/5] [FLINK-39785][table] Honor source.sleep-* in TestValues watermark-push-down source Wire the existing source.sleep-after-elements / source.sleep-time options into TestValuesScanTableSourceWithWatermarkPushDown Generated-By: Claude Opus 4.8 (1M context) --- .../factories/TestValuesRuntimeFunctions.java | 18 +++++++++++++- .../factories/TestValuesTableFactory.java | 24 +++++++++++++++---- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesRuntimeFunctions.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesRuntimeFunctions.java index df439bd2c9e7f3..851275d4ef804a 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesRuntimeFunctions.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesRuntimeFunctions.java @@ -257,15 +257,24 @@ public static class FromElementSourceFunctionWithWatermark private final TerminatingLogic terminating; + /** Sleep for {@link #sleepTimeMillis} after emitting every {@code sleepAfterElements}. */ + private final int sleepAfterElements; + + private final long sleepTimeMillis; + public FromElementSourceFunctionWithWatermark( String tableName, TypeSerializer serializer, Iterable elements, WatermarkStrategy watermarkStrategy, - TerminatingLogic terminating) + TerminatingLogic terminating, + int sleepAfterElements, + long sleepTimeMillis) throws IOException { this.tableName = tableName; this.terminating = terminating; + this.sleepAfterElements = sleepAfterElements; + this.sleepTimeMillis = sleepTimeMillis; ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(baos); @@ -325,6 +334,13 @@ public RelativeClock getInputActivityClock() { generator.onEvent(next, Long.MIN_VALUE, output); generator.onPeriodicEmit(output); } + + // If enabled, throttle emission of values + if (sleepAfterElements > 0 + && sleepTimeMillis > 0 + && numElementsEmitted % sleepAfterElements == 0) { + Thread.sleep(sleepTimeMillis); + } } if (terminating == TerminatingLogic.INFINITE) { diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java index 3387f522232e44..4e260a3e75fc6c 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java @@ -716,7 +716,9 @@ public DynamicTableSource createDynamicTableSource(Context context) { partitions, readableMetadata, null, - enableAggregatePushDown); + enableAggregatePushDown, + sleepAfterElements, + sleepTimeMillis); source.setEnableMetadataFilterPushDown(enableMetadataFilterPushDown); return source; } else { @@ -1750,6 +1752,8 @@ private static class TestValuesScanTableSourceWithWatermarkPushDown extends TestValuesScanTableSource implements SupportsWatermarkPushDown, SupportsSourceWatermark { private final String tableName; + private final int sleepAfterElements; + private final long sleepTimeMillis; private WatermarkStrategy watermarkStrategy = WatermarkStrategy.noWatermarks(); @@ -1771,7 +1775,9 @@ private TestValuesScanTableSourceWithWatermarkPushDown( List> allPartitions, Map readableMetadata, @Nullable int[] projectedMetadataFields, - boolean enableAggregatePushDown) { + boolean enableAggregatePushDown, + int sleepAfterElements, + long sleepTimeMillis) { super( producedDataType, changelogMode, @@ -1792,6 +1798,8 @@ private TestValuesScanTableSourceWithWatermarkPushDown( projectedMetadataFields, enableAggregatePushDown); this.tableName = tableName; + this.sleepAfterElements = sleepAfterElements; + this.sleepTimeMillis = sleepTimeMillis; } @Override @@ -1817,7 +1825,13 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext runtimeProviderCon try { return SourceFunctionProvider.of( new TestValuesRuntimeFunctions.FromElementSourceFunctionWithWatermark( - tableName, serializer, values, watermarkStrategy, terminating), + tableName, + serializer, + values, + watermarkStrategy, + terminating, + sleepAfterElements, + sleepTimeMillis), false); } catch (IOException e) { throw new TableException("Fail to init source function", e); @@ -1845,7 +1859,9 @@ public DynamicTableSource copy() { allPartitions, readableMetadata, projectedMetadataFields, - enableAggregatePushDown); + enableAggregatePushDown, + sleepAfterElements, + sleepTimeMillis); newSource.watermarkStrategy = watermarkStrategy; newSource.setEnableMetadataFilterPushDown(enableMetadataFilterPushDown); return newSource; From 9a486a1c76e9589d8b690f1173d81a4b660919d4 Mon Sep 17 00:00:00 2001 From: Fabian Hueske Date: Thu, 9 Jul 2026 12:19:23 +0200 Subject: [PATCH 5/5] [FLINK-39785][table] Add LATERAL SNAPSHOT e2e and restore tests Adds end-to-end coverage for the LATERAL SNAPSHOT processing-time temporal join: * LateralSnapshotJoinITCase: result tests over HEAP and ROCKSDB backends covering different configurations and input scenarios * LateralSnapshotJoinRestoreTest / LateralSnapshotJoinTestPrograms: savepoint restore tests Generated-By: Claude Opus 4.8 (1M context) --- .../LateralSnapshotJoinRestoreTest.java | 40 ++ .../LateralSnapshotJoinTestPrograms.java | 145 +++++ .../testutils/RestoreTestCompleteness.java | 6 - .../sql/join/LateralSnapshotJoinITCase.java | 606 ++++++++++++++++++ .../plan/lateral-snapshot-join-inner.json | 469 ++++++++++++++ .../savepoint/_metadata | Bin 0 -> 19149 bytes .../plan/lateral-snapshot-join-left.json | 468 ++++++++++++++ .../savepoint/_metadata | Bin 0 -> 19163 bytes 8 files changed, 1728 insertions(+), 6 deletions(-) create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinRestoreTest.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinTestPrograms.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/join/LateralSnapshotJoinITCase.java create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-inner/plan/lateral-snapshot-join-inner.json create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-inner/savepoint/_metadata create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-left/plan/lateral-snapshot-join-left.json create mode 100644 flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-left/savepoint/_metadata diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinRestoreTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinRestoreTest.java new file mode 100644 index 00000000000000..5a4fdf644c49ce --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinRestoreTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.planner.plan.nodes.exec.testutils.RestoreTestBase; +import org.apache.flink.table.test.program.TableTestProgram; + +import java.util.Arrays; +import java.util.List; + +/** Restore tests for {@link StreamExecLateralSnapshotJoin}. */ +public class LateralSnapshotJoinRestoreTest extends RestoreTestBase { + + public LateralSnapshotJoinRestoreTest() { + super(StreamExecLateralSnapshotJoin.class); + } + + @Override + public List programs() { + return Arrays.asList( + LateralSnapshotJoinTestPrograms.LATERAL_SNAPSHOT_JOIN_INNER, + LateralSnapshotJoinTestPrograms.LATERAL_SNAPSHOT_JOIN_LEFT); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinTestPrograms.java new file mode 100644 index 00000000000000..9af4e9049d1d27 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/LateralSnapshotJoinTestPrograms.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.nodes.exec.stream; + +import org.apache.flink.table.api.config.TableConfigOptions; +import org.apache.flink.table.test.program.SinkTestStep; +import org.apache.flink.table.test.program.SourceTestStep; +import org.apache.flink.table.test.program.TableTestProgram; +import org.apache.flink.types.Row; + +/** + * {@link TableTestProgram} definitions for testing {@link StreamExecLateralSnapshotJoin}. + * + *

Both programs load the whole build side before the savepoint: the {@code 'user_time'} gate at + * {@code 00:00:03} is reached by the last build-side row, so the operator has flipped to the JOIN + * phase and its build-side snapshot is materialized and frozen when the stop-with-savepoint fires + * (after the "before restore" rows have been joined). After restore, no further build changes + * arrive; the "after restore" probe rows join the restored snapshot, which verifies that the + * materialized build state and the LOAD/JOIN phase (union operator state) survive the savepoint. + */ +public class LateralSnapshotJoinTestPrograms { + + static final String[] PROBE_SCHEMA = { + "pk STRING", + "pv INT", + "pts_str STRING", + "pts AS TO_TIMESTAMP(pts_str)", + "WATERMARK FOR pts AS pts" + }; + + static final String[] BUILD_SCHEMA = { + "bk STRING", + "bv INT", + "bts_str STRING", + "bts AS TO_TIMESTAMP(bts_str)", + "WATERMARK FOR bts AS bts" + }; + + static final String[] SINK_SCHEMA = {"pk STRING", "pv INT", "bk STRING", "bv INT"}; + + // Two rows for key 'a' exercise the per-key multi-set; the last row's watermark (00:00:03) + // reaches the gate and flips the operator to JOIN. + static final Row[] BUILD_BEFORE_DATA = { + Row.of("a", 10, "2020-01-01 00:00:01"), + Row.of("b", 20, "2020-01-01 00:00:02"), + Row.of("a", 11, "2020-01-01 00:00:03") + }; + + static final Row[] PROBE_BEFORE_DATA = { + Row.of("a", 100, "2020-01-01 00:00:06"), Row.of("b", 200, "2020-01-01 00:00:07") + }; + + // 'a' matches the restored snapshot; 'c' has no match. + static final Row[] PROBE_AFTER_DATA = { + Row.of("a", 101, "2020-01-01 00:00:10"), Row.of("c", 300, "2020-01-01 00:00:11") + }; + + private static final String SNAPSHOT_BUILD = + "LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2020-01-01 00:00:03' AS TIMESTAMP_LTZ(3))" + + ")) AS s ON probe.pk = s.bk"; + + public static final TableTestProgram LATERAL_SNAPSHOT_JOIN_INNER = + TableTestProgram.of( + "lateral-snapshot-join-inner", + "validates a LATERAL SNAPSHOT inner join across a restore") + .setupConfig(TableConfigOptions.LOCAL_TIME_ZONE, "UTC") + .setupTableSource( + SourceTestStep.newBuilder("probe") + .addSchema(PROBE_SCHEMA) + .producedBeforeRestore(PROBE_BEFORE_DATA) + .producedAfterRestore(PROBE_AFTER_DATA) + .build()) + .setupTableSource( + SourceTestStep.newBuilder("b") + .addSchema(BUILD_SCHEMA) + .producedBeforeRestore(BUILD_BEFORE_DATA) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(SINK_SCHEMA) + .consumedBeforeRestore( + "+I[a, 100, a, 10]", + "+I[a, 100, a, 11]", + "+I[b, 200, b, 20]") + .consumedAfterRestore("+I[a, 101, a, 10]", "+I[a, 101, a, 11]") + .build()) + .runSql( + "INSERT INTO sink SELECT probe.pk, probe.pv, s.bk, s.bv " + + "FROM probe JOIN " + + SNAPSHOT_BUILD) + .build(); + + public static final TableTestProgram LATERAL_SNAPSHOT_JOIN_LEFT = + TableTestProgram.of( + "lateral-snapshot-join-left", + "validates a LATERAL SNAPSHOT left join across a restore") + .setupConfig(TableConfigOptions.LOCAL_TIME_ZONE, "UTC") + .setupTableSource( + SourceTestStep.newBuilder("probe") + .addSchema(PROBE_SCHEMA) + .producedBeforeRestore(PROBE_BEFORE_DATA) + .producedAfterRestore(PROBE_AFTER_DATA) + .build()) + .setupTableSource( + SourceTestStep.newBuilder("b") + .addSchema(BUILD_SCHEMA) + .producedBeforeRestore(BUILD_BEFORE_DATA) + .build()) + .setupTableSink( + SinkTestStep.newBuilder("sink") + .addSchema(SINK_SCHEMA) + .consumedBeforeRestore( + "+I[a, 100, a, 10]", + "+I[a, 100, a, 11]", + "+I[b, 200, b, 20]") + .consumedAfterRestore( + "+I[a, 101, a, 10]", + "+I[a, 101, a, 11]", + "+I[c, 300, null, null]") + .build()) + .runSql( + "INSERT INTO sink SELECT probe.pk, probe.pv, s.bk, s.bv " + + "FROM probe LEFT JOIN " + + SNAPSHOT_BUILD) + .build(); +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java index 29d9f0db3b9c0e..ea183bf3dbad39 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/testutils/RestoreTestCompleteness.java @@ -19,7 +19,6 @@ package org.apache.flink.table.planner.plan.nodes.exec.testutils; import org.apache.flink.table.planner.plan.nodes.exec.ExecNode; -import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecLateralSnapshotJoin; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonAsyncCalc; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonCalc; import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecPythonCorrelate; @@ -50,11 +49,6 @@ public class RestoreTestCompleteness { private static final Set>> SKIP_EXEC_NODES = new HashSet>>() { { - // TODO: FLINK-39781 - the LATERAL SNAPSHOT runtime operator is still a stub, - // so a restore test cannot generate a savepoint yet. Remove this entry and - // add LateralSnapshotJoinRestoreTest once the operator is implemented. - add(StreamExecLateralSnapshotJoin.class); - /** Ignoring python based exec nodes temporarily. */ add(StreamExecPythonCalc.class); add(StreamExecPythonCorrelate.class); diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/join/LateralSnapshotJoinITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/join/LateralSnapshotJoinITCase.java new file mode 100644 index 00000000000000..1aa873271a3f46 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/join/LateralSnapshotJoinITCase.java @@ -0,0 +1,606 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.runtime.stream.sql.join; + +import org.apache.flink.table.planner.factories.TestValuesTableFactory; +import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase; +import org.apache.flink.testutils.junit.extensions.parameterized.ParameterizedTestExtension; +import org.apache.flink.testutils.junit.extensions.parameterized.Parameters; +import org.apache.flink.types.Row; +import org.apache.flink.types.RowKind; +import org.apache.flink.util.CollectionUtil; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThatList; + +/** + * End-to-end result tests for the {@code LATERAL SNAPSHOT} processing-time temporal table join. + * + *

Processing-time semantics make the join non-deterministic in general. To get stable results, + * each build source appends a non-matching "flip-trigger" row at the gate timestamp ({@link + * #MID_GATE}) while all real build rows are earlier. With per-record ({@code on-event}) watermarks + * the operator flips to the JOIN phase. Since no more build-records are received, all probe-side + * records see the same build-side table and the append-only output can be asserted as an unordered + * set. + * + *

Some tests throttle source emission ({@code source.sleep-*}) to place probes deterministically + * around the flip. Purely processing-time-driven behavior (idle-timeout flip, state-TTL eviction) + * is covered deterministically by {@code LateralSnapshotJoinOperatorTest}. + */ +@ExtendWith(ParameterizedTestExtension.class) +public class LateralSnapshotJoinITCase extends StreamingWithStateTestBase { + + /** The {@code 'user_time'} gate reached mid-stream by the build-side flip-trigger row. */ + private static final String MID_GATE = + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2020-01-01 00:00:10' AS TIMESTAMP_LTZ(3))"; + + /** A far-future gate: the flip happens only at end of all input. */ + private static final String END_GATE = + "load_completed_condition => 'user_time', " + + "load_completed_time => CAST(TIMESTAMP '2100-01-01 00:00:00' AS TIMESTAMP_LTZ(3))"; + + /** Event time of the flip-trigger row; equal to the {@link #MID_GATE} timestamp. */ + private static final String FLIP_TRIGGER_TS = "00:00:10"; + + /** A build-side key that never matches any probe row. */ + private static final String FLIP_TRIGGER_KEY = "__flip_trigger__"; + + public LateralSnapshotJoinITCase(StateBackendMode state) { + super(state); + } + + @BeforeEach + @Override + public void before() { + super.before(); + env().setParallelism(1); + tEnv().getConfig().setLocalTimeZone(ZoneId.of("UTC")); + } + + @Parameters(name = "StateBackend={0}") + public static Collection parameters() { + return Arrays.asList( + new Object[][] { + {StreamingWithStateTestBase.HEAP_BACKEND()}, + {StreamingWithStateTestBase.ROCKSDB_BACKEND()} + }); + } + + // ------------------------------------------------------------------------------------------ + // Core join semantics + // ------------------------------------------------------------------------------------------ + + @TestTemplate + void testInnerJoin() { + // Throttle the probe so the fast build flips first; the probes are then joined live in the + // JOIN phase (the snapshot is frozen, so buffered-vs-live does not change the result). + createProbe(defaultProbe(), 40L); + createAppendBuild(defaultBuild()); + + final List actual = + collect( + "SELECT probe.pk, probe.pv, s.bk, s.bv " + + "FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_GATE + + ")) AS s ON probe.pk = s.bk"); + + assertThatList(actual) + .containsExactlyInAnyOrder( + Row.of("a", 100, "a", 10), + Row.of("a", 100, "a", 11), + Row.of("b", 200, "b", 20)); + } + + @TestTemplate + void testLeftJoin() { + createProbe(defaultProbe(), 40L); + createAppendBuild(defaultBuild()); + + final List actual = + collect( + "SELECT probe.pk, probe.pv, s.bk, s.bv " + + "FROM probe LEFT JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_GATE + + ")) AS s ON probe.pk = s.bk"); + + assertThatList(actual) + .containsExactlyInAnyOrder( + Row.of("a", 100, "a", 10), + Row.of("a", 100, "a", 11), + Row.of("b", 200, "b", 20), + Row.of("c", 300, null, null)); + } + + @TestTemplate + void testSelectStarMaterializesBuildRowtime() { + createProbe(Arrays.asList(Row.of("a", 100, ts("00:01:00")))); + createAppendBuild(Arrays.asList(Row.of("a", 10, ts("00:00:01")))); + + // SELECT * yields probe columns then build columns; the build rowtime is materialized as a + // regular TIMESTAMP(3), not a time attribute. + final List actual = + collect( + "SELECT * FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_GATE + + ")) AS s ON probe.pk = s.bk"); + + assertThatList(actual) + .containsExactlyInAnyOrder( + Row.of( + "a", + 100, + LocalDateTime.parse("2020-01-01T00:01:00"), + "a", + 10, + LocalDateTime.parse("2020-01-01T00:00:01"))); + } + + @TestTemplate + void testCompositeKeys() { + createProbe( + Arrays.asList( + Row.of("a", 10, ts("00:01:00")), + Row.of("a", 11, ts("00:01:01")), + Row.of("b", 20, ts("00:01:02")))); + createAppendBuild( + Arrays.asList( + Row.of("a", 10, ts("00:00:01")), + Row.of("a", 99, ts("00:00:02")), + Row.of("b", 20, ts("00:00:03")))); + + final List actual = + collect( + "SELECT probe.pk, probe.pv, s.bk, s.bv " + + "FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_GATE + + ")) AS s ON probe.pk = s.bk AND probe.pv = s.bv"); + + assertThatList(actual) + .containsExactlyInAnyOrder(Row.of("a", 10, "a", 10), Row.of("b", 20, "b", 20)); + } + + @TestTemplate + void testNonEquiCondition() { + createProbe(Arrays.asList(Row.of("a", 15, ts("00:01:00")), Row.of("a", 5, ts("00:01:01")))); + createAppendBuild( + Arrays.asList(Row.of("a", 10, ts("00:00:01")), Row.of("a", 20, ts("00:00:02")))); + + final List actual = + collect( + "SELECT probe.pk, probe.pv, s.bv " + + "FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_GATE + + ")) AS s ON probe.pk = s.bk AND probe.pv > s.bv"); + + assertThatList(actual).containsExactlyInAnyOrder(Row.of("a", 15, 10)); + } + + @TestTemplate + void testEmptyBuildSide() { + createProbe(defaultProbe()); + createAppendBuild(Collections.emptyList()); + + final List inner = + collect( + "SELECT probe.pk, s.bk FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_GATE + + ")) AS s ON probe.pk = s.bk"); + assertThatList(inner).isEmpty(); + + final List left = + collect( + "SELECT probe.pk, s.bk FROM probe LEFT JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_GATE + + ")) AS s ON probe.pk = s.bk"); + assertThatList(left) + .containsExactlyInAnyOrder(Row.of("a", null), Row.of("b", null), Row.of("c", null)); + } + + // ------------------------------------------------------------------------------------------ + // Build-side changelog consolidation (all changes have rowtime < the gate) + // ------------------------------------------------------------------------------------------ + + @TestTemplate + void testRetractingBuildChangelog() { + createProbe( + Arrays.asList(Row.of("a", 100, ts("00:01:00")), Row.of("b", 200, ts("00:01:01")))); + // Key a is updated 10 -> 11; key b is inserted then deleted. + createChangelogBuild( + Arrays.asList( + Row.ofKind(RowKind.INSERT, "a", 10, ts("00:00:01")), + Row.ofKind(RowKind.INSERT, "b", 20, ts("00:00:03")), + Row.ofKind(RowKind.UPDATE_BEFORE, "a", 10, ts("00:00:01")), + Row.ofKind(RowKind.UPDATE_AFTER, "a", 11, ts("00:00:02")), + Row.ofKind(RowKind.DELETE, "b", 20, ts("00:00:03")))); + + final List actual = + collect( + "SELECT probe.pk, s.bv FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_GATE + + ")) AS s ON probe.pk = s.bk"); + + assertThatList(actual).containsExactlyInAnyOrder(Row.of("a", 11)); + } + + @TestTemplate + void testUpsertBuildSource() { + createProbe( + Arrays.asList(Row.of("a", 100, ts("00:01:00")), Row.of("b", 200, ts("00:01:01")))); + // Upsert source (I,UA,D with PK): a is upserted 10 -> 11, b is deleted. + createUpsertBuild( + Arrays.asList( + Row.ofKind(RowKind.INSERT, "a", 10, ts("00:00:01")), + Row.ofKind(RowKind.UPDATE_AFTER, "a", 11, ts("00:00:02")), + Row.ofKind(RowKind.INSERT, "b", 20, ts("00:00:03")), + Row.ofKind(RowKind.DELETE, "b", 20, ts("00:00:04")))); + + final List actual = + collect( + "SELECT probe.pk, s.bv FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_GATE + + ")) AS s ON probe.pk = s.bk"); + + assertThatList(actual).containsExactlyInAnyOrder(Row.of("a", 11)); + } + + // ------------------------------------------------------------------------------------------ + // Flip timing (when LOAD flips to JOIN, and probe placement around the flip) + // ------------------------------------------------------------------------------------------ + + @TestTemplate + void testInnerJoinFlipsAtEndOfInput() { + // Far-future gate: the operator stays in LOAD and flips only at end-of-input, buffering and + // draining every probe. The frozen snapshot and result match the mid-flip case. + createProbe(defaultProbe()); + createAppendBuild(defaultBuild()); + + final List actual = + collect( + "SELECT probe.pk, probe.pv, s.bk, s.bv " + + "FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + END_GATE + + ")) AS s ON probe.pk = s.bk"); + + assertThatList(actual) + .containsExactlyInAnyOrder( + Row.of("a", 100, "a", 10), + Row.of("a", 100, "a", 11), + Row.of("b", 200, "b", 20)); + } + + @TestTemplate + void testDefaultCompileTimeCondition() { + // No options: 'load_completed_condition' defaults to 'compile_time', so the gate is the + // wall-clock time at planning. The 2020 build watermarks never reach it, so the flip + // happens at end-of-input. + createProbe(defaultProbe()); + createAppendBuild(defaultBuild()); + + final List actual = + collect( + "SELECT probe.pk, probe.pv, s.bk, s.bv " + + "FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b" + + ")) AS s ON probe.pk = s.bk"); + + assertThatList(actual) + .containsExactlyInAnyOrder( + Row.of("a", 100, "a", 10), + Row.of("a", 100, "a", 11), + Row.of("b", 200, "b", 20)); + } + + @TestTemplate + void testProbesJoinedLiveInJoinPhase() { + // Fast build flips almost immediately; the throttled probe stream is consumed entirely in + // the JOIN phase, exercising the live per-record path (including a non-matching key). The + // snapshot is static after the flip, so the result is deterministic. + final List probes = + Arrays.asList( + Row.of("a", 1, ts("00:00:01")), + Row.of("b", 2, ts("00:00:02")), + Row.of("c", 3, ts("00:00:03")), // non-matching + Row.of("a", 4, ts("00:00:04")), + Row.of("b", 5, ts("00:00:05")), + Row.of("c", 6, ts("00:00:06"))); // non-matching + createProbe(probes, 40L); + createAppendBuild( + Arrays.asList(Row.of("a", 10, ts("00:00:01")), Row.of("b", 20, ts("00:00:02")))); + + final List actual = + collect( + "SELECT probe.pv, probe.pk, s.bv FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_GATE + + ")) AS s ON probe.pk = s.bk"); + + assertThatList(actual) + .containsExactlyInAnyOrder( + Row.of(1, "a", 10), + Row.of(4, "a", 10), + Row.of(2, "b", 20), + Row.of(5, "b", 20)); + } + + @TestTemplate + void testManyProbesBufferedDuringLoadThenDrained() { + // Throttled build flips at ~240 ms, by which time the un-throttled probe stream is fully + // buffered in LOAD. The flip drains all buffered probes against the final loaded version; + // all must match and none may be lost. + final int probeCount = 20; + final List probes = new ArrayList<>(); + for (int i = 1; i <= probeCount; i++) { + probes.add(Row.of("k", i, ts(String.format("00:00:%02d", i)))); + } + createProbe(probes); // un-throttled: buffered within a few ms, long before the flip + createUpsertBuildVerbatim( + Arrays.asList( + Row.ofKind(RowKind.INSERT, "k", 1, ts("00:00:01")), + Row.ofKind(RowKind.UPDATE_AFTER, "k", 2, ts("00:00:02")), + Row.ofKind(RowKind.UPDATE_AFTER, "k", 3, ts("00:00:05")), + Row.ofKind(RowKind.INSERT, FLIP_TRIGGER_KEY, 0, ts(FLIP_TRIGGER_TS))), + 80L); // ~240 ms until the gate-crossing trigger row is emitted + + final List actual = + collect( + "SELECT probe.pv, s.bv FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_GATE + + ")) AS s ON probe.pk = s.bk"); + + assertThat(actual).hasSize(probeCount); + assertThat(actual).allSatisfy(r -> assertThat((int) r.getField(1)).isEqualTo(3)); + } + + @TestTemplate + void testProbesStraddlingFlip() { + // Throttled build loads v1, flips at ~100 ms, then applies v2 at ~150 ms. The throttled + // probe stream straddles both: the first probe is buffered then drained (sees v1) while the + // last is joined live after v2 (sees v2). Per-probe versions vary with timing, but the + // endpoints and non-decreasing order are deterministic. + final int probeCount = 8; + final List probes = new ArrayList<>(); + for (int i = 1; i <= probeCount; i++) { + probes.add(Row.of("k", i, ts(String.format("00:00:%02d", i)))); + } + createProbe(probes, 200L); // last probe at ~1400 ms, well past the post-flip update + createUpsertBuildVerbatim( + Arrays.asList( + Row.ofKind(RowKind.INSERT, "k", 1, ts("00:00:01")), + Row.ofKind(RowKind.INSERT, FLIP_TRIGGER_KEY, 0, ts(FLIP_TRIGGER_TS)), + Row.ofKind(RowKind.UPDATE_AFTER, "k", 2, ts("00:00:20"))), + 50L); + + final List byProbeId = + sortedByProbeId( + collect( + "SELECT probe.pv, s.bv FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_GATE + + ")) AS s ON probe.pk = s.bk")); + + assertThat(byProbeId).hasSize(probeCount); + assertMonotonicVersions(byProbeId); + assertThat((int) byProbeId.get(0).getField(1)).isEqualTo(1); // buffered -> v1 + assertThat((int) byProbeId.get(probeCount - 1).getField(1)).isEqualTo(2); // live -> v2 + } + + @TestTemplate + void testProbesObserveProgressiveBuildVersions() { + // Throttled build applies v2, v3, v4 progressively after the flip while a throttled probe + // stream spans the progression. Observed versions are non-decreasing; the first probe sees + // v1 and the last sees v4. Exact per-probe versions are not asserted (source timing and + // post-flip visibility latency are not precise enough across environments). + final int probeCount = 8; + final List probes = new ArrayList<>(); + for (int i = 1; i <= probeCount; i++) { + probes.add(Row.of("k", i, ts(String.format("00:00:%02d", i)))); + } + createProbe(probes, 200L); // last probe at ~1400 ms, well past the last post-flip update + createUpsertBuildVerbatim( + Arrays.asList( + Row.ofKind(RowKind.INSERT, "k", 1, ts("00:00:01")), + Row.ofKind(RowKind.INSERT, FLIP_TRIGGER_KEY, 0, ts(FLIP_TRIGGER_TS)), + Row.ofKind(RowKind.UPDATE_AFTER, "k", 2, ts("00:00:20")), + Row.ofKind(RowKind.UPDATE_AFTER, "k", 3, ts("00:00:30")), + Row.ofKind(RowKind.UPDATE_AFTER, "k", 4, ts("00:00:40"))), + 50L); + + final List byProbeId = + sortedByProbeId( + collect( + "SELECT probe.pv, s.bv FROM probe JOIN LATERAL TABLE(SNAPSHOT(" + + "input => TABLE b, " + + MID_GATE + + ")) AS s ON probe.pk = s.bk")); + + assertThat(byProbeId).hasSize(probeCount); + assertMonotonicVersions(byProbeId); + assertThat((Integer) byProbeId.get(0).getField(1)).isEqualTo(1); + assertThat((Integer) byProbeId.get(probeCount - 1).getField(1)).isEqualTo(4); + } + + // ------------------------------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------------------------------ + + private List defaultProbe() { + return Arrays.asList( + Row.of("a", 100, ts("00:01:00")), + Row.of("b", 200, ts("00:01:01")), + Row.of("c", 300, ts("00:01:02"))); + } + + private List defaultBuild() { + return Arrays.asList( + Row.of("a", 10, ts("00:00:01")), + Row.of("b", 20, ts("00:00:02")), + Row.of("a", 11, ts("00:00:03"))); + } + + private List collect(String query) { + return CollectionUtil.iteratorToList(tEnv().executeSql(query).collect()); + } + + /** + * Appends a non-matching build row at the {@link #MID_GATE} timestamp; its watermark flips the + * operator to the JOIN phase mid-stream (all real build rows are earlier). + */ + private List withFlipTrigger(List data) { + final List withTrigger = new ArrayList<>(data); + withTrigger.add(Row.of(FLIP_TRIGGER_KEY, 0, ts(FLIP_TRIGGER_TS))); + return withTrigger; + } + + private void createProbe(List data) { + createProbe(data, 0L); + } + + /** + * Creates the append-only probe table {@code probe}. {@code sleepMillis > 0} throttles emission + * to control when probe rows are consumed relative to the build-side flip. + */ + private void createProbe(List data, long sleepMillis) { + final String id = TestValuesTableFactory.registerData(data); + tEnv().executeSql( + "CREATE TABLE probe (" + + " pk STRING," + + " pv INT," + + " pts TIMESTAMP(3)," + + " WATERMARK FOR pts AS pts" + + ") WITH (" + + valuesOptions(id, sleepMillis) + + ")"); + } + + private void createAppendBuild(List data) { + final String id = TestValuesTableFactory.registerData(withFlipTrigger(data)); + tEnv().executeSql( + "CREATE TABLE b (" + + " bk STRING," + + " bv INT," + + " bts TIMESTAMP(3)," + + " WATERMARK FOR bts AS bts" + + ") WITH (" + + valuesOptions(id, 0L) + + ")"); + } + + private void createChangelogBuild(List data) { + final String id = TestValuesTableFactory.registerData(withFlipTrigger(data)); + tEnv().executeSql( + "CREATE TABLE b (" + + " bk STRING," + + " bv INT," + + " bts TIMESTAMP(3)," + + " WATERMARK FOR bts AS bts" + + ") WITH (" + + valuesOptions(id, 0L, "'changelog-mode' = 'I,UB,UA,D'") + + ")"); + } + + private void createUpsertBuild(List data) { + createUpsertBuildVerbatim(withFlipTrigger(data), 0L); + } + + /** + * Creates the upsert build table {@code b} from the given rows verbatim -- the caller places + * the flip-trigger row explicitly, so the flip and any post-flip updates can be ordered + * precisely. + */ + private void createUpsertBuildVerbatim(List data, long sleepMillis) { + final String id = TestValuesTableFactory.registerData(data); + tEnv().executeSql( + "CREATE TABLE b (" + + " bk STRING," + + " bv INT," + + " bts TIMESTAMP(3)," + + " WATERMARK FOR bts AS bts," + + " PRIMARY KEY (bk) NOT ENFORCED" + + ") WITH (" + + valuesOptions(id, sleepMillis, "'changelog-mode' = 'I,UA,D'") + + ")"); + } + + private static String valuesOptions(String id, long sleepMillis, String... extraOptions) { + final StringBuilder options = + new StringBuilder() + .append(" 'connector' = 'values',") + .append(" 'bounded' = 'false',") + .append(" 'disable-lookup' = 'true',") + .append(" 'enable-watermark-push-down' = 'true',") + .append(" 'scan.watermark.emit.strategy' = 'on-event',"); + for (String extra : extraOptions) { + options.append(" ").append(extra).append(","); + } + if (sleepMillis > 0) { + options.append(" 'source.sleep-after-elements' = '1',") + .append(" 'source.sleep-time' = '") + .append(sleepMillis) + .append("ms',"); + } + return options.append(" 'data-id' = '").append(id).append("'").toString(); + } + + /** Sorts join results (pv, bv) by the probe id pv, i.e. into probe processing order. */ + private static List sortedByProbeId(List results) { + return results.stream() + .sorted(Comparator.comparingInt(r -> (int) r.getField(0))) + .collect(Collectors.toList()); + } + + /** Asserts the build version (field 1) is non-decreasing across the probe-ordered results. */ + private static void assertMonotonicVersions(List byProbeId) { + int previous = Integer.MIN_VALUE; + for (Row r : byProbeId) { + final int version = (int) r.getField(1); + assertThat(version).isGreaterThanOrEqualTo(previous); + previous = version; + } + } + + private static LocalDateTime ts(String time) { + return LocalDateTime.parse("2020-01-01T" + time); + } +} diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-inner/plan/lateral-snapshot-join-inner.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-inner/plan/lateral-snapshot-join-inner.json new file mode 100644 index 00000000000000..eea074a15352b3 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-inner/plan/lateral-snapshot-join-inner.json @@ -0,0 +1,469 @@ +{ + "flinkVersion" : "2.4", + "nodes" : [ { + "id" : 1, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`probe`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "pts_str", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pts", + "kind" : "COMPUTED", + "expression" : { + "rexNode" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "TO_TIMESTAMP(`pts_str`)" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "pts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 3, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`pts`" + } + } ] + } + } + }, + "abilities" : [ { + "type" : "WatermarkPushDown", + "watermarkExpr" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "rowtimeExpr" : { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$REINTERPRET$1", + "operands" : [ { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + } ], + "type" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, + "idleTimeoutMillis" : -1, + "producedType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `pts_str` VARCHAR(2147483647)> NOT NULL", + "watermarkParams" : { + "emitStrategy" : "ON_EVENT", + "alignGroupName" : null, + "alignMaxDrift" : "PT0S", + "alignUpdateInterval" : "PT1S", + "sourceIdleTimeout" : -1 + } + } ] + }, + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `pts_str` VARCHAR(2147483647)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, probe, watermark=[TO_TIMESTAMP(pts_str)], watermarkEmitStrategy=[on-event]]], fields=[pk, pv, pts_str])" + }, { + "id" : 2, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "INT" + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT>", + "description" : "Calc(select=[pk, pv])" + }, { + "id" : 3, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT>", + "description" : "Exchange(distribution=[hash[pk]])" + }, { + "id" : 4, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`b`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts_str", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bts", + "kind" : "COMPUTED", + "expression" : { + "rexNode" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "TO_TIMESTAMP(`bts_str`)" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "bts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 3, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`bts`" + } + } ] + } + } + }, + "abilities" : [ { + "type" : "WatermarkPushDown", + "watermarkExpr" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "rowtimeExpr" : { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$REINTERPRET$1", + "operands" : [ { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + } ], + "type" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, + "idleTimeoutMillis" : -1, + "producedType" : "ROW<`bk` VARCHAR(2147483647), `bv` INT, `bts_str` VARCHAR(2147483647)> NOT NULL", + "watermarkParams" : { + "emitStrategy" : "ON_EVENT", + "alignGroupName" : null, + "alignMaxDrift" : "PT0S", + "alignUpdateInterval" : "PT1S", + "sourceIdleTimeout" : -1 + } + } ] + }, + "outputType" : "ROW<`bk` VARCHAR(2147483647), `bv` INT, `bts_str` VARCHAR(2147483647)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, b, watermark=[TO_TIMESTAMP(bts_str)], watermarkEmitStrategy=[on-event]]], fields=[bk, bv, bts_str])" + }, { + "id" : 5, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "INT" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$REINTERPRET$1", + "operands" : [ { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + } ], + "type" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Calc(select=[bk, bv, Reinterpret(TO_TIMESTAMP(bts_str)) AS bts])" + }, { + "id" : 6, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[bk]])" + }, { + "id" : 7, + "type" : "stream-exec-lateral-snapshot-join_1", + "joinSpec" : { + "joinType" : "INNER", + "leftKeys" : [ 0 ], + "rightKeys" : [ 0 ], + "filterNulls" : [ true ], + "nonEquiCondition" : null + }, + "rightTimeAttributeIndex" : 2, + "loadCompletedCondition" : "user_time", + "loadCompletedTime" : 1577836803000, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + }, { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `bk` VARCHAR(2147483647), `bv` INT, `bts` TIMESTAMP(3)>", + "description" : "LateralSnapshotJoin(joinType=[InnerJoin], where=[(pk = bk)], select=[pk, pv, bk, bv, bts], loadCompletedCondition=[user_time], loadCompletedTime=[1577836803000])" + }, { + "id" : 8, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "INT" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 3, + "type" : "INT" + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `bk` VARCHAR(2147483647), `bv` INT>", + "description" : "Calc(select=[pk, pv, bk, bv])" + }, { + "id" : 9, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + } ] + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "upsertMaterializeStrategy" : "VALUE", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `bk` VARCHAR(2147483647), `bv` INT>", + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[pk, pv, bk, bv])" + } ], + "edges" : [ { + "source" : 1, + "target" : 2, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 2, + "target" : 3, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 4, + "target" : 5, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 5, + "target" : 6, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 3, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 6, + "target" : 7, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 7, + "target" : 8, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 8, + "target" : 9, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-inner/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-inner/savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..ea9e93db4c1e8b7483a49fb3ecae269359be01c1 GIT binary patch literal 19149 zcmeGkZEV}d_4E7(}e^i9`EDbyLa#I-M#nj5@%*8gisg!@Z-{7XfxV8#){l@=rrmIdtIT_Dq;}eNkGBFjOfLJ#=DJwHHTVc6bo-T<%by{N!B2Oz-NfXLEtzD?_RZS38 zIzF%Qio}XpUJ+PPxX3F5tx;Ktt*EoI2CzM7$KP>px~{&DHmM5w(h4=<8dUnIz>7t7 zoYmMIe1G`*)dN3Yd^zt%t|Z!7!f9rbs-fQG99v@eJ{w;n8>OCc$W@XWThE~M2#lVHc_4tV1Qt>G!#feJ&aB-hR};4 z^kPUW7$Yx}Y?K+DuhbN@zZE^w>H}XPlq^(CTq}Vdl)?;bt3AQpije3mq zjI0V8Ag-Z3(-x%Dg*l$n0Bi4AK`LsB7skQYTyvpD(gsM{Ap&Y2iTP|( zkE}k~#x%JsaOR}A|Jm#pKDziwzzqzHYjneW)!SfWo&J|UxDvTI`pY9YQ^giN28!)! z6oYp2GrY3f-wuRMMA#r|zC{Pnp8c%y}+Dh8t! zu&)&-g-Mc06NLIs{THaAi_6>!yxA(?fG;sgEb^@-rb)a3?;E1+-Y{Of47_erqH8^- zMC$-E$l0EVu^CtoCCVP@^s5W3BSpVMe^vy(N_E#UmG-4WqMR zq*vW>HgXD@#-<70VaEG_8J!IyR&%c+Jvy<;#VYtm@gIIz{k3Q+=FRV|dOZHWW8Q5xc?){n z;+saA`Q2xe*Q|2`aA;tI4C8nNOxF=&?mqXQGk+eAeC1OjzKS9j_-X%X)a}o!B<}5hA9(HhCp~|uz4^xT)9-%nRVZEuhrm3;f~6G< zt%fc2I2e|v!>7?_!g*0op;RM@nuD|#_=U@l?$=)WeE(0&?6+@Xqgx%Ci5#)3yU^Yi z0F_x#ir`#EPS@deH;+eyLs5UIz)`_s$xj6%p%TRw!U2loLs5?N4-G{Ezz(iKKoZR3 zD>lMtLpuiYO_M1wmXiYZR`JXl6iV34XVEA%f0}&=b$QU^El`_n?Q#XWU6U1+#@$Mr zolM*CL)yqXkiq>2(6RM69UDj9@ITUax zu)!2q_gTGE3=M(1(;yWp2E$ZvmUaX{q@~t$ zLUJs%4bP;zF^*YQ>#mNQce>HGLRAop6cLxkKysnp$Kp@K#|P=GI@M^PAptG#&aqPN|U01bpE!;qzQJ`fg=}TGcmetUl=fHzh z)!1^S*#pX*KwByb&k5M|vhJ%GY|nGA4wLvV-hY0C>!QJXD3>@92lw9-nf$3l?pS&< zm&Y;rsdOq16QBnich=ZLn3~4SN<0~{TU8ghzk$OdhHRPbJRP6!JE7?~vjW4mx`EXh ztXUT`utK~=HsCF^2d!oAn3ScNhL{;>{Yv!Jn0<3e{4HR}3z>ujFrMXdY|fnU&WI*iW`W+$JKnE*M-L zUF$jv27QftS8hxew_nFA5sU9N?-rVkOluTJ&Vs?_P{m!?shkA^oM2j$|KIv87&b<< z@6@F^U!+)#I&U6Sv6JeX$iXoX`)*{vukMWW-59cLC)Sz=ZmH8r-J9D{U^ohl`$mCb z*~z*;l{?vunHPJl^CG3f**pDt;04B3O%o|N5Gg1Z9FpkgzL~rRia#Y!^bZU$4#%r- zI)A3seSze|XZekD{b1dAO9>pDPt8pLR-u{_;chb_-hZ8>AiUEmJO(-B?5KiIV(8R zmk}rV@E?B6NYlab5C2*j94WBfbQLU22fc9)(hZXux*VMCv1h>0#?ZEWHu8|jsFjo?>U43 literal 0 HcmV?d00001 diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-left/plan/lateral-snapshot-join-left.json b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-left/plan/lateral-snapshot-join-left.json new file mode 100644 index 00000000000000..fa475e5b1c68ab --- /dev/null +++ b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-left/plan/lateral-snapshot-join-left.json @@ -0,0 +1,468 @@ +{ + "flinkVersion" : "2.4", + "nodes" : [ { + "id" : 10, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`probe`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "pts_str", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pts", + "kind" : "COMPUTED", + "expression" : { + "rexNode" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "TO_TIMESTAMP(`pts_str`)" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "pts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 3, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`pts`" + } + } ] + } + } + }, + "abilities" : [ { + "type" : "WatermarkPushDown", + "watermarkExpr" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "rowtimeExpr" : { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$REINTERPRET$1", + "operands" : [ { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + } ], + "type" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, + "idleTimeoutMillis" : -1, + "producedType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `pts_str` VARCHAR(2147483647)> NOT NULL", + "watermarkParams" : { + "emitStrategy" : "ON_EVENT", + "alignGroupName" : null, + "alignMaxDrift" : "PT0S", + "alignUpdateInterval" : "PT1S", + "sourceIdleTimeout" : -1 + } + } ] + }, + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `pts_str` VARCHAR(2147483647)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, probe, watermark=[TO_TIMESTAMP(pts_str)], watermarkEmitStrategy=[on-event]]], fields=[pk, pv, pts_str])" + }, { + "id" : 11, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "INT" + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT>", + "description" : "Calc(select=[pk, pv])" + }, { + "id" : 12, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT>", + "description" : "Exchange(distribution=[hash[pk]])" + }, { + "id" : 13, + "type" : "stream-exec-table-source-scan_2", + "scanTableSource" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`b`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + }, { + "name" : "bts_str", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bts", + "kind" : "COMPUTED", + "expression" : { + "rexNode" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "TO_TIMESTAMP(`bts_str`)" + } + } ], + "watermarkSpecs" : [ { + "rowtimeAttribute" : "bts", + "expression" : { + "rexNode" : { + "kind" : "INPUT_REF", + "inputIndex" : 3, + "type" : "TIMESTAMP(3)" + }, + "serializableString" : "`bts`" + } + } ] + } + } + }, + "abilities" : [ { + "type" : "WatermarkPushDown", + "watermarkExpr" : { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + }, + "rowtimeExpr" : { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$REINTERPRET$1", + "operands" : [ { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + } ], + "type" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + }, + "idleTimeoutMillis" : -1, + "producedType" : "ROW<`bk` VARCHAR(2147483647), `bv` INT, `bts_str` VARCHAR(2147483647)> NOT NULL", + "watermarkParams" : { + "emitStrategy" : "ON_EVENT", + "alignGroupName" : null, + "alignMaxDrift" : "PT0S", + "alignUpdateInterval" : "PT1S", + "sourceIdleTimeout" : -1 + } + } ] + }, + "outputType" : "ROW<`bk` VARCHAR(2147483647), `bv` INT, `bts_str` VARCHAR(2147483647)>", + "description" : "TableSourceScan(table=[[default_catalog, default_database, b, watermark=[TO_TIMESTAMP(bts_str)], watermarkEmitStrategy=[on-event]]], fields=[bk, bv, bts_str])" + }, { + "id" : 14, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "INT" + }, { + "kind" : "CALL", + "syntax" : "SPECIAL", + "internalName" : "$REINTERPRET$1", + "operands" : [ { + "kind" : "CALL", + "internalName" : "$TO_TIMESTAMP$1", + "operands" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + } ], + "type" : "TIMESTAMP(3)" + } ], + "type" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Calc(select=[bk, bv, Reinterpret(TO_TIMESTAMP(bts_str)) AS bts])" + }, { + "id" : 15, + "type" : "stream-exec-exchange_1", + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "HASH", + "keys" : [ 0 ] + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : { + "type" : "ROW", + "fields" : [ { + "name" : "bk", + "fieldType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "fieldType" : "INT" + }, { + "name" : "bts", + "fieldType" : { + "type" : "TIMESTAMP_WITHOUT_TIME_ZONE", + "precision" : 3, + "kind" : "ROWTIME" + } + } ] + }, + "description" : "Exchange(distribution=[hash[bk]])" + }, { + "id" : 16, + "type" : "stream-exec-lateral-snapshot-join_1", + "joinSpec" : { + "joinType" : "LEFT", + "leftKeys" : [ 0 ], + "rightKeys" : [ 0 ], + "filterNulls" : [ true ], + "nonEquiCondition" : null + }, + "rightTimeAttributeIndex" : 2, + "loadCompletedCondition" : "user_time", + "loadCompletedTime" : 1577836803000, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + }, { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `bk` VARCHAR(2147483647), `bv` INT, `bts` TIMESTAMP(3)>", + "description" : "LateralSnapshotJoin(joinType=[LeftOuterJoin], where=[(pk = bk)], select=[pk, pv, bk, bv, bts], loadCompletedCondition=[user_time], loadCompletedTime=[1577836803000])" + }, { + "id" : 17, + "type" : "stream-exec-calc_1", + "projection" : [ { + "kind" : "INPUT_REF", + "inputIndex" : 0, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 1, + "type" : "INT" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 2, + "type" : "VARCHAR(2147483647)" + }, { + "kind" : "INPUT_REF", + "inputIndex" : 3, + "type" : "INT" + } ], + "condition" : null, + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `bk` VARCHAR(2147483647), `bv` INT>", + "description" : "Calc(select=[pk, pv, bk, bv])" + }, { + "id" : 18, + "type" : "stream-exec-sink_2", + "configuration" : { + "table.exec.sink.keyed-shuffle" : "AUTO", + "table.exec.sink.not-null-enforcer" : "ERROR", + "table.exec.sink.rowtime-inserter" : "ENABLED", + "table.exec.sink.type-length-enforcer" : "IGNORE", + "table.exec.sink.upsert-materialize" : "AUTO" + }, + "dynamicTableSink" : { + "table" : { + "identifier" : "`default_catalog`.`default_database`.`sink`", + "resolvedTable" : { + "schema" : { + "columns" : [ { + "name" : "pk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "pv", + "dataType" : "INT" + }, { + "name" : "bk", + "dataType" : "VARCHAR(2147483647)" + }, { + "name" : "bv", + "dataType" : "INT" + } ] + } + } + } + }, + "inputChangelogMode" : [ "INSERT" ], + "inputProperties" : [ { + "requiredDistribution" : { + "type" : "UNKNOWN" + }, + "damBehavior" : "PIPELINED", + "priority" : 0 + } ], + "outputType" : "ROW<`pk` VARCHAR(2147483647), `pv` INT, `bk` VARCHAR(2147483647), `bv` INT>", + "description" : "Sink(table=[default_catalog.default_database.sink], fields=[pk, pv, bk, bv])" + } ], + "edges" : [ { + "source" : 10, + "target" : 11, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 11, + "target" : 12, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 13, + "target" : 14, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 14, + "target" : 15, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 12, + "target" : 16, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 15, + "target" : 16, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 16, + "target" : 17, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + }, { + "source" : 17, + "target" : 18, + "shuffle" : { + "type" : "FORWARD" + }, + "shuffleMode" : "PIPELINED" + } ] +} \ No newline at end of file diff --git a/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-left/savepoint/_metadata b/flink-table/flink-table-planner/src/test/resources/restore-tests/stream-exec-lateral-snapshot-join_1/lateral-snapshot-join-left/savepoint/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..5b03d9a12596802a5b473ab1829b91076c1ba980 GIT binary patch literal 19163 zcmeGkTWlOx_3k4NCynhSrfpuWQQh*&?s)d)wJo(-+iSAvX4iPvNoykt0nKdbBZt_Frf1j8YU z;Zga^H~#*{&$`!te(cbR`L|D4+nY1fv2FanFU@IK;l!N8Rn-MqJ0T0w92+}>?q_2K zQLhWAMrB|TY6pI>ek}QcXRa-ceRKA~>plN!Hj2kjOYe06s4j3SCb$^*c#e9|i)BH? zBg`_VFlAXRVMS%4!;G?WVR=wl91bm>V~3V2;dnf9u2Nix55hHHW%A5oO%k*~I5spC zi^oH;cr+S`M}~$5BaF%|<0@dP1&YJ55L*sJN_;6C8jf+%VUEG27#E9$h6BUJVk8iW zg@yvf2+IeGI26Ls;c$eF@_^qUAPH>YK`ka>%)x{Dvs20GL_RxR$mJ9HWMLwe%TK4q zX7Z_Qrf@Plkwn{OGVn}0^qxwjXOe|nayperr)HDWQ0hffvN9j!svN(7gJn^Wmf%GQ z@^YmjOF?b5iffu6s=*@AE|`>Rl^j+CP82R+Wx)Pwt|CD7p(h+rHLfV)pi+}Gp@M^Q z6&7BT6*WkfHE1q!un9+Gjiv)rWc?`4{&Y4mf&3@3DKax-L3hx{`lc}6THgBiwye5o zMgO;MTJ2W46mTi9sT8>5QoY26xgst{1I6-iG!Th|;VBjh2jcN!B!ojeAB~g{>VF5Q zLY2K2<=;a*HstQKe13w{IE&PEl-L9>2*a!YMB7va4YnAAz*d8zJTHJ{5j@2y;|rXU zheBh~JYM8Uq3t4;=Cy^DcHyLTcy>76S+q+~Gc{2p!at36lCh^{S<_G~4d^kt-eIWT zVW@^+D0+;69%BJZ`i-tsJr@-eX=`d5`kFks4vQ;F&E zuMzBZg_+Y7L7LyNcFhWH%eNElF4hFG6rjW~R|NGw zo_sPnVJP|hlT%6LJDHe*YAvLIG-<7wC zbq#VJbufc&jM1%Dmjo$E%iE*?*(U7?5evE|%X-n>yZKX@eDdSTX@h#9wy2i}t5(&x zO4Uw7Zwhr)70e5&AWLcFUlOE}h9d0=zdFpvq1Wj9bTtm;Qzw()-JF~%oKEGBXJ_&S zQc{@BW|F{cK2Qg@xI%=Q2Ad+H49=m8XgkRK>Fj9&*&^?KsMGR3<6y4VavC;hLrxFa z-fU9)tOKUq_iE^M6&} zOuY+RPs71evQWx!uo1zX#g@TZl-ow#uj^fY0ZalMDAvklteC=VE9u``@z?D{9 zn8&p?6Jbjj44b3cyLD}jO94&`oUhzOt8b%6q)yE$aJhsJ&0ccxA!&)&T5 z0fu{kp>F!=rD2a)JX!Bu>B(-6TV#ej;>xe9|g6h5>|)5HK?sS3)fOA?(VOhJQ# zKczs=9~fX99;+&x`#aMYXgihCQ74|K?W22DQ!rNns5CU+2%VE-8l`c}(Zmz?+C)Gn zHJ=-Gy1kybW#K#>m))H_SNo`Khu~!_Y3}Bz=h`$2#JC~ZEftdO4Q7f=8&!D5CM{k!RqoJy5%iey?6{0t$lOh*q_TS&VdidLGFBc$BC5^hvq|U5THPo9XnH>yC!eB_Gg2YW9zA1FG^DjPsz3*$@ zPgfvxokl&pEGdtMNS34dv(AhWrU$cqiM8TY^1$tDAZn9*K$@5`7Fx{I_N|X zrl|%Y(?@4ZQ)U<=AZ{GgQj5{;?i+WT_D4+R3Yc6y-WjuZ;z70I&P=9t0Wfd z*?N&~bSILId=P*J8a_fZlL-F?$7NB3C4;3@2WF&-ToTFx&7=ap{$J0-?DzliLNDC~ zei2Gc+8upw01dzg#?YnLo?3r^g7NSgy4SM8jb{4t?r%U&C2S+!?}R2f zHycJUj&d5%i%azR0S{@|PZHEDb<6a5fYv=i6B`cDlKX8da&uvu(Bz81n?mv6XL28T z=fXoFuaPL>1$os=*?{KG>GZ$+_LcaB(eI6sM%AY1G0<&qqZ`Z{&tqlZpD$loUwbZ2 zGY+>cb7Bp{-nW3y)G9@+JbU#^M|OYl_2(PljTzFR7($3p^HE9F5Unz4f^3deCp}`n zdlu}T1#g+NV3!$PX52E&XpzDGjvsDyBE{}*ObguT1G^3*F+g31u{5^oI*hmbS1PW< zNY4XYhjD{DXZc+bbxT}_(RCR0NHQFjxHh_Lqi-1--DO6X8Mh2Gx(=iB$Hca1S|^-b zas>Zj;~c&omo^XPIlW@d3`w`s){Hq@+nNF7Ul{|L?`A;R+SVL%6L2uFqc`K@H!KNW zv-IOP-+1iVTHl3{mxivfU;j?`MaQ>&ncyGPU0~Hhf1?(k5M$q!KZBCy~lhPBtc!UmgH(hwN@b| zCYW4JtzxM}inf>;6V@- zVcQ`>&isT9>3;dS-|f8eMBl444zvq!tZ?*a${LC~!qvsqj59sr-(0SRaUfyA(b2n6q79q(WhyeSFcUppT! z=o(LCvYFI)A`L-nV}<+cFaZG|$A(Y-G-1Vh2eJxHXJf^QxH(-A5X;TvrjnV7{{zqV BIFtYY literal 0 HcmV?d00001