diff --git a/src/hooks/useAdaptivePoll.ts b/src/hooks/useAdaptivePoll.ts index ab3bc5481..cf66295bc 100644 --- a/src/hooks/useAdaptivePoll.ts +++ b/src/hooks/useAdaptivePoll.ts @@ -1,4 +1,55 @@ import { useEffect, useRef, useState } from "react" +import { sleep } from "../utils/sleep" + +type AdaptivePollLoopOptions = { + fetchFn: () => Promise + signal: AbortSignal + minIntervalMs: number + maxIntervalMs: number + sampleSize?: number + multiplier?: number + skipInitialFetch?: boolean + onIntervalChange?: (intervalMs: number) => void +} + +export const runAdaptivePollLoop = async ({ + fetchFn, + signal, + minIntervalMs, + maxIntervalMs, + sampleSize = 3, + multiplier = 2, + skipInitialFetch = false, + onIntervalChange, +}: AdaptivePollLoopOptions): Promise => { + let samples: number[] = [] + let skip = skipInitialFetch + while (!signal.aborted) { + let nextInterval = minIntervalMs + if (skip) { + // Data was just transferred in — wait one interval before the first + // background refresh instead of re-querying it immediately. + skip = false + } else { + const start = performance.now() + try { + await fetchFn() + } catch { + // fetchFn handles its own errors; never let one kill the loop + } + if (signal.aborted) break + samples = [...samples, performance.now() - start].slice(-sampleSize) + const avg = samples.reduce((a, b) => a + b, 0) / samples.length + nextInterval = Math.min( + maxIntervalMs, + Math.max(minIntervalMs, Math.round(avg * multiplier)), + ) + onIntervalChange?.(nextInterval) + } + const aborted = await sleep(nextInterval, signal) + if (aborted) break + } +} type AdaptivePollOptions = { fetchFn: () => Promise @@ -29,85 +80,28 @@ export const useAdaptivePoll = ( getSkipInitialFetch, } = options - const samplesRef = useRef([]) const abortControllerRef = useRef(null) const [currentInterval, setCurrentInterval] = useState(minIntervalMs) useEffect(() => { - samplesRef.current = [] setCurrentInterval(minIntervalMs) - - // Abort any previous polling loop - if (abortControllerRef.current) { - abortControllerRef.current.abort() - } + abortControllerRef.current?.abort() if (!enabled) return const abortController = new AbortController() abortControllerRef.current = abortController - const calculateInterval = (samples: number[]): number => { - if (samples.length === 0) return minIntervalMs - - const avg = samples.reduce((a, b) => a + b, 0) / samples.length - const calculated = avg * multiplier - - return Math.min( - maxIntervalMs, - Math.max(minIntervalMs, Math.round(calculated)), - ) - } - - const sleep = (ms: number, signal: AbortSignal) => - new Promise((resolve, reject) => { - const timeoutId = setTimeout(resolve, ms) - signal.addEventListener("abort", () => { - clearTimeout(timeoutId) - reject(new DOMException("Aborted", "AbortError")) - }) - }) - - const runPollingLoop = async () => { - let skipFetch = getSkipInitialFetch?.() ?? false - while (!abortController.signal.aborted) { - let nextInterval = minIntervalMs - - if (skipFetch) { - // Data was just transferred in — wait one interval before the first - // background refresh instead of re-querying it immediately. - skipFetch = false - } else { - const startTime = performance.now() - - try { - await fetchFn() - } catch (error) { - // Silently handle errors - the fetchFn should handle its own errors - } - - // Check if we should stop after the fetch completed - if (abortController.signal.aborted) break - - const responseTime = performance.now() - startTime - samplesRef.current = [...samplesRef.current, responseTime].slice( - -sampleSize, - ) - - nextInterval = calculateInterval(samplesRef.current) - setCurrentInterval(nextInterval) - } - - try { - await sleep(nextInterval, abortController.signal) - } catch (error) { - // Sleep was aborted, exit the loop - break - } - } - } - - void runPollingLoop() + void runAdaptivePollLoop({ + fetchFn, + signal: abortController.signal, + minIntervalMs, + maxIntervalMs, + sampleSize, + multiplier, + skipInitialFetch: getSkipInitialFetch?.() ?? false, + onIntervalChange: setCurrentInterval, + }) return () => { abortController.abort() diff --git a/src/scenes/Editor/Notebook/DrawCanvas/index.tsx b/src/scenes/Editor/Notebook/DrawCanvas/index.tsx index 8f3fc6997..60ed022bb 100644 --- a/src/scenes/Editor/Notebook/DrawCanvas/index.tsx +++ b/src/scenes/Editor/Notebook/DrawCanvas/index.tsx @@ -8,52 +8,13 @@ import { type ChartRendererHandle, } from "../CellChart/ChartRenderer" import { ChartSettingsDrawer } from "../CellChart/ChartSettingsDrawer" -import { useAdaptivePoll } from "../../../../hooks/useAdaptivePoll" -import { useQueryExecution } from "../../../../hooks/useQueryExecution" -import type { QueryExecResult } from "../../../../hooks/useQueryExecution" -import { getQueriesFromText } from "../../Monaco/utils" -import { - resolveDraw, - resultMatchesQueries, - resultsEquivalent, - successResults, - toExecResult, -} from "./drawCanvasUtils" -import { useNotebookActions } from "../NotebookProvider" -import { useValidateWithGlobals } from "../globals/useValidateWithGlobals" +import { resolveDraw } from "./drawCanvasUtils" import { toast } from "../../../../components/Toast" import { CircleNotchSpinner } from "../../Monaco/icons" -import { - autoRefreshIntervalMs, - capResultBytes, - NOTEBOOK_BYTE_CAP, - NOTEBOOK_ROW_CAP, - singleResultFromExec, - sqlHash, -} from "../notebookUtils" -import { - deleteCellSnapshot, - loadCellSnapshot, -} from "../../../../store/notebookResults" -import { persistCellSnapshot } from "../persistCellSnapshot" import { eventBus } from "../../../../modules/EventBus" import { EventType } from "../../../../modules/EventBus/types" - -const REFRESH_MIN_MS = 2000 -const REFRESH_MAX_MS = 60000 -const FETCH_DEBOUNCE_MS = 300 - -const errorExecResult = (query: string): QueryExecResult => ({ - type: "error", - query, - columns: [], - dataset: [], - count: 0, - error: "Query failed", -}) -// Draw auto-refresh can poll every few seconds; throttle snapshot writes so a -// live chart doesn't churn IndexedDB. A reload restores the last saved frame. -const SNAPSHOT_THROTTLE_MS = 10000 +import { useChartFetchState } from "../chartRefresh/ChartRefreshContext" +import { pendingChartFetchState } from "../chartRefresh/chartRefreshEngine" const Wrapper = styled.div` display: flex; @@ -81,48 +42,36 @@ const EmptyState = styled.div` type Props = { cell: NotebookCell - bufferId?: number isFocused: boolean onConfigChange: (config: ChartConfig) => void } export const DrawCanvas: React.FC = ({ cell, - bufferId, isFocused, onConfigChange, }) => { - const { getVariables, mirrorCellResult } = useNotebookActions() - const { executeSingle } = useQueryExecution(getVariables()) - const validateWithGlobals = useValidateWithGlobals() - const [results, setResults] = useState([]) - const [fetching, setFetching] = useState(false) const [settingsOpen, setSettingsOpen] = useState(false) - const [settledKey, setSettledKey] = useState(null) - const [lastFetchHadError, setLastFetchHadError] = useState(false) - const [classifyBlock, setClassifyBlock] = useState< - | { kind: "write"; queryType: string } - | { kind: "failed"; message: string } - | null - >(null) const [zoomStart, setZoomStart] = useState(0) const [zoomEnd, setZoomEnd] = useState(100) - const [debouncedSql, setDebouncedSql] = useState(cell.value) - useEffect(() => { - const timer = setTimeout( - () => setDebouncedSql(cell.value), - FETCH_DEBOUNCE_MS, - ) - return () => clearTimeout(timer) - }, [cell.value]) - - const classifyCache = useMemo( - () => new Map(), - [debouncedSql], - ) const configAtSettingsOpenRef = useRef(undefined) const chartRendererRef = useRef(null) + + const fetchState = useChartFetchState(cell.id) + const pendingState = useMemo( + () => pendingChartFetchState(cell.value), + [cell.value], + ) + const { + queries, + queriesKey, + results, + settledKey, + lastFetchHadError, + classifyBlock, + } = fetchState ?? pendingState + const isZoomed = zoomStart > 0 || zoomEnd < 100 const handleZoomChange = useCallback((start: number, end: number) => { @@ -136,286 +85,6 @@ export const DrawCanvas: React.FC = ({ setZoomEnd(100) }, []) - const queries = useMemo( - () => getQueriesFromText(debouncedSql), - [debouncedSql], - ) - const queriesKey = queries.join("\u0001") - - const inFlightRef = useRef(null) - const mountedRef = useRef(true) - const lastSnapshotAtRef = useRef(0) - // Latest cell.result, read at mount to transfer existing data into the chart - // without re-querying when toggling over from the grid. - const cellResultRef = useRef(cell.result) - // Last results mirrored into cell.result, so the grid shows the chart's - // current data on switch-back; guards against redundant cell writes. - const lastMirrorRef = useRef([]) - // Set when we transfer existing data in on mount, so the poll skips its - // otherwise-immediate first fetch (the data is already current). - const skipNextPollFetchRef = useRef(false) - const getSkipInitialFetch = useCallback(() => { - const skip = skipNextPollFetchRef.current - skipNextPollFetchRef.current = false - return skip - }, []) - // Whether the poll is driving the fetch. The mount effect's fallback reads it - // (live, via a ref) so it only fetches when polling is off — otherwise the - // poll's immediate first tick is the single fetch, not a duplicate. - const pollEnabledRef = useRef(false) - const lastSavedRef = useRef<{ - sqlHash: string - results: QueryExecResult[] - } | null>(null) - - // Persist a bounded, throttled copy of the chart's rows — shared with run - // mode (one snapshot per cell) so the chart survives reload without re-fetch. - // Frames identical to the last saved one are skipped; a changed frame blocked - // by the throttle retries on the next poll tick, so the final frame persists. - const saveDrawSnapshot = useCallback( - (execResults: QueryExecResult[]) => { - if (bufferId === undefined) return - const currentSqlHash = sqlHash(debouncedSql) - const last = lastSavedRef.current - if ( - last && - last.sqlHash === currentSqlHash && - resultsEquivalent(last.results, execResults) - ) - return - const now = Date.now() - if (now - lastSnapshotAtRef.current < SNAPSHOT_THROTTLE_MS) return - lastSnapshotAtRef.current = now - lastSavedRef.current = { sqlHash: currentSqlHash, results: execResults } - const results = execResults.map((r) => - capResultBytes(singleResultFromExec(r, r.query), NOTEBOOK_BYTE_CAP), - ) - void persistCellSnapshot({ - bufferId, - cellId: cell.id, - results, - savedAt: now, - }) - }, - [bufferId, cell.id, debouncedSql], - ) - - const clearDrawSnapshot = useCallback(() => { - if (bufferId === undefined) return - const currentSqlHash = sqlHash(debouncedSql) - const last = lastSavedRef.current - if (last && last.sqlHash === currentSqlHash && last.results.length === 0) - return - lastSavedRef.current = { sqlHash: currentSqlHash, results: [] } - void deleteCellSnapshot(bufferId, cell.id) - }, [bufferId, cell.id, debouncedSql]) - - const mirrorToCellResult = useCallback( - (next: QueryExecResult[]) => { - if (resultsEquivalent(lastMirrorRef.current, next)) return - lastMirrorRef.current = next - mirrorCellResult( - cell.id, - next.length === 0 - ? undefined - : { - results: next.map((r) => - capResultBytes( - singleResultFromExec(r, r.query), - NOTEBOOK_BYTE_CAP, - ), - ), - activeResultIndex: 0, - timestamp: Date.now(), - }, - ) - }, - [mirrorCellResult, cell.id], - ) - - const fetchAll = useCallback(async () => { - // Supersede any in-flight fetch up front, so a slow earlier response can't - // land after the query changed — including when it's cleared to empty. - inFlightRef.current?.abort() - inFlightRef.current = null - if (queries.length === 0) { - setResults((prev) => (prev.length === 0 ? prev : [])) - setFetching(false) - setSettledKey(queriesKey) - setLastFetchHadError(false) - setClassifyBlock(null) - // No query → drop the grid mirror and the saved frame too. - mirrorToCellResult([]) - clearDrawSnapshot() - return - } - const ac = new AbortController() - inFlightRef.current = ac - setFetching(true) - try { - // Runtime backstop: a user typing DDL into an already-draw cell would - // otherwise reach executeSingle on the next poll tick. - try { - await Promise.all( - queries.map(async (q) => { - if (classifyCache.has(q)) return - const res = await validateWithGlobals(q, ac.signal) - if ("error" in res) classifyCache.set(q, "ERROR") - else if ("columns" in res) classifyCache.set(q, "DQL") - else classifyCache.set(q, "DDL_DML") - }), - ) - } catch (e) { - if (ac.signal.aborted) return - const message = e instanceof Error ? e.message : "validate failed" - setClassifyBlock({ kind: "failed", message }) - setSettledKey(queriesKey) - return - } - if (ac.signal.aborted) return - const offender = queries - .map((q) => ({ q, klass: classifyCache.get(q) })) - .find((x) => x.klass === "DDL_DML") - if (offender) { - const validateResult = await validateWithGlobals( - offender.q, - ac.signal, - ).catch(() => null) - if (ac.signal.aborted) return - const queryType = - validateResult && "queryType" in validateResult - ? validateResult.queryType - : "write" - setClassifyBlock({ kind: "write", queryType }) - setResults((prev) => (prev.length === 0 ? prev : [])) - setLastFetchHadError(false) - setSettledKey(queriesKey) - // The cell now holds a write — drop any stale rows the grid would show. - mirrorToCellResult([]) - return - } - setClassifyBlock(null) - const out = await Promise.all( - queries.map((q) => - executeSingle(q, ac.signal, NOTEBOOK_ROW_CAP).catch(() => null), - ), - ) - if (ac.signal.aborted) return - const next = successResults(out) - setResults((prev) => (resultsEquivalent(prev, next) ? prev : next)) - setLastFetchHadError(out.some((r) => r === null)) - setSettledKey(queriesKey) - if (next.length > 0) saveDrawSnapshot(next) - else clearDrawSnapshot() - // Mirror EVERY statement (not just chartable ones) so a switch to the grid - // shows the same tabs a real run would — including errors and empty - // results — instead of dropping them or leaving stale rows behind. - mirrorToCellResult(out.map((r, i) => r ?? errorExecResult(queries[i]))) - } finally { - // Only clear when still the active fetch — a superseded (aborted) run - // must not flip `fetching` off while its replacement is in flight. - if (inFlightRef.current === ac) { - inFlightRef.current = null - if (mountedRef.current) setFetching(false) - } - } - }, [ - classifyCache, - executeSingle, - validateWithGlobals, - queries, - queriesKey, - saveDrawSnapshot, - clearDrawSnapshot, - mirrorToCellResult, - ]) - - useEffect(() => { - cellResultRef.current = cell.result - }, [cell.result]) - - // On mount, render instantly from existing data instead of re-querying: - // 1. cell.result — the just-run grid, or the chart's own mirrored frame. - // Transfers the data when toggling grid↔chart (no spinner, no re-run). - // 2. the persisted snapshot — survives reload (cell.result is stripped). - // autoRefresh-off cells then stay on that frame; autoRefresh-on cells let the - // poll refresh in the background. Falls back to a live fetch when neither has - // chartable rows. - useEffect(() => { - const existing = cellResultRef.current - // Transfer only a result produced by the CURRENT queries — a result left - // over from edited-but-not-rerun SQL is stale and must be re-fetched. - const transferred = resultMatchesQueries(existing, queries) - ? successResults(existing.results.map(toExecResult)) - : [] - if (transferred.length > 0) { - setResults((prev) => (prev.length > 0 ? prev : transferred)) - setSettledKey(queriesKey) - skipNextPollFetchRef.current = true - return - } - let cancelled = false - void (async () => { - if (bufferId !== undefined) { - // Best-effort: a failed read falls through to a live fetch instead of - // leaving the chart on its loading spinner forever. - const snap = await loadCellSnapshot(bufferId, cell.id).catch( - () => undefined, - ) - if (cancelled) return - // Only render a snapshot produced by the CURRENT queries — one left - // over from edited-but-not-rerun SQL is stale and must re-fetch (the - // snapshot is keyed by cell, not SQL, so it can outlive an edit). - const snapResult = snap && { - results: snap.results, - activeResultIndex: 0, - timestamp: snap.savedAt, - } - if (resultMatchesQueries(snapResult, queries)) { - const hydrated = successResults(snapResult.results.map(toExecResult)) - if (hydrated.length > 0) { - // Don't clobber live data that may already have landed. - setResults((prev) => (prev.length > 0 ? prev : hydrated)) - setSettledKey(queriesKey) - // Seed cell.result too, so a switch to the grid shows this frame - // instead of an empty cell (cell.result is stripped on reload). - mirrorToCellResult(hydrated) - return - } - } - } - if (!cancelled && !pollEnabledRef.current) void fetchAll() - })() - return () => { - cancelled = true - } - }, [fetchAll, bufferId, cell.id, queriesKey, queries, mirrorToCellResult]) - - useEffect( - () => () => { - mountedRef.current = false - inFlightRef.current?.abort() - }, - [], - ) - - const autoRefresh = cell.autoRefresh ?? true - const fixedIntervalMs = autoRefreshIntervalMs(autoRefresh) - const pollEnabled = autoRefresh !== false && queries.length > 0 - - useEffect(() => { - pollEnabledRef.current = pollEnabled - }, [pollEnabled]) - - useAdaptivePoll({ - fetchFn: fetchAll, - enabled: pollEnabled, - key: `${cell.id}:${queriesKey}:${autoRefresh}`, - minIntervalMs: fixedIntervalMs ?? REFRESH_MIN_MS, - maxIntervalMs: fixedIntervalMs ?? REFRESH_MAX_MS, - getSkipInitialFetch, - }) - const resolution = useMemo( () => resolveDraw(queries, results, cell.chartConfig), [queries, results, cell.chartConfig], @@ -449,7 +118,7 @@ export const DrawCanvas: React.FC = ({ } else if (queries.length === 0) { emptyMessage = "Type a query to draw." } else if (!settledForCurrentQueries) { - emptyMessage = "Drawing\u2026" + emptyMessage = "Drawing…" } else if (lastFetchHadError) { emptyMessage = "Query failed — check the SQL editor for the error." } else { @@ -471,18 +140,15 @@ export const DrawCanvas: React.FC = ({ (run: () => void) => (payload?: { cellId?: string }) => { if (payload?.cellId === cell.id) run() } - const refresh = forThisCell(() => void fetchAll()) const open = forThisCell(openSettings) const reset = forThisCell(handleResetZoom) - eventBus.subscribe(EventType.NOTEBOOK_CELL_REFRESH_CHART, refresh) eventBus.subscribe(EventType.NOTEBOOK_CELL_OPEN_CHART_SETTINGS, open) eventBus.subscribe(EventType.NOTEBOOK_CELL_RESET_ZOOM, reset) return () => { - eventBus.unsubscribe(EventType.NOTEBOOK_CELL_REFRESH_CHART, refresh) eventBus.unsubscribe(EventType.NOTEBOOK_CELL_OPEN_CHART_SETTINGS, open) eventBus.unsubscribe(EventType.NOTEBOOK_CELL_RESET_ZOOM, reset) } - }, [cell.id, fetchAll, openSettings, handleResetZoom]) + }, [cell.id, openSettings, handleResetZoom]) useEffect(() => { eventBus.publish(EventType.NOTEBOOK_CELL_CHART_ZOOM, { @@ -491,29 +157,6 @@ export const DrawCanvas: React.FC = ({ }) }, [cell.id, isZoomed]) - // Surface fetch state to the cell toolbar. `loading` is the first-time draw - // (no data yet) — the view toggle spins for it. `refreshing` is a re-fetch - // over existing data (auto-poll or manual refresh) — the refresh button spins - // for it. Publish both false on unmount so no control is stranded spinning. - const refreshing = fetching && !loading - useEffect(() => { - eventBus.publish(EventType.NOTEBOOK_CELL_CHART_LOADING, { - cellId: cell.id, - loading, - refreshing, - }) - }, [cell.id, loading, refreshing]) - useEffect( - () => () => { - eventBus.publish(EventType.NOTEBOOK_CELL_CHART_LOADING, { - cellId: cell.id, - loading: false, - refreshing: false, - }) - }, - [cell.id], - ) - return ( {loading ? ( diff --git a/src/scenes/Editor/Notebook/NotebookProvider.tsx b/src/scenes/Editor/Notebook/NotebookProvider.tsx index 977174a7c..7daaf3f21 100644 --- a/src/scenes/Editor/Notebook/NotebookProvider.tsx +++ b/src/scenes/Editor/Notebook/NotebookProvider.tsx @@ -54,6 +54,11 @@ import { } from "../../../store/notebookResults" import { removeNotebookCellLayouts } from "./notebookColumnLayoutStore" import type { QueryKey } from "../../../store/Query/types" +import { createValidateWithGlobals } from "./declareUtils" +import { + ChartRefreshProvider, + useChartRefreshEngine, +} from "./chartRefresh/ChartRefreshContext" // State and actions live in SEPARATE contexts: action-only consumers never // re-render when state changes (the actions value is ref-stable for life). @@ -189,7 +194,7 @@ export const NotebookProvider: React.FC<{ preview?: boolean }> = ({ initialState, bufferId, preview = false, children }) => { const { updateBuffer } = useEditor() - const { questExecution } = useContext(QuestContext) + const { quest, questExecution } = useContext(QuestContext) const [focusedCellId, setFocusedCellState] = useState( initialState.focusedCellId ?? null, @@ -413,6 +418,23 @@ export const NotebookProvider: React.FC<{ [hydrateCells], ) + const validateWithGlobals = useMemo( + () => createValidateWithGlobals(quest, () => settingsRef.current.variables), + [quest], + ) + + const chartRefreshEngine = useChartRefreshEngine({ + bufferId, + cells: store.cells, + deps: { + executeSingle, + validateWithGlobals, + mirrorCellResult, + getCellResult: (cellId) => + store.cellsRef.current.find((c) => c.id === cellId)?.result, + }, + }) + const runCellNow = useCallback( async ( cellId: string, @@ -627,7 +649,9 @@ export const NotebookProvider: React.FC<{ - {children} + + {children} + diff --git a/src/scenes/Editor/Notebook/cells/Cell.tsx b/src/scenes/Editor/Notebook/cells/Cell.tsx index b3b4130d1..97f9916ae 100644 --- a/src/scenes/Editor/Notebook/cells/Cell.tsx +++ b/src/scenes/Editor/Notebook/cells/Cell.tsx @@ -912,7 +912,6 @@ const CellInner: React.FC = ({ {isDrawMode ? ( diff --git a/src/scenes/Editor/Notebook/chartRefresh/ChartRefreshContext.tsx b/src/scenes/Editor/Notebook/chartRefresh/ChartRefreshContext.tsx new file mode 100644 index 000000000..a0a4bcf8a --- /dev/null +++ b/src/scenes/Editor/Notebook/chartRefresh/ChartRefreshContext.tsx @@ -0,0 +1,70 @@ +import { + createContext, + useContext, + useEffect, + useMemo, + useRef, + useState, +} from "react" +import type { NotebookCell } from "../../../../store/notebook" +import { + ChartRefreshEngine, + type ChartFetchState, + type ChartRefreshDeps, +} from "./chartRefreshEngine" + +const ChartRefreshContext = createContext(null) + +export const ChartRefreshProvider = ChartRefreshContext.Provider + +export const useChartRefresh = () => useContext(ChartRefreshContext) + +export const useChartRefreshEngine = (options: { + bufferId: number + cells: NotebookCell[] + deps: ChartRefreshDeps +}): ChartRefreshEngine => { + const { bufferId, cells, deps } = options + const depsRef = useRef(deps) + const engine = useMemo( + () => new ChartRefreshEngine(bufferId, () => depsRef.current), + [bufferId], + ) + + useEffect(() => { + depsRef.current = deps + }) + + useEffect(() => { + engine.attach() + return () => engine.destroy() + }, [engine]) + + useEffect(() => { + engine.sync(cells) + }, [engine, cells]) + + return engine +} + +export const useChartFetchState = ( + cellId: string, +): ChartFetchState | undefined => { + const engine = useContext(ChartRefreshContext) + const [state, setState] = useState(() => + engine?.getState(cellId), + ) + + useEffect(() => { + if (!engine) return + const listener = () => setState(engine.getState(cellId)) + + // Catch up on anything published between render and subscription. + setState(engine.getState(cellId)) + + engine.subscribe(cellId, listener) + return () => engine.unsubscribe(cellId, listener) + }, [engine, cellId]) + + return state +} diff --git a/src/scenes/Editor/Notebook/chartRefresh/chartRefreshEngine.test.ts b/src/scenes/Editor/Notebook/chartRefresh/chartRefreshEngine.test.ts new file mode 100644 index 000000000..dfc9ddfb2 --- /dev/null +++ b/src/scenes/Editor/Notebook/chartRefresh/chartRefreshEngine.test.ts @@ -0,0 +1,441 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import type { QueryExecResult } from "../../../../hooks/useQueryExecution" +import type { CellResult, NotebookCell } from "../../../../store/notebook" +import type { AutoRefresh } from "../../../../store/notebook" +import { eventBus } from "../../../../modules/EventBus" +import { EventType } from "../../../../modules/EventBus/types" +import { loadCellSnapshot } from "../../../../store/notebookResults" +import { ChartRefreshEngine, type ChartRefreshDeps } from "./chartRefreshEngine" + +vi.mock("../persistCellSnapshot", () => ({ + persistCellSnapshot: vi.fn().mockResolvedValue(true), +})) + +vi.mock("../../../../store/notebookResults", () => ({ + loadCellSnapshot: vi.fn().mockResolvedValue(undefined), + deleteCellSnapshot: vi.fn().mockResolvedValue(undefined), +})) + +const BUFFER_ID = 1 + +const dqlResult = (query: string): QueryExecResult => ({ + type: "dql", + query, + columns: [ + { name: "x", type: "INT" }, + { name: "y", type: "INT" }, + ], + dataset: [[1, 2]], + count: 1, +}) + +const drawCell = ( + id: string, + value: string, + autoRefresh: AutoRefresh, +): NotebookCell => ({ + id, + position: 0, + value, + mode: "draw", + autoRefresh, +}) + +const dqlValidation = { query: "q", columns: [], timestamp: 0 } + +const makeDeps = () => ({ + executeSingle: vi.fn((sql: string) => Promise.resolve(dqlResult(sql))), + validateWithGlobals: vi.fn().mockResolvedValue(dqlValidation), + mirrorCellResult: vi.fn<[string, CellResult | undefined], void>(), + getCellResult: vi.fn((): CellResult | undefined => undefined), +}) + +const flushAsync = async () => { + await vi.advanceTimersByTimeAsync(0) +} + +describe("ChartRefreshEngine", () => { + let deps: ReturnType + let engine: ChartRefreshEngine + + beforeEach(() => { + vi.useFakeTimers() + // clearAllMocks keeps implementations; reset the module mock explicitly so + // one test's snapshot fixture can't hydrate another test's cells. + vi.mocked(loadCellSnapshot).mockResolvedValue(undefined) + deps = makeDeps() + // Jitter off: tests assert exact fetch timing. + engine = new ChartRefreshEngine(BUFFER_ID, () => deps as ChartRefreshDeps, { + initialFetchJitterMs: 0, + }) + engine.attach() + }) + + afterEach(() => { + engine.destroy() + vi.useRealTimers() + vi.clearAllMocks() + }) + + it("fetches once for a draw cell with auto-refresh off", async () => { + // Given a draw cell with auto-refresh disabled and no prior data + const cell = drawCell("c1", "select 1", false) + + // When the engine syncs it + engine.sync([cell]) + await flushAsync() + + // Then it executes the query exactly once and settles the state + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + expect(deps.executeSingle).toHaveBeenCalledWith( + "select 1", + expect.any(AbortSignal), + 10_000, + ) + const state = engine.getState("c1") + expect(state?.results).toHaveLength(1) + expect(state?.settledKey).toBe(state?.queriesKey) + + // And no further fetches happen over time + await vi.advanceTimersByTimeAsync(120_000) + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + }) + + it("mirrors fetched results into the cell result", async () => { + // Given a draw cell whose query succeeds + engine.sync([drawCell("c1", "select 1", false)]) + + // When the fetch completes + await flushAsync() + + // Then every statement is mirrored so the grid shows the chart's data + expect(deps.mirrorCellResult).toHaveBeenCalledTimes(1) + const [cellId, result] = deps.mirrorCellResult.mock.calls[0] + expect(cellId).toBe("c1") + expect(result?.results).toHaveLength(1) + expect(result?.results[0]).toMatchObject({ type: "dql", query: "select 1" }) + }) + + it("transfers a matching existing cell result instead of fetching", async () => { + // Given a cell result produced by the exact same query (grid → chart toggle) + deps.getCellResult.mockReturnValue({ + results: [ + { + type: "dql", + query: "select 1", + columns: [{ name: "x", type: "INT" }], + dataset: [[1]], + count: 1, + }, + ], + activeResultIndex: 0, + timestamp: 0, + }) + + // When the engine syncs the draw cell with auto-refresh off + engine.sync([drawCell("c1", "select 1", false)]) + await flushAsync() + + // Then the data is transferred without a fetch + expect(deps.executeSingle).not.toHaveBeenCalled() + const state = engine.getState("c1") + expect(state?.results).toHaveLength(1) + expect(state?.settledKey).toBe(state?.queriesKey) + }) + + it("polls on a fixed interval without any mounted component", async () => { + // Given a draw cell with a fixed 1s auto-refresh interval + engine.sync([drawCell("c1", "select 1", "1s")]) + + // When the poll loop runs — an immediate first fetch, then one per interval + await flushAsync() + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + + // Then each elapsed interval triggers exactly one more fetch + await vi.advanceTimersByTimeAsync(1000) + expect(deps.executeSingle).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(1000) + expect(deps.executeSingle).toHaveBeenCalledTimes(3) + }) + + it("skips the poll's immediate first fetch when data was transferred in", async () => { + // Given transferable data and a fixed 1s interval + deps.getCellResult.mockReturnValue({ + results: [ + { + type: "dql", + query: "select 1", + columns: [{ name: "x", type: "INT" }], + dataset: [[1]], + count: 1, + }, + ], + activeResultIndex: 0, + timestamp: 0, + }) + + // When the engine syncs the cell + engine.sync([drawCell("c1", "select 1", "1s")]) + await flushAsync() + + // Then the transferred frame serves as the first tick, and the poll only + // fetches after one full interval + expect(deps.executeSingle).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1000) + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + }) + + it("stops polling when the cell leaves draw mode", async () => { + // Given a polling draw cell + engine.sync([drawCell("c1", "select 1", "1s")]) + await flushAsync() + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + + // When the cell is no longer in draw mode + engine.sync([{ ...drawCell("c1", "select 1", "1s"), mode: "run" }]) + + // Then no further fetches happen and the entry's state is gone + await vi.advanceTimersByTimeAsync(10_000) + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + expect(engine.getState("c1")).toBeUndefined() + }) + + it("refetches when the refresh-chart event fires, even with auto-refresh off", async () => { + // Given a settled draw cell with auto-refresh off + engine.sync([drawCell("c1", "select 1", false)]) + await flushAsync() + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + + // When the toolbar publishes a manual refresh + eventBus.publish(EventType.NOTEBOOK_CELL_REFRESH_CHART, { cellId: "c1" }) + await flushAsync() + + // Then the cell refetches once + expect(deps.executeSingle).toHaveBeenCalledTimes(2) + }) + + it("debounces SQL changes before refetching", async () => { + // Given a settled draw cell + engine.sync([drawCell("c1", "select 1", false)]) + await flushAsync() + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + + // When the SQL changes + engine.sync([drawCell("c1", "select 2", false)]) + + // Then nothing refetches inside the debounce window + await vi.advanceTimersByTimeAsync(299) + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + + // And the new SQL fetches once the debounce elapses + await vi.advanceTimersByTimeAsync(301) + await flushAsync() + expect(deps.executeSingle).toHaveBeenCalledTimes(2) + expect(deps.executeSingle).toHaveBeenLastCalledWith( + "select 2", + expect.any(AbortSignal), + 10_000, + ) + }) + + it("blocks write queries from executing", async () => { + // Given a cell whose SQL classifies as a write + deps.validateWithGlobals.mockResolvedValue({ queryType: "update" }) + + // When the engine syncs it + engine.sync([drawCell("c1", "update t set x = 1", false)]) + await flushAsync() + + // Then the query never executes and the block reaches the state + expect(deps.executeSingle).not.toHaveBeenCalled() + const state = engine.getState("c1") + expect(state?.classifyBlock).toEqual({ kind: "write", queryType: "update" }) + }) + + it("hydrates from a persisted snapshot matching the current SQL", async () => { + // Given a persisted snapshot produced by the same query + vi.mocked(loadCellSnapshot).mockResolvedValue({ + bufferId: BUFFER_ID, + cellId: "c1", + results: [ + { + type: "dql", + query: "select 1", + columns: [{ name: "x", type: "INT" }], + dataset: [[1]], + count: 1, + }, + ], + savedAt: 123, + }) + + // When the engine syncs the cell with auto-refresh off + engine.sync([drawCell("c1", "select 1", false)]) + await flushAsync() + + // Then the snapshot renders without a fetch and is mirrored for the grid + expect(deps.executeSingle).not.toHaveBeenCalled() + expect(engine.getState("c1")?.results).toHaveLength(1) + expect(deps.mirrorCellResult).toHaveBeenCalledTimes(1) + }) + + it("does not poll a cell that is hidden before it enters draw mode", async () => { + // Given the observer reported the cell offscreen before it became a chart + engine.setVisible("c1", false) + + // When the engine syncs the polling draw cell + engine.sync([drawCell("c1", "select 1", "1s")]) + await flushAsync() + + // Then no fetch happens no matter how much time passes + await vi.advanceTimersByTimeAsync(10_000) + expect(deps.executeSingle).not.toHaveBeenCalled() + }) + + it("catches up immediately on reveal when the cell never fetched", async () => { + // Given a hidden, never-fetched polling cell + engine.setVisible("c1", false) + engine.sync([drawCell("c1", "select 1", "1s")]) + await flushAsync() + expect(deps.executeSingle).not.toHaveBeenCalled() + + // When the cell scrolls into view + engine.setVisible("c1", true) + await flushAsync() + + // Then it fetches immediately and keeps polling + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1000) + expect(deps.executeSingle).toHaveBeenCalledTimes(2) + }) + + it("pauses polling when the cell scrolls out of view", async () => { + // Given a visible polling cell that has fetched once + engine.sync([drawCell("c1", "select 1", "1s")]) + await flushAsync() + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + + // When it scrolls out of view + engine.setVisible("c1", false) + + // Then polling stops while its data stays intact + await vi.advanceTimersByTimeAsync(10_000) + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + expect(engine.getState("c1")?.results).toHaveLength(1) + }) + + it("skips the reveal catch-up when the data is still fresh", async () => { + // Given a cell hidden right after a fetch + engine.sync([drawCell("c1", "select 1", "1s")]) + await flushAsync() + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + engine.setVisible("c1", false) + + // When it is revealed again well within its refresh interval + await vi.advanceTimersByTimeAsync(100) + engine.setVisible("c1", true) + await flushAsync() + + // Then there is no immediate refetch — the next one waits a full interval + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1000) + expect(deps.executeSingle).toHaveBeenCalledTimes(2) + }) + + it("defers an auto-refresh-off cell's initial fetch to its first reveal", async () => { + // Given a hidden cell with auto-refresh off + engine.setVisible("c1", false) + engine.sync([drawCell("c1", "select 1", false)]) + await flushAsync() + expect(deps.executeSingle).not.toHaveBeenCalled() + + // When it is revealed for the first time + engine.setVisible("c1", true) + await flushAsync() + + // Then it fetches exactly once + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + + // And hiding and revealing it again does not refetch settled data + engine.setVisible("c1", false) + engine.setVisible("c1", true) + await flushAsync() + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + }) + + it("bounds concurrent fetches across cells", async () => { + // Given an engine capped at one in-flight fetch and a slow first query + engine.destroy() + engine = new ChartRefreshEngine(BUFFER_ID, () => deps as ChartRefreshDeps, { + initialFetchJitterMs: 0, + maxConcurrentFetches: 1, + }) + engine.attach() + let releaseFirst!: (value: QueryExecResult) => void + deps.executeSingle + .mockImplementationOnce( + () => new Promise((res) => (releaseFirst = res)), + ) + .mockImplementation((sql: string) => Promise.resolve(dqlResult(sql))) + + // When two cells want to fetch at the same time + engine.sync([ + drawCell("c1", "select 1", false), + drawCell("c2", "select 2", false), + ]) + await flushAsync() + + // Then only the first is in flight; the second waits its turn + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + releaseFirst(dqlResult("select 1")) + await flushAsync() + expect(deps.executeSingle).toHaveBeenCalledTimes(2) + }) + + it("spreads a loop's first fetch by the configured jitter", async () => { + // Given an engine with 300ms of initial jitter + engine.destroy() + engine = new ChartRefreshEngine(BUFFER_ID, () => deps as ChartRefreshDeps, { + initialFetchJitterMs: 300, + }) + engine.attach() + + // When a polling cell syncs + engine.sync([drawCell("c1", "select 1", "1s")]) + await flushAsync() + + // Then the first fetch waits for the jitter window instead of firing at t=0 + expect(deps.executeSingle).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(300) + expect(deps.executeSingle).toHaveBeenCalledTimes(1) + }) + + it("publishes loading state for the cell toolbar", async () => { + // Given a listener on the chart loading event + const events: Array<{ loading: boolean; refreshing: boolean }> = [] + const handler = (payload?: { + cellId?: string + loading?: boolean + refreshing?: boolean + }) => { + if (payload?.cellId === "c1") + events.push({ + loading: !!payload.loading, + refreshing: !!payload.refreshing, + }) + } + eventBus.subscribe(EventType.NOTEBOOK_CELL_CHART_LOADING, handler) + + // When a draw cell fetches for the first time + engine.sync([drawCell("c1", "select 1", false)]) + await flushAsync() + eventBus.unsubscribe(EventType.NOTEBOOK_CELL_CHART_LOADING, handler) + + // Then it reports loading during the fetch and idle after it settles + expect(events[0]).toEqual({ loading: true, refreshing: false }) + expect(events[events.length - 1]).toEqual({ + loading: false, + refreshing: false, + }) + }) +}) diff --git a/src/scenes/Editor/Notebook/chartRefresh/chartRefreshEngine.ts b/src/scenes/Editor/Notebook/chartRefresh/chartRefreshEngine.ts new file mode 100644 index 000000000..b10507f33 --- /dev/null +++ b/src/scenes/Editor/Notebook/chartRefresh/chartRefreshEngine.ts @@ -0,0 +1,635 @@ +import type { QueryExecResult } from "../../../../hooks/useQueryExecution" +import { runAdaptivePollLoop } from "../../../../hooks/useAdaptivePoll" +import { sleep } from "../../../../utils/sleep" +import { createLimiter } from "../../../../utils/limiter" +import type { + AutoRefresh, + CellResult, + NotebookCell, +} from "../../../../store/notebook" +import type { ValidateQueryResult } from "../../../../utils/questdb/types" +import { eventBus } from "../../../../modules/EventBus" +import { EventType } from "../../../../modules/EventBus/types" +import { getQueriesFromText } from "../../Monaco/utils" +import { + autoRefreshIntervalMs, + capResultBytes, + NOTEBOOK_BYTE_CAP, + NOTEBOOK_ROW_CAP, + singleResultFromExec, + sqlHash, +} from "../notebookUtils" +import { + resultMatchesQueries, + resultsEquivalent, + successResults, + toExecResult, +} from "../DrawCanvas/drawCanvasUtils" +import { + deleteCellSnapshot, + loadCellSnapshot, +} from "../../../../store/notebookResults" +import { persistCellSnapshot } from "../persistCellSnapshot" + +const REFRESH_MIN_MS = 2000 +const REFRESH_MAX_MS = 60000 +const SQL_DEBOUNCE_MS = 300 +// Draw auto-refresh can poll every few seconds; throttle snapshot writes so a +// live chart doesn't churn IndexedDB. A reload restores the last saved frame. +const SNAPSHOT_THROTTLE_MS = 10000 +const MAX_CONCURRENT_FETCHES = 6 +const INITIAL_FETCH_JITTER_MS = 300 + +export type ChartClassifyBlock = + | { kind: "write"; queryType: string } + | { kind: "failed"; message: string } + +export type ChartFetchState = { + queries: string[] + queriesKey: string + results: QueryExecResult[] + fetching: boolean + settledKey: string | null + lastFetchHadError: boolean + classifyBlock: ChartClassifyBlock | null +} + +export type ChartRefreshDeps = { + executeSingle: ( + sql: string, + signal?: AbortSignal, + limit?: number, + ) => Promise + validateWithGlobals: ( + sql: string, + signal?: AbortSignal, + ) => Promise + mirrorCellResult: (cellId: string, result: CellResult | undefined) => void + getCellResult: (cellId: string) => CellResult | null | undefined +} + +export const pendingChartFetchState = (sql: string): ChartFetchState => { + const queries = getQueriesFromText(sql) + return { + queries, + queriesKey: queries.join("\u0001"), + results: [], + fetching: false, + settledKey: null, + lastFetchHadError: false, + classifyBlock: null, + } +} + +const errorExecResult = (query: string): QueryExecResult => ({ + type: "error", + query, + columns: [], + dataset: [], + count: 0, + error: "Query failed", +}) + +export type ChartRefreshEngineOptions = { + maxConcurrentFetches?: number + initialFetchJitterMs?: number +} + +type Entry = { + cellId: string + sql: string + autoRefresh: AutoRefresh + visible: boolean + lastFetchedAt: number + state: ChartFetchState + classifyCache: Map + sqlDebounce: ReturnType | null + pendingSql: string | null + inFlight: AbortController | null + poll: AbortController | null + pollKey: string | null + // Set when existing data was transferred in, so the poll skips its + // otherwise-immediate first fetch (the data is already current). + skipNextPollFetch: boolean + lastSnapshotAt: number + lastSaved: { sqlHash: string; results: QueryExecResult[] } | null + // Last results mirrored into cell.result, so the grid shows the chart's + // current data on switch-back; guards against redundant cell writes. + lastMirror: QueryExecResult[] + lastPublished: { loading: boolean; refreshing: boolean } | null +} + +export class ChartRefreshEngine { + private entries = new Map() + private listeners = new Map void>>() + private visibilityByCell = new Map() + private documentHidden = false + private limitFetch: (task: () => Promise) => Promise + private initialFetchJitterMs: number + + private refreshHandler = (payload?: { cellId?: string }) => { + if (payload?.cellId) this.refresh(payload.cellId) + } + + private documentVisibilityHandler = () => { + const hidden = document.hidden + if (hidden === this.documentHidden) return + this.documentHidden = hidden + for (const entry of this.entries.values()) { + if (hidden) this.updatePoll(entry) + else if (entry.visible) this.resume(entry) + } + } + + constructor( + private bufferId: number, + private getDeps: () => ChartRefreshDeps, + options: ChartRefreshEngineOptions = {}, + ) { + this.limitFetch = createLimiter( + options.maxConcurrentFetches ?? MAX_CONCURRENT_FETCHES, + ) + this.initialFetchJitterMs = + options.initialFetchJitterMs ?? INITIAL_FETCH_JITTER_MS + } + + attach() { + eventBus.subscribe( + EventType.NOTEBOOK_CELL_REFRESH_CHART, + this.refreshHandler, + ) + if (typeof document !== "undefined") { + this.documentHidden = document.hidden + document.addEventListener( + "visibilitychange", + this.documentVisibilityHandler, + ) + } + } + + destroy() { + eventBus.unsubscribe( + EventType.NOTEBOOK_CELL_REFRESH_CHART, + this.refreshHandler, + ) + if (typeof document !== "undefined") { + document.removeEventListener( + "visibilitychange", + this.documentVisibilityHandler, + ) + } + for (const cellId of [...this.entries.keys()]) this.removeEntry(cellId) + this.visibilityByCell.clear() + } + + sync(cells: NotebookCell[]) { + const present = new Set() + for (const cell of cells) { + if (cell.mode !== "draw") continue + present.add(cell.id) + const entry = this.entries.get(cell.id) + if (entry) this.updateEntry(entry, cell) + else this.createEntry(cell) + } + for (const cellId of [...this.entries.keys()]) { + if (!present.has(cellId)) this.removeEntry(cellId) + } + } + + refresh(cellId: string) { + const entry = this.entries.get(cellId) + if (entry) void this.fetchOnce(entry) + } + + // Called by the notebook's cell visibility observer. Hiding pauses the poll + // (in-flight fetches finish and land); revealing resumes it, fetching + // immediately when the data is older than the cell's interval. + setVisible(cellId: string, visible: boolean) { + this.visibilityByCell.set(cellId, visible) + const entry = this.entries.get(cellId) + if (!entry || entry.visible === visible) return + entry.visible = visible + if (visible) this.resume(entry) + else this.updatePoll(entry) + } + + private resume(entry: Entry) { + if (this.shouldAutoRefresh(entry)) { + const interval = + autoRefreshIntervalMs(entry.autoRefresh) ?? REFRESH_MIN_MS + entry.skipNextPollFetch = Date.now() - entry.lastFetchedAt < interval + this.updatePoll(entry) + return + } + // Auto-refresh is off, and has not run yet + if (entry.state.settledKey === null && entry.state.queries.length > 0) { + void this.fetchOnce(entry) + } + } + + getState(cellId: string): ChartFetchState | undefined { + return this.entries.get(cellId)?.state + } + + subscribe(cellId: string, listener: () => void) { + let set = this.listeners.get(cellId) + if (!set) { + set = new Set() + this.listeners.set(cellId, set) + } + set.add(listener) + } + + unsubscribe(cellId: string, listener: () => void) { + const set = this.listeners.get(cellId) + if (!set) return + set.delete(listener) + if (set.size === 0) this.listeners.delete(cellId) + } + + private createEntry(cell: NotebookCell) { + const queries = getQueriesFromText(cell.value) + const entry: Entry = { + cellId: cell.id, + sql: cell.value, + autoRefresh: cell.autoRefresh ?? true, + state: { + queries, + queriesKey: queries.join("\u0001"), + results: [], + fetching: false, + settledKey: null, + lastFetchHadError: false, + classifyBlock: null, + }, + visible: this.visibilityByCell.get(cell.id) ?? true, + lastFetchedAt: 0, + classifyCache: new Map(), + sqlDebounce: null, + pendingSql: null, + inFlight: null, + poll: null, + pollKey: null, + skipNextPollFetch: false, + lastSnapshotAt: 0, + lastSaved: null, + lastMirror: [], + lastPublished: null, + } + this.entries.set(cell.id, entry) + this.hydrate(entry) + this.publishLoading(entry) + } + + private updateEntry(entry: Entry, cell: NotebookCell) { + const autoRefresh = cell.autoRefresh ?? true + if (autoRefresh !== entry.autoRefresh) { + entry.autoRefresh = autoRefresh + this.updatePoll(entry) + } + const target = entry.pendingSql ?? entry.sql + if (cell.value === target) return + entry.pendingSql = cell.value + if (entry.sqlDebounce) clearTimeout(entry.sqlDebounce) + entry.sqlDebounce = setTimeout(() => { + entry.sqlDebounce = null + const sql = entry.pendingSql + entry.pendingSql = null + if (sql != null && sql !== entry.sql) this.applySql(entry, sql) + }, SQL_DEBOUNCE_MS) + } + + private removeEntry(cellId: string) { + const entry = this.entries.get(cellId) + if (!entry) return + if (entry.sqlDebounce) clearTimeout(entry.sqlDebounce) + entry.inFlight?.abort() + entry.poll?.abort() + this.entries.delete(cellId) + // The toolbar must not be stranded spinning after the entry is gone. + eventBus.publish(EventType.NOTEBOOK_CELL_CHART_LOADING, { + cellId, + loading: false, + refreshing: false, + }) + this.notify(cellId) + } + + private applySql(entry: Entry, sql: string) { + entry.sql = sql + entry.classifyCache = new Map() + entry.lastFetchedAt = 0 + const queries = getQueriesFromText(sql) + this.setState(entry, { queries, queriesKey: queries.join("\u0001") }) + this.hydrate(entry) + } + + // Populate the entry from existing data instead of re-querying: + // 1. cell.result — the just-run grid, or the chart's own mirrored frame. + // Transfers the data when toggling grid↔chart (no spinner, no re-run). + // 2. the persisted snapshot — survives reload (cell.result is stripped). + // autoRefresh-off cells then stay on that frame; autoRefresh-on cells let the + // poll refresh in the background. Falls back to a live fetch when neither has + // chartable rows. Only data produced by the CURRENT queries is used — a + // result left over from edited-but-not-rerun SQL is stale and must re-fetch. + private hydrate(entry: Entry) { + const { queries, queriesKey } = entry.state + const existing = this.getDeps().getCellResult(entry.cellId) + const transferred = resultMatchesQueries(existing, queries) + ? successResults(existing.results.map(toExecResult)) + : [] + if (transferred.length > 0) { + this.setState(entry, { + results: + entry.state.results.length > 0 ? entry.state.results : transferred, + settledKey: queriesKey, + }) + entry.skipNextPollFetch = true + this.updatePoll(entry) + return + } + const sqlAtStart = entry.sql + void (async () => { + // Best-effort: a failed read falls through to a live fetch instead of + // leaving the chart on its loading spinner forever. + const snap = await loadCellSnapshot(this.bufferId, entry.cellId).catch( + () => undefined, + ) + if (this.entries.get(entry.cellId) !== entry || entry.sql !== sqlAtStart) + return + const snapResult = snap && { + results: snap.results, + activeResultIndex: 0, + timestamp: snap.savedAt, + } + if (resultMatchesQueries(snapResult, queries)) { + const hydrated = successResults(snapResult.results.map(toExecResult)) + if (hydrated.length > 0) { + // Don't clobber live data that may already have landed. + if (entry.state.results.length === 0) { + this.setState(entry, { results: hydrated, settledKey: queriesKey }) + } + // Seed cell.result too, so a switch to the grid shows this frame + // instead of an empty cell (cell.result is stripped on reload). + this.mirror(entry, hydrated) + return + } + } + if ( + !this.shouldAutoRefresh(entry) && + entry.visible && + !this.documentHidden + ) { + void this.fetchOnce(entry) + } + })() + this.updatePoll(entry) + } + + private shouldAutoRefresh(entry: Entry): boolean { + return entry.autoRefresh !== false && entry.state.queries.length > 0 + } + + private shouldPoll(entry: Entry): boolean { + return ( + this.shouldAutoRefresh(entry) && entry.visible && !this.documentHidden + ) + } + + private updatePoll(entry: Entry) { + const enabled = this.shouldPoll(entry) + const key = enabled + ? `${entry.state.queriesKey}\u0001${String(entry.autoRefresh)}` + : null + if (entry.pollKey === key) return + entry.poll?.abort() + entry.poll = null + entry.pollKey = key + if (!enabled) return + const abort = new AbortController() + entry.poll = abort + void this.runPollLoop(entry, abort) + } + + // The jitter offsets each loop's start so charts starting together don't tick together. + private async runPollLoop(entry: Entry, abort: AbortController) { + if (this.initialFetchJitterMs > 0) { + const jitter = Math.random() * this.initialFetchJitterMs + const aborted = await sleep(jitter, abort.signal) + if (aborted) return + } + const fixed = autoRefreshIntervalMs(entry.autoRefresh) + const skipInitialFetch = entry.skipNextPollFetch + entry.skipNextPollFetch = false + await runAdaptivePollLoop({ + fetchFn: () => this.fetchOnce(entry), + signal: abort.signal, + minIntervalMs: fixed ?? REFRESH_MIN_MS, + maxIntervalMs: fixed ?? REFRESH_MAX_MS, + skipInitialFetch, + }) + } + + private async fetchOnce(entry: Entry) { + // Supersede any in-flight fetch up front, so a slow earlier response can't + // land after the query changed — including when it's cleared to empty. + entry.inFlight?.abort() + entry.inFlight = null + const { queries, queriesKey } = entry.state + if (queries.length === 0) { + this.setState(entry, { + results: [], + fetching: false, + settledKey: queriesKey, + lastFetchHadError: false, + classifyBlock: null, + }) + // No query → drop the grid mirror and the saved frame too. + this.mirror(entry, []) + this.clearSnapshot(entry) + return + } + const ac = new AbortController() + entry.inFlight = ac + this.setState(entry, { fetching: true }) + await this.limitFetch(async () => { + if (ac.signal.aborted) return + await this.runFetch(entry, ac) + }) + } + + private async runFetch(entry: Entry, ac: AbortController) { + const deps = this.getDeps() + const { queries, queriesKey } = entry.state + try { + // Runtime backstop: a user typing DDL into an already-draw cell would + // otherwise reach executeSingle on the next poll tick. + try { + await Promise.all( + queries.map(async (q) => { + if (entry.classifyCache.has(q)) return + const res = await deps.validateWithGlobals(q, ac.signal) + if ("error" in res) entry.classifyCache.set(q, "ERROR") + else if ("columns" in res) entry.classifyCache.set(q, "DQL") + else entry.classifyCache.set(q, "DDL_DML") + }), + ) + } catch (e) { + if (ac.signal.aborted) return + const message = e instanceof Error ? e.message : "validate failed" + this.setState(entry, { + classifyBlock: { kind: "failed", message }, + settledKey: queriesKey, + }) + return + } + if (ac.signal.aborted) return + const offender = queries + .map((q) => ({ q, klass: entry.classifyCache.get(q) })) + .find((x) => x.klass === "DDL_DML") + if (offender) { + const validateResult = await deps + .validateWithGlobals(offender.q, ac.signal) + .catch(() => null) + if (ac.signal.aborted) return + const queryType = + validateResult && "queryType" in validateResult + ? validateResult.queryType + : "write" + this.setState(entry, { + classifyBlock: { kind: "write", queryType }, + results: [], + lastFetchHadError: false, + settledKey: queriesKey, + }) + // The cell now holds a write — drop any stale rows the grid would show. + this.mirror(entry, []) + return + } + if (entry.state.classifyBlock !== null) { + this.setState(entry, { classifyBlock: null }) + } + const out = await Promise.all( + queries.map((q) => + deps.executeSingle(q, ac.signal, NOTEBOOK_ROW_CAP).catch(() => null), + ), + ) + if (ac.signal.aborted) return + const next = successResults(out) + this.setState(entry, { + results: resultsEquivalent(entry.state.results, next) + ? entry.state.results + : next, + lastFetchHadError: out.some((r) => r === null), + settledKey: queriesKey, + }) + if (next.length > 0) this.saveSnapshot(entry, next) + else this.clearSnapshot(entry) + // Mirror EVERY statement (not just chartable ones) so a switch to the grid + // shows the same tabs a real run would — including errors and empty + // results — instead of dropping them or leaving stale rows behind. + this.mirror( + entry, + out.map((r, i) => r ?? errorExecResult(queries[i])), + ) + } finally { + // Only clear when still the active fetch — a superseded (aborted) run + // must not flip `fetching` off while its replacement is in flight. + if (entry.inFlight === ac) { + entry.inFlight = null + entry.lastFetchedAt = Date.now() + this.setState(entry, { fetching: false }) + } + } + } + + private mirror(entry: Entry, next: QueryExecResult[]) { + if (resultsEquivalent(entry.lastMirror, next)) return + entry.lastMirror = next + this.getDeps().mirrorCellResult( + entry.cellId, + next.length === 0 + ? undefined + : { + results: next.map((r) => + capResultBytes( + singleResultFromExec(r, r.query), + NOTEBOOK_BYTE_CAP, + ), + ), + activeResultIndex: 0, + timestamp: Date.now(), + }, + ) + } + + // Persist a bounded, throttled copy of the chart's rows — shared with run + // mode (one snapshot per cell) so the chart survives reload without re-fetch. + // Frames identical to the last saved one are skipped; a changed frame blocked + // by the throttle retries on the next poll tick, so the final frame persists. + private saveSnapshot(entry: Entry, execResults: QueryExecResult[]) { + const currentSqlHash = sqlHash(entry.sql) + const last = entry.lastSaved + if ( + last && + last.sqlHash === currentSqlHash && + resultsEquivalent(last.results, execResults) + ) + return + const now = Date.now() + if (now - entry.lastSnapshotAt < SNAPSHOT_THROTTLE_MS) return + entry.lastSnapshotAt = now + entry.lastSaved = { sqlHash: currentSqlHash, results: execResults } + const results = execResults.map((r) => + capResultBytes(singleResultFromExec(r, r.query), NOTEBOOK_BYTE_CAP), + ) + void persistCellSnapshot({ + bufferId: this.bufferId, + cellId: entry.cellId, + results, + savedAt: now, + }) + } + + private clearSnapshot(entry: Entry) { + const currentSqlHash = sqlHash(entry.sql) + const last = entry.lastSaved + if (last && last.sqlHash === currentSqlHash && last.results.length === 0) + return + entry.lastSaved = { sqlHash: currentSqlHash, results: [] } + void deleteCellSnapshot(this.bufferId, entry.cellId) + } + + private setState(entry: Entry, patch: Partial) { + entry.state = { ...entry.state, ...patch } + this.notify(entry.cellId) + this.publishLoading(entry) + } + + private notify(cellId: string) { + this.listeners.get(cellId)?.forEach((listener) => listener()) + } + + // Surface fetch state to the cell toolbar. `loading` is the first-time draw + // (no data yet) — the view toggle spins for it. `refreshing` is a re-fetch + // over existing data (auto-poll or manual refresh) — the refresh button + // spins for it. + private publishLoading(entry: Entry) { + const s = entry.state + const loading = + s.queries.length > 0 && + s.classifyBlock === null && + s.settledKey !== s.queriesKey && + s.results.length === 0 + const refreshing = s.fetching && !loading + const last = entry.lastPublished + if (last && last.loading === loading && last.refreshing === refreshing) + return + entry.lastPublished = { loading, refreshing } + eventBus.publish(EventType.NOTEBOOK_CELL_CHART_LOADING, { + cellId: entry.cellId, + loading, + refreshing, + }) + } +} diff --git a/src/scenes/Editor/Notebook/chartRefresh/useChartCellVisibility.ts b/src/scenes/Editor/Notebook/chartRefresh/useChartCellVisibility.ts new file mode 100644 index 000000000..1528708b3 --- /dev/null +++ b/src/scenes/Editor/Notebook/chartRefresh/useChartCellVisibility.ts @@ -0,0 +1,61 @@ +import { useEffect } from "react" +import { useChartRefresh } from "./ChartRefreshContext" + +// One scroll-container height of overscan in both directions +const OVERSCAN_ROOT_MARGIN = "100% 0px 100% 0px" +const CELL_SELECTOR = "[data-notebook-cell][data-cell-id]" +const SCROLL_CONTAINER_SELECTOR = "#notebook-scroll-container" + +export const useChartCellVisibility = () => { + const engine = useChartRefresh() + + useEffect(() => { + if (!engine || typeof IntersectionObserver === "undefined") return + + let observer: IntersectionObserver | null = null + let observed = new Set() + + const handleEntries = (entries: IntersectionObserverEntry[]) => { + for (const entry of entries) { + const cellId = entry.target.getAttribute("data-cell-id") + if (cellId) engine.setVisible(cellId, entry.isIntersecting) + } + } + + const observeCells = () => { + const nodes = Array.from( + document.querySelectorAll(CELL_SELECTOR), + ) + const unchanged = + nodes.length === observed.size && nodes.every((n) => observed.has(n)) + if (unchanged) return + observer?.disconnect() + observer = null + observed = new Set(nodes) + if (nodes.length === 0) return + const root = nodes[0].closest(SCROLL_CONTAINER_SELECTOR) + if (!root) { + // Scroll container not found, do not block any cells + for (const node of nodes) { + const cellId = node.getAttribute("data-cell-id") + if (cellId) engine.setVisible(cellId, true) + } + return + } + observer = new IntersectionObserver(handleEntries, { + root, + rootMargin: OVERSCAN_ROOT_MARGIN, + }) + for (const node of nodes) observer.observe(node) + } + + const mutations = new MutationObserver(observeCells) + mutations.observe(document.body, { childList: true, subtree: true }) + observeCells() + + return () => { + mutations.disconnect() + observer?.disconnect() + } + }, [engine]) +} diff --git a/src/scenes/Editor/Notebook/declareUtils.ts b/src/scenes/Editor/Notebook/declareUtils.ts index 52a04b83d..ab1dca73b 100644 --- a/src/scenes/Editor/Notebook/declareUtils.ts +++ b/src/scenes/Editor/Notebook/declareUtils.ts @@ -1,6 +1,7 @@ import { parse, tokenize } from "@questdb/sql-parser" import type { CstNode, IToken } from "@chevrotain/types" import type { NotebookVariable } from "../../../store/notebook" +import type { Client } from "../../../utils/questdb/client" const NAME_RE = /^[a-zA-Z\u0080-\uFFFF_][a-zA-Z0-9\u0080-\uFFFF_]*$/ @@ -215,6 +216,31 @@ export const mapWireErrorPosition = ( return { kind: "shifted", position: wirePosition - range.delta } } +export const createValidateWithGlobals = + ( + quest: Pick, + getVariables: () => NotebookVariable[] | undefined, + ) => + async (sql: string, signal?: AbortSignal) => { + const variables = normalizeVariables(getVariables()) + const prepared = + variables.length > 0 + ? prependGlobalsDeclare(sql, variables) + : { sql, insertedRange: null } + const result = await quest.validateQuery(prepared.sql, signal) + if (!("error" in result) || !prepared.insertedRange) return result + const mapped = mapWireErrorPosition(prepared.insertedRange, result.position) + if (mapped.kind === "passthrough") return result + if (mapped.kind === "inDeclareBlock") { + return { + ...result, + position: mapped.position, + error: `${result.error} (in DECLARE block)`, + } + } + return { ...result, position: mapped.position } + } + const NO_OP = (sql: string): PreparedSql => ({ sql, insertedRange: null }) const renderMergedDeclare = ( diff --git a/src/scenes/Editor/Notebook/globals/useValidateWithGlobals.ts b/src/scenes/Editor/Notebook/globals/useValidateWithGlobals.ts index aa51979ed..8d85ff777 100644 --- a/src/scenes/Editor/Notebook/globals/useValidateWithGlobals.ts +++ b/src/scenes/Editor/Notebook/globals/useValidateWithGlobals.ts @@ -1,44 +1,14 @@ -import { useCallback, useContext } from "react" +import { useContext, useMemo } from "react" import { QuestContext } from "../../../../providers/QuestProvider" import { useNotebookActions } from "../NotebookProvider" -import { - mapWireErrorPosition, - normalizeVariables, - prependGlobalsDeclare, -} from "../declareUtils" +import { createValidateWithGlobals } from "../declareUtils" -// Wraps quest.validateQuery so callers can pass the user's original SQL while -// the server sees the wire form (with notebook globals injected as a DECLARE -// block). When the server reports an error position, we translate it back to -// the original-SQL coordinate system so Monaco markers land on the user's -// typo, not on our injected DECLARE prefix. export const useValidateWithGlobals = () => { const { quest } = useContext(QuestContext) const { getVariables } = useNotebookActions() - return useCallback( - async (sql: string, signal?: AbortSignal) => { - const variables = normalizeVariables(getVariables()) - const prepared = - variables.length > 0 - ? prependGlobalsDeclare(sql, variables) - : { sql, insertedRange: null } - const result = await quest.validateQuery(prepared.sql, signal) - if (!("error" in result) || !prepared.insertedRange) return result - const mapped = mapWireErrorPosition( - prepared.insertedRange, - result.position, - ) - if (mapped.kind === "passthrough") return result - if (mapped.kind === "inDeclareBlock") { - return { - ...result, - position: mapped.position, - error: `${result.error} (in DECLARE block)`, - } - } - return { ...result, position: mapped.position } - }, + return useMemo( + () => createValidateWithGlobals(quest, getVariables), [quest, getVariables], ) } diff --git a/src/scenes/Editor/Notebook/index.tsx b/src/scenes/Editor/Notebook/index.tsx index c1b441d8e..5c5d4035a 100644 --- a/src/scenes/Editor/Notebook/index.tsx +++ b/src/scenes/Editor/Notebook/index.tsx @@ -55,6 +55,7 @@ import { import { eventBus } from "../../../modules/EventBus" import { EventType } from "../../../modules/EventBus/types" import { consumeReveal, getPendingReveal } from "./cellReveal" +import { useChartCellVisibility } from "./chartRefresh/useChartCellVisibility" const GRID_COLS = NOTEBOOK_GRID_COLS const ROW_HEIGHT = NOTEBOOK_GRID_ROW_HEIGHT @@ -334,6 +335,7 @@ const ListLayout: React.FC = () => { return ( { const target = e.target as HTMLElement if (!target.closest("[data-notebook-cell]")) setFocusedCell(null) @@ -538,6 +540,7 @@ const GridLayout: React.FC = () => { return ( } + id="notebook-scroll-container" $suppressTransitions={!gridReady} onMouseDown={(e) => { const target = e.target as HTMLElement @@ -718,6 +721,7 @@ const NotebookContent: React.FC = () => { useScrollRestoredCellIntoView(maximizedCellId) useScrollFocusedCellIntoViewOnOpen(focusedCellId, isHydrating) useNotebookSearchReveal() + useChartCellVisibility() if (cells.length === 0) { return ( diff --git a/src/utils/index.ts b/src/utils/index.ts index 3fdf53168..b8f92b23d 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -37,3 +37,5 @@ export * from "./fetchUserLocale" export * from "./getLocaleFromLanguage" export * from "./uniq" export * from "./hashString" +export * from "./sleep" +export * from "./limiter" diff --git a/src/utils/limiter.ts b/src/utils/limiter.ts new file mode 100644 index 000000000..35f5dbef4 --- /dev/null +++ b/src/utils/limiter.ts @@ -0,0 +1,28 @@ +export const createLimiter = (limit: number) => { + let active = 0 + const waiting: Array<() => void> = [] + const acquire = () => + new Promise((resolve) => { + if (active < limit) { + active++ + resolve() + } else { + waiting.push(() => { + active++ + resolve() + }) + } + }) + const release = () => { + active-- + waiting.shift()?.() + } + return async (task: () => Promise): Promise => { + await acquire() + try { + return await task() + } finally { + release() + } + } +} diff --git a/src/utils/sleep.ts b/src/utils/sleep.ts new file mode 100644 index 000000000..e1ee10fc8 --- /dev/null +++ b/src/utils/sleep.ts @@ -0,0 +1,12 @@ +export const sleep = (ms: number, signal?: AbortSignal): Promise => + new Promise((resolve) => { + if (signal?.aborted) { + resolve(true) + return + } + const timeoutId = setTimeout(() => resolve(false), ms) + signal?.addEventListener("abort", () => { + clearTimeout(timeoutId) + resolve(true) + }) + })