import { useData } from '@jaspers-ai/sdk' import { useEffect, useRef, useState } from 'react' import type { FeedResult } from './Feeds' import { mergeBars, type Bar, type Interval } from './plan' import { databentoArgs, databentoDaily, databentoRetryEnd, isRateLimit, massiveArgs, massiveBars, weeklyBars } from './sources' /** Bars kept across source and interval switches: "massive::" and "databento:" (daily). */ export type BarCache = Map interface Job { id: number symbol: string kind: 'full' | 'recent' attempt: number args: Record /** Databento: the earlier end a refused call is retried at, and how many times it was moved. */ end?: string endTries: number } // Massive's limit is per minute: the first wait is half a window, then whole windows. const BACKOFF_S = [30, 65, 65, 90, 120, 120] /** * Loads Massive or Databento bars one symbol at a time, through the installed plugins' sources: a full * history on first sight of a symbol, a short recent window merged in on each refresh tick, and a * rate-limited call waited out and tried again. Draws nothing; reports each symbol under * "::". */ export function QueuedFeeds(props: { source: 'massive' | 'databento' symbols: string[] interval: Interval tick: number cache: BarCache onResult: (key: string, result: FeedResult) => void }) { const { source, symbols, interval, tick, cache, onResult } = props const [queue, setQueue] = useState([]) const [waiting, setWaiting] = useState(false) const known = useRef(new Set()) const ids = useRef(1) const timer = useRef | null>(null) const firstTick = useRef(tick) useEffect(() => () => void (timer.current && clearTimeout(timer.current)), []) const cacheKey = (s: string) => (source === 'databento' ? `databento:${s}` : `massive:${interval}:${s}`) const shown = (s: string) => { const bars = cache.get(cacheKey(s)) return bars && source === 'databento' && interval === '1wk' ? weeklyBars(bars) : bars } const report = (s: string, r: Partial) => onResult(`${source}:${interval}:${s}`, { bars: shown(s) ?? [], error: null, loading: false, ...r }) const job = (symbol: string, kind: 'full' | 'recent', attempt = 0, end?: string, endTries = 0): Job => { const now = new Date() const id = ids.current++ const args = source === 'massive' ? massiveArgs(symbol, interval, kind, now) : databentoArgs(symbol, kind, now, id + attempt, end) return { id, symbol, kind, attempt, args, end, endTries } } const local = (s: string) => (source === 'databento' && s.startsWith('^') ? 'no index data on Databento' : null) // New symbols: cached bars at once, else a full load queued. Removed symbols leave the queue. useEffect(() => { const add: Job[] = [] for (const s of symbols) { if (known.current.has(s)) continue known.current.add(s) const refused = local(s) if (refused) report(s, { error: refused, note: 'failed' }) else if (cache.has(cacheKey(s))) report(s, {}) else { report(s, { loading: true, note: 'queued' }) add.push(job(s, 'full')) } } for (const s of [...known.current]) if (!symbols.includes(s)) known.current.delete(s) setQueue((q) => [...q.filter((j) => symbols.includes(j.symbol)), ...add]) }, [symbols.join(',')]) // Each refresh: a recent window for every cached symbol, the full load again for one that failed. useEffect(() => { if (tick === firstTick.current) return setQueue((q) => { const add: Job[] = [] for (const s of symbols) { if (local(s) || q.some((j) => j.symbol === s)) continue add.push(job(s, cache.has(cacheKey(s)) ? 'recent' : 'full')) } return [...q, ...add] }) }, [tick]) const done = (j: Job, result: { rows?: unknown[]; error?: string }) => { const key = cacheKey(j.symbol) const had = cache.has(key) if (result.error !== undefined) { const message = result.error.split('\n')[0].slice(0, 200) // Databento has not published up to the end asked: try once more at the end it names, quietly. const earlier = source === 'databento' && j.endTries < 1 ? databentoRetryEnd(result.error, String(j.args.end)) : null if (earlier) { const next = job(j.symbol, j.kind, j.attempt, earlier, j.endTries + 1) setQueue((q) => q.map((x) => (x.id === j.id ? next : x))) return } if (isRateLimit(message) && j.attempt < BACKOFF_S.length) { const wait = BACKOFF_S[j.attempt] report(j.symbol, { loading: !had, note: `rate-limited, retrying in ${wait}s`, error: null, detail: message }) setWaiting(true) timer.current = setTimeout(() => { setQueue((q) => (q[0]?.id === j.id ? [job(j.symbol, j.kind, j.attempt + 1, j.end, j.endTries), ...q.slice(1)] : q)) setWaiting(false) }, wait * 1000) return } // An index the Massive plan does not cover says so, rather than the raw 403. const shownError = source === 'massive' && String(j.args.ticker).startsWith('I:') && /\b403\b|not_authorized|not entitled|does not include/i.test(message) ? 'no index data on your plan' : message // A refresh that fails keeps the bars already drawn. report(j.symbol, had ? { note: 'refresh failed', detail: message } : { error: shownError, note: 'failed', detail: message }) } else { const rows = result.rows ?? [] const bars = source === 'massive' ? massiveBars(rows, interval, j.kind) : databentoDaily(rows) if (j.kind === 'full') { if (bars.length > 0) cache.set(key, bars) } else if (bars.length > 0) cache.set(key, mergeBars([cache.get(key) ?? [], bars])) if (cache.has(key)) report(j.symbol, {}) else report(j.symbol, { error: 'No data', note: 'failed' }) // Databento's end is exclusive: the full load ends yesterday, so ask at once for the last published day. if (source === 'databento' && j.kind === 'full' && bars.length > 0) { const next = job(j.symbol, 'recent') setQueue((q) => [...q.filter((x) => x.id !== j.id), next]) return } } setQueue((q) => q.filter((x) => x.id !== j.id)) } const head = waiting ? undefined : queue[0] // The symbol now loading shows as loading, not queued; a refresh keeps its chip as it is. useEffect(() => { if (head && head.kind === 'full') report(head.symbol, { loading: true, note: head.attempt > 0 ? 'retrying' : undefined }) }, [head?.id]) if (!head) return null return ( done(head, r)} /> ) } /** One call to another plugin's source; reports once when it answers. */ function Request(props: { sourceId: string; args: Record; onDone: (r: { rows?: unknown[]; error?: string }) => void }) { const { data, text, error, loading } = useData(props.sourceId, props.args) const fired = useRef(false) const onDone = useRef(props.onDone) onDone.current = props.onDone useEffect(() => { if (fired.current) return if (error) { fired.current = true onDone.current({ error: String(error) }) } else if (!loading && Array.isArray(data)) { fired.current = true onDone.current({ rows: data as unknown[] }) } else if (!loading && data === undefined && text) { fired.current = true onDone.current({ error: text }) } }, [data, text, error, loading]) return null }