// The three bar sources the chart can read, what each offers, and how the Massive and Databento // plugins' rows become chart bars. Yahoo's own plan stays in plan.ts. import { INTERVALS, isIntraday, mergeBars, toBars, type Bar, type Interval } from './plan' export const SOURCES = ['yahoo', 'massive', 'databento'] as const export type Source = (typeof SOURCES)[number] export const SOURCE_NAMES: Record = { yahoo: 'Yahoo Finance', massive: 'Massive (delayed)', databento: 'Databento DBEQ.BASIC', } export const SOURCE_INTERVALS: Record = { yahoo: INTERVALS, massive: INTERVALS, databento: ['1d', '1wk'], } export const DATABENTO_INTRADAY_TIP = "Databento intraday bars aren't available in the installed databento plugin" /** The interval the chart draws: the one asked for when the source offers it, else 1d. */ export function effectiveInterval(source: Source, interval: Interval): Interval { return SOURCE_INTERVALS[source].includes(interval) ? interval : '1d' } const day = (now: Date, offset: number) => new Date(now.getTime() + offset * 86_400_000).toISOString().slice(0, 10) // ---- Massive ------------------------------------------------------------------------------------ const INDEX: Record = { '^GSPC': 'I:SPX', '^SPX': 'I:SPX', '^NDX': 'I:NDX', '^DJI': 'I:DJI', '^RUT': 'I:RUT', '^VIX': 'I:VIX', } /** A Yahoo-style symbol as Massive's ticker: ^GSPC is I:SPX, any other ^XYZ is tried as I:XYZ. */ export function massiveTicker(symbol: string): string { const s = symbol.toUpperCase() return INDEX[s] ?? (s.startsWith('^') ? `I:${s.slice(1)}` : s) } // How far back one call goes. Massive's minute and hour aggregates cover regular AND extended hours // (pre-market and after-hours, about 04:00-20:00 ET), so a busy day is up to 960 one-minute bars; each // window keeps a liquid stock's bar count under the 50,000 cap of one call (1m ~40k, 2m ~40k, 5m ~24k, // 15m ~24k, 30m ~16k, 1h ~8k, 1d ~5k, 1wk ~1k). recent is the window a 5-minute refresh asks for. const MASSIVE: Record = { '1m': { multiplier: 1, timespan: 'minute', days: 60, recent: 4 }, '2m': { multiplier: 2, timespan: 'minute', days: 120, recent: 4 }, '5m': { multiplier: 5, timespan: 'minute', days: 180, recent: 4 }, '15m': { multiplier: 15, timespan: 'minute', days: 548, recent: 4 }, '30m': { multiplier: 30, timespan: 'minute', days: 730, recent: 4 }, '1h': { multiplier: 1, timespan: 'hour', days: 730, recent: 4 }, '1d': { multiplier: 1, timespan: 'day', days: 7305, recent: 10 }, '1wk': { multiplier: 1, timespan: 'week', days: 7305, recent: 21 }, } /** One massive/aggregates call. `to` is now in milliseconds, so every call is a fresh run. */ export function massiveArgs(symbol: string, interval: Interval, kind: 'full' | 'recent', now: Date) { const plan = MASSIVE[interval] return { ticker: massiveTicker(symbol), multiplier: plan.multiplier, timespan: plan.timespan, from: day(now, -(kind === 'full' ? plan.days : plan.recent)), to: String(now.getTime()), adjusted: true, sort: 'asc', limit: 50000, } } /** Massive's rows (time as ISO UTC) as bars: New York wall clock intraday, dates for 1d and 1wk. */ export function massiveBars(rows: unknown[], interval: Interval, kind: 'full' | 'recent'): Bar[] { const bars = toBars(rows, interval) // A weekly window that starts mid-week answers a partial first week: leave the cached one. return kind === 'recent' && interval === '1wk' ? bars.slice(1) : bars } // ---- Databento ---------------------------------------------------------------------------------- const NY_DATE = new Intl.DateTimeFormat('en-CA', { timeZone: 'America/New_York', year: 'numeric', month: '2-digit', day: '2-digit', }) /** A YYYY-MM-DD date moved by whole days. */ export function shiftDate(date: string, days: number): string { return new Date(Date.parse(`${date}T00:00:00Z`) + days * 86_400_000).toISOString().slice(0, 10) } /** The New York calendar date of an instant, moved by whole days. */ export function nyDate(now: Date, offset = 0): string { const p: Record = {} for (const part of NY_DATE.formatToParts(now)) p[part.type] = part.value return shiftDate(`${p.year}-${p.month}-${p.day}`, offset) } const DATABENTO_START = '2023-03-28' /** * DBEQ.BASIC begins 2023-03-28: an earlier start is refused (422 data_start_before_available_start), so * the full load starts there. Databento refuses the whole call (422 data_end_after_available_end) when * end is past what it has published, so the full load ends yesterday (New York) and a refresh tries * today, each retried once at the end the error names (see databentoRetryEnd). A weekend or holiday * end is fine: Databento answers the trading days before it. nonce varies the limit a little so a * retry or a refresh is a fresh run. */ export function databentoArgs(symbol: string, kind: 'full' | 'recent', now: Date, nonce: number, endOverride?: string) { const end = endOverride ?? nyDate(now, kind === 'full' ? -1 : 0) const start = kind === 'full' ? DATABENTO_START : shiftDate(end, -7) return { ticker: symbol.toUpperCase(), start: start < end ? start : shiftDate(end, -1), end, limit: 50000 - (nonce % 1000), } } /** * When Databento refused a call because its end is past the published data, the end to try instead: * the date the error names ("data available up to '2026-10-22 00:00:00+00:00'") when it is earlier than * the end tried, else one day before it. Null for any other error. */ export function databentoRetryEnd(message: string, triedEnd: string): string | null { if (!/data_end_after_available_end|after the available range|available up to|data.?not.?available|not yet available/i.test(message)) return null const named = /available up to '?(\d{4}-\d{2}-\d{2})/i.exec(message)?.[1] const end = named && named < triedEnd ? named : shiftDate(triedEnd, -1) return end > DATABENTO_START ? end : null } interface VenueRow { date?: unknown open?: unknown high?: unknown low?: unknown close?: unknown volume?: unknown } /** One bar per day from one row per day per venue: high max, low min, volume summed, open and close from the busiest venue. */ export function databentoDaily(rows: unknown[]): Bar[] { const days = new Map() for (const item of rows) { if (!item || typeof item !== 'object') continue const r = item as VenueRow const date = typeof r.date === 'string' ? r.date.slice(0, 10) : '' const [open, high, low, close, volume] = [r.open, r.high, r.low, r.close, r.volume].map(Number) if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || !Number.isFinite(close)) continue const v = Number.isFinite(volume) ? volume : 0 const o = Number.isFinite(open) ? open : close const h = Number.isFinite(high) ? high : close const l = Number.isFinite(low) ? low : close const seen = days.get(date) if (!seen) { days.set(date, { bar: { time: date, open: o, high: h, low: l, close, volume: v }, top: v }) continue } const b = seen.bar b.high = Math.max(b.high, h) b.low = Math.min(b.low, l) b.volume += v if (v > seen.top) { seen.top = v b.open = o b.close = close } } return mergeBars([[...days.values()].map((d) => d.bar)]) } /** Daily bars as weeks starting Monday, each dated its Monday. */ export function weeklyBars(daily: Bar[]): Bar[] { const weeks = new Map() for (const d of daily) { const t = new Date(`${String(d.time)}T00:00:00Z`) const monday = new Date(t.getTime() - ((t.getUTCDay() + 6) % 7) * 86_400_000).toISOString().slice(0, 10) const w = weeks.get(monday) if (!w) weeks.set(monday, { ...d, time: monday }) else { w.high = Math.max(w.high, d.high) w.low = Math.min(w.low, d.low) w.close = d.close w.volume += d.volume } } return mergeBars([[...weeks.values()]]) } /** Whether a source's error is a rate limit worth waiting out. */ export function isRateLimit(message: string): boolean { return /\b429\b|rate.?limit|too many requests|maximum requests|exceeded the maximum/i.test(message) } export { isIntraday }