// Documents the user uploads, and the pages agents read: their text in the research store under an
// id the store gives, which an agent's documents name. Pure: the names, which files are read, and a
// PDF's text as pdf.js gives it.

export interface Doc {
  name: string
  /** Its id in the store, which an agent's documents name. */
  id: string
  /** Characters of text. */
  size: number
  /** Milliseconds since the epoch. */
  modified: number
}

export const FOLDER = 'docs'
/** Files that are read: text, and PDFs with a text layer. Anything else (Word, images, scans) is not read yet. */
export const READ_TYPES = ['txt', 'md', 'markdown', 'csv', 'tsv', 'json', 'html', 'htm', 'xml', 'log', 'pdf']
const NAME_MAX = 120

/** A file's name as a document's: letters, digits, dot, dash, underscore; the rest become dashes. */
export function docName(fileName: string): string {
  const base = fileName.slice(fileName.lastIndexOf('/') + 1).trim()
  const name = base
    .replace(/[^\w.-]+/g, '-')
    .replace(/^[.-]+/, '')
    .slice(0, NAME_MAX)
  if (!name) throw new Error(`${fileName} has no usable name.`)
  return name
}

export function extensionOf(name: string): string {
  const dot = name.lastIndexOf('.')
  return dot === -1 ? '' : name.slice(dot + 1).toLowerCase()
}

export function isReadable(name: string): boolean {
  return READ_TYPES.includes(extensionOf(name))
}

export function isPdf(name: string): boolean {
  return extensionOf(name) === 'pdf'
}

export function isMarkup(name: string): boolean {
  return ['html', 'htm'].includes(extensionOf(name))
}

/** One run of text on a PDF page, as pdf.js's getTextContent gives it; marked-content items have no `str`. */
export interface PdfItem {
  str?: string
  hasEOL?: boolean
}

/**
 * A PDF's text, page by page, each page under a `[page N]` line so a passage says where it came from.
 * Runs are joined as pdf.js's own viewer joins them: pdf.js puts the spaces in, and a line ends where it
 * says one does.
 */
export function pdfText(pages: PdfItem[][]): string {
  return pages
    .map((items, i) => {
      const text = items
        .map((item) => (item.str ?? '') + (item.hasEOL ? '\n' : ''))
        .join('')
        .replace(/[ \t]+\n/g, '\n')
        .trim()
      return text ? `[page ${i + 1}]\n${text}` : ''
    })
    .filter(Boolean)
    .join('\n\n')
}