token-classification.ts

// =============================================================================
// Token Classification for Autocomplete
// =============================================================================
// These sets define how tokens are classified for autocomplete suggestions.
//
// This is the SINGLE SOURCE OF TRUTH for autocomplete - the UI should NOT
// duplicate these definitions.
// =============================================================================

import {
  IDENTIFIER_KEYWORD_NAMES,
  TOKEN_NAME_TO_KEYWORD,
} from "../parser/tokens"

/**
 * Token types that represent actual identifiers
 */
export const IDENTIFIER_TOKENS = new Set([
  "Identifier",
  "QuotedIdentifier",
  "IdentifierKeyword",
])

/**
 * Keywords that can be used as identifiers in the grammar.
 *
 * These are tokens that appear as alternatives in the parser's `identifier` rule.
 * When these appear in nextTokenTypes, it means an identifier/column is expected,
 * so we should suggest columns and tables, not these keywords.
 *
 * Derived from IDENTIFIER_KEYWORD_NAMES in tokens.ts — single source of truth.
 */
export const IDENTIFIER_KEYWORD_TOKENS = IDENTIFIER_KEYWORD_NAMES

/**
 * Expression-level operators and keywords (as opposed to clause-level keywords).
 */
export const EXPRESSION_OPERATORS = new Set([
  "And",
  "Or",
  "Not",
  "Between",
  "In",
  "Is",
  "Like",
  "Ilike",
  "Within",
  "All",
  "Any",
  "Some",
  "Case",
  "Cast",
])

/**
 * Punctuation tokens — worth suggesting in fallback (e.g., "(" after "VALUES (1), ")
 */
export const PUNCTUATION_TOKENS = new Set([
  "LParen",
  "RParen",
  "Comma",
  "Semicolon",
  "LBracket",
  "RBracket",
])

/**
 * Token types that should NOT be suggested (internal/structural tokens)
 */
export const SKIP_TOKENS = new Set([
  // Punctuation
  "LParen",
  "RParen",
  "Comma",
  "Dot",
  "Semicolon",
  "AtSign",
  "ColonEquals",
  "LBracket",
  "RBracket",
  // Comparison operators (suggest columns/values, not operators)
  "Equals",
  "NotEquals",
  "LessThan",
  "LessThanOrEqual",
  "GreaterThan",
  "GreaterThanOrEqual",
  // Arithmetic operators
  "Plus",
  "Minus",
  "Star", // Note: Star is also SELECT * - handled specially
  "Divide",
  "Modulo",
  "Concat",
  "DoubleColon",
  "RegexMatch",
  "RegexNotMatch",
  "RegexNotEquals",
  // IPv4 containment operators
  "IPv4ContainedBy",
  "IPv4ContainedByOrEqual",
  "IPv4Contains",
  "IPv4ContainsOrEqual",
  // Bitwise operators
  "BitAnd",
  "BitXor",
  "BitOr",
  // Variable references (user-defined, can't suggest)
  "VariableReference",
  // Literals (don't suggest literal tokens)
  "StringLiteral",
  "NumberLiteral",
  "LongLiteral",
  "DecimalLiteral",
  "DurationLiteral",
  "GeohashLiteral",
  "GeohashBinaryLiteral",
  "Nan",
  // Whitespace
  "WhiteSpace",
])

/**
 * Operator map for converting token names to display strings
 */
export const OPERATOR_MAP: Record<string, string> = {
  Star: "*",
  Plus: "+",
  Minus: "-",
  Divide: "/",
  Modulo: "%",
  Equals: "=",
  NotEquals: "!=",
  LessThan: "<",
  LessThanOrEqual: "<=",
  GreaterThan: ">",
  GreaterThanOrEqual: ">=",
  LParen: "(",
  RParen: ")",
  Comma: ",",
  Semicolon: ";",
}

/**
 * Map from token name → actual SQL keyword.
 * Built during token generation in tokens.ts — no regex parsing needed.
 * e.g., "DataPageSize" → "DATA_PAGE_SIZE", "Select" → "SELECT"
 */
const TOKEN_KEYWORD_MAP = TOKEN_NAME_TO_KEYWORD

/**
 * Convert a token type name to a keyword string for display
 * e.g., "Table" → "TABLE", "DataPageSize" → "DATA_PAGE_SIZE"
 */
export function tokenNameToKeyword(name: string): string {
  if (OPERATOR_MAP[name]) {
    return OPERATOR_MAP[name]
  }

  // Use the actual keyword from the lexer pattern (preserves underscores)
  const keyword = TOKEN_KEYWORD_MAP.get(name)
  if (keyword) return keyword

  // Fallback for non-keyword tokens
  return name.replace(/([a-z])([A-Z])/g, "$1 $2").toUpperCase()
}

/**
 * Check if a token represents an identifier context
 */
export function isIdentifierToken(tokenName: string): boolean {
  return (
    IDENTIFIER_TOKENS.has(tokenName) || IDENTIFIER_KEYWORD_TOKENS.has(tokenName)
  )
}

/**
 * Check if a token should be skipped in suggestions
 */
export function shouldSkipToken(tokenName: string): boolean {
  return SKIP_TOKENS.has(tokenName)
}