feat: refactor data source management and enhance UI components for better usability

Author: JulianFun123

packages/backend/package.json

@@ -14,6 +14,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "@elastic/elasticsearch": "^9.3.4",
     "@types/body-parser": "^1.19.5",
     "@types/express": "^5.0.2",
     "@types/express-ws": "^3.0.5",
@@ -22,8 +23,10 @@
     "drizzle-orm": "^0.45.1",
     "express": "^5.0.2",
     "express-ws": "^5.0.2",
+    "minio": "^8.0.7",
     "mysql2": "^3.14.1",
     "nodemon": "^3.1.10",
+    "openid-client": "^6.8.2",
     "pg": "^8.20.0",
     "ts-node": "^10.9.2",
     "tsx": "^4.21.0",

packages/backend/src/adapters/index.ts

@@ -0,0 +1,130 @@
+import type { NextFunction, Response } from 'express'
+import type { AppContext } from '../app-context.ts'
+import { adminPermissionTargets, hasAnyPermission } from './permissions.ts'
+import type { AuthenticatedRequest } from './request.ts'
+import type { AuthPrincipal, AuthStatusPayload } from './types.ts'
+import { hashAuthToken } from './tokens.ts'
+
+function createAnonymousPrincipal(): AuthPrincipal {
+  return {
+    kind: 'anonymous',
+    user: null,
+    roles: [],
+    permissions: [],
+    token: null,
+  }
+}
+
+function createLocalPrincipal(): AuthPrincipal {
+  return {
+    kind: 'local',
+    user: null,
+    roles: [],
+    permissions: ['*'],
+    token: null,
+  }
+}
+
+function getBearerToken(req: AuthenticatedRequest) {
+  const authorization = req.headers.authorization
+  if (!authorization?.startsWith('Bearer ')) {
+    return null
+  }
+
+  return authorization.slice('Bearer '.length).trim()
+}
+
+export async function buildAuthStatus(context: AppContext): Promise<AuthStatusPayload> {
+  return {
+    enabled: context.config.auth.enabled,
+    onboardingRequired: context.config.auth.enabled ? (await context.metaStore.countUsers()) === 0 : false,
+    openIdEnabled: Boolean(context.config.auth.openId),
+  }
+}
+
+export async function authenticateRequest(context: AppContext, req: AuthenticatedRequest) {
+  if (!context.config.auth.enabled) {
+    req.auth = createLocalPrincipal()
+    return req.auth
+  }
+
+  const rawToken = getBearerToken(req)
+  if (!rawToken) {
+    req.auth = createAnonymousPrincipal()
+    return req.auth
+  }
+
+  const tokenRecord = await context.metaStore.getAuthTokenByHash(hashAuthToken(rawToken))
+  if (!tokenRecord) {
+    req.auth = createAnonymousPrincipal()
+    return req.auth
+  }
+
+  if (tokenRecord.expiresAt && new Date(tokenRecord.expiresAt).getTime() < Date.now()) {
+    await context.metaStore.deleteAuthToken(tokenRecord.id)
+    req.auth = createAnonymousPrincipal()
+    return req.auth
+  }
+
+  const user = await context.metaStore.getUserWithRoles(tokenRecord.userId)
+  if (!user || user.disabled) {
+    req.auth = createAnonymousPrincipal()
+    return req.auth
+  }
+
+  await context.metaStore.updateAuthTokenLastUsed(tokenRecord.id)
+
+  req.auth = {
+    kind: tokenRecord.kind,
+    user,
+    roles: user.roles,
+    permissions: Array.from(new Set([...user.permissions, ...user.roles.flatMap((role) => role.permissions)])),
+    token: tokenRecord,
+  }
+
+  return req.auth
+}
+
+export function attachAuth(context: AppContext) {
+  return async (req: AuthenticatedRequest, _res: Response, next: NextFunction) => {
+    try {
+      await authenticateRequest(context, req)
+      next()
+    } catch (error) {
+      next(error)
+    }
+  }
+}
+
+export function getRequestAuth(req: AuthenticatedRequest) {
+  return req.auth ?? createAnonymousPrincipal()
+}
+
+export function requireAuthenticated(req: AuthenticatedRequest, res: Response) {
+  const auth = getRequestAuth(req)
+  if (auth.kind === 'anonymous') {
+    res.status(401).json({ error: 'Authentication required' })
+    return false
+  }
+
+  return true
+}
+
+export function requirePermission(req: AuthenticatedRequest, res: Response, requiredPermissions: string[]) {
+  const auth = getRequestAuth(req)
+  if (auth.kind === 'anonymous') {
+    res.status(401).json({ error: 'Authentication required' })
+    return false
+  }
+
+  if (!hasAnyPermission(auth.permissions, requiredPermissions)) {
+    res.status(403).json({ error: 'Forbidden', requiredPermissions })
+    return false
+  }
+
+  return true
+}
+
+export function requireAdminAccess(req: AuthenticatedRequest, res: Response) {
+  return requirePermission(req, res, adminPermissionTargets('access', 'read'))
+}

packages/backend/src/auth/middleware.ts

@@ -0,0 +1,26 @@
+import * as client from 'openid-client'
+import type { AppConfig } from '../config/app-config.ts'
+
+let cachedConfiguration: client.Configuration | null = null
+
+export async function getOpenIdConfiguration(config: AppConfig) {
+  if (!config.auth.openId) {
+    return null
+  }
+
+  if (cachedConfiguration) {
+    return cachedConfiguration
+  }
+
+  cachedConfiguration = await client.discovery(
+    new URL(config.auth.openId.issuer),
+    config.auth.openId.clientId,
+    config.auth.openId.clientSecret || undefined,
+  )
+
+  return cachedConfiguration
+}
+
+export {
+  client,
+}

packages/backend/src/auth/openid.ts

@@ -0,0 +1,29 @@
+import crypto from 'node:crypto'
+
+const SCRYPT_OPTIONS = {
+  N: 16384,
+  r: 8,
+  p: 1,
+}
+
+export function createPasswordHash(password: string) {
+  const salt = crypto.randomBytes(16).toString('hex')
+  const hash = crypto.scryptSync(password, salt, 64, SCRYPT_OPTIONS).toString('hex')
+
+  return {
+    salt,
+    hash,
+  }
+}
+
+export function verifyPasswordHash(password: string, salt: string, hash: string) {
+  const candidate = crypto.scryptSync(password, salt, 64, SCRYPT_OPTIONS)
+  const target = Buffer.from(hash, 'hex')
+
+  if (candidate.length !== target.length) {
+    return false
+  }
+
+  return crypto.timingSafeEqual(candidate, target)
+}
+

packages/backend/src/auth/passwords.ts

@@ -0,0 +1,90 @@
+export type PermissionAccess = 'read' | 'write'
+
+function escapeRegex(value: string) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+function parsePermission(value: string) {
+  const match = value.match(/^(.*?)(?::(read|write))?$/)
+  return {
+    path: match?.[1] ?? value,
+    access: (match?.[2] as PermissionAccess | undefined) ?? null,
+  }
+}
+
+function applyPermissionAccess(target: string, access?: PermissionAccess) {
+  if (!access || target === '*') {
+    return parsePermission(target).path
+  }
+
+  return `${parsePermission(target).path}:${access}`
+}
+
+function withPermissionAccess(targets: string[], access?: PermissionAccess) {
+  return Array.from(new Set(targets.map((target) => applyPermissionAccess(target, access))))
+}
+
+export function permissionPatternMatches(pattern: string, requiredPermission: string) {
+  if (pattern === '*') {
+    return true
+  }
+
+  const parsedPattern = parsePermission(pattern)
+  const parsedRequired = parsePermission(requiredPermission)
+  const regex = new RegExp(`^${escapeRegex(parsedPattern.path).replace(/\\\*/g, '.*')}$`)
+  if (!regex.test(parsedRequired.path)) {
+    return false
+  }
+
+  if (!parsedRequired.access) {
+    return true
+  }
+
+  return !parsedPattern.access || parsedPattern.access === parsedRequired.access
+}
+
+export function hasPermission(permissionPatterns: string[], requiredPermission: string) {
+  return permissionPatterns.some((pattern) => permissionPatternMatches(pattern, requiredPermission))
+}
+
+export function hasAnyPermission(permissionPatterns: string[], requiredPermissions: string[]) {
+  return requiredPermissions.some((permission) => hasPermission(permissionPatterns, permission))
+}
+
+export function projectPermissionTargets(
+  projectId: string,
+  action: 'view' | 'manage' | 'create' | 'users',
+  access?: PermissionAccess,
+) {
+  const targets = [`project.${action}`, `project.${action}.*`]
+
+  if (action !== 'create') {
+    targets.push(`project.${action}.${projectId}`)
+  }
+
+  return withPermissionAccess(targets, access)
+}
+
+export function dataSourcePermissionTargets(
+  projectId: string,
+  sourceId: string | '*',
+  action: 'view' | 'manage' | 'query' | 'table.edit',
+  access?: PermissionAccess,
+) {
+  return withPermissionAccess(
+    [
+      `datasource.${action}`,
+      `datasource.${action}.*`,
+      `datasource.${action}.${projectId}.*`,
+      `datasource.${action}.${projectId}.${sourceId}`,
+    ],
+    access,
+  )
+}
+
+export function adminPermissionTargets(
+  action: 'access' | 'users' | 'roles' | 'apiKeys',
+  access?: PermissionAccess,
+) {
+  return withPermissionAccess([`admin.${action}`, 'admin.*', '*'], access)
+}

packages/backend/src/auth/permissions.ts

@@ -0,0 +1,7 @@
+import type { Request } from 'express'
+import type { AuthPrincipal } from './types.ts'
+
+export type AuthenticatedRequest = Request & {
+  auth: AuthPrincipal
+}
+

packages/backend/src/auth/request.ts

@@ -0,0 +1,43 @@
+import type { AuthTokenRecord, RoleRecord, UserWithRolesRecord } from '../meta/types.ts'
+
+export function serializeRole(role: RoleRecord) {
+  return {
+    id: role.id,
+    slug: role.slug,
+    name: role.name,
+    description: role.description,
+    permissions: role.permissions,
+    createdAt: role.createdAt,
+    updatedAt: role.updatedAt,
+  }
+}
+
+export function serializeUser(user: UserWithRolesRecord) {
+  return {
+    id: user.id,
+    email: user.email,
+    name: user.name,
+    authProvider: user.authProvider,
+    externalSubject: user.externalSubject,
+    disabled: user.disabled,
+    permissions: user.permissions,
+    roleIds: user.roleIds,
+    roles: user.roles.map((role) => serializeRole(role)),
+    createdAt: user.createdAt,
+    updatedAt: user.updatedAt,
+  }
+}
+
+export function serializeAuthToken(token: AuthTokenRecord) {
+  return {
+    id: token.id,
+    userId: token.userId,
+    kind: token.kind,
+    name: token.name,
+    tokenPrefix: token.tokenPrefix,
+    storage: token.storage,
+    expiresAt: token.expiresAt,
+    lastUsedAt: token.lastUsedAt,
+    createdAt: token.createdAt,
+  }
+}

packages/backend/src/auth/serializers.ts

@@ -0,0 +1,21 @@
+import crypto from 'node:crypto'
+
+export function createRawAuthToken() {
+  const raw = crypto.randomBytes(32).toString('base64url')
+  const prefix = raw.slice(0, 12)
+
+  return {
+    raw,
+    prefix,
+    hash: crypto.createHash('sha256').update(raw).digest('hex'),
+  }
+}
+
+export function hashAuthToken(raw: string) {
+  return crypto.createHash('sha256').update(raw).digest('hex')
+}
+
+export function createOpaqueStateToken() {
+  return crypto.randomBytes(24).toString('base64url')
+}
+