Skip to content

Commit 9fa2413

Browse files
ysyneuysyneu
committed
Add Meilisearch upload workflow and script for AI Q&A bot indexing
Made-with: Cursor
1 parent 218113f commit 9fa2413

3 files changed

Lines changed: 314 additions & 0 deletions

File tree

‎.github/workflows/upload.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# Upload documentation to Meilisearch for AI Q&A bot indexing.
2+
#
3+
# Uses GitHub Environments for dev/prod separation. Create two environments in
4+
# Settings > Environments: "development" and "production".
5+
#
6+
# Per-environment secrets (all required, keep as secrets to hide Meilisearch endpoint):
7+
# MEILI_ENDPOINT - Meilisearch instance URL
8+
# MEILI_API_KEY - API key with documents write permission
9+
# MEILI_INDEX - Target index name
10+
#
11+
# Branch mapping: main -> production, test -> development
12+
13+
name: Upload docs to Meilisearch
14+
15+
on:
16+
push:
17+
branches: [main, test]
18+
workflow_dispatch:
19+
inputs:
20+
environment:
21+
description: 'Target environment'
22+
required: true
23+
default: 'production'
24+
type: choice
25+
options:
26+
- production
27+
- development
28+
29+
jobs:
30+
upload:
31+
runs-on: ubuntu-latest
32+
environment: ${{ github.event.inputs.environment || (github.ref == 'refs/heads/main' && 'production' || 'development') }}
33+
steps:
34+
- name: Checkout
35+
uses: actions/checkout@v4
36+
37+
- name: Install dependencies
38+
run: |
39+
sudo apt-get update
40+
sudo apt-get install -y jq
41+
jq --version
42+
43+
- name: Upload to Meilisearch
44+
env:
45+
MEILI_ENDPOINT: ${{ secrets.MEILI_ENDPOINT }}
46+
MEILI_API_KEY: ${{ secrets.MEILI_API_KEY }}
47+
MEILI_INDEX: ${{ secrets.MEILI_INDEX }}
48+
run: |
49+
chmod +x ./scripts/upload.sh
50+
bash ./scripts/upload.sh

‎AGENTS.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ mint dev
6262

6363
# Check broken links
6464
mint broken-links
65+
66+
# Upload docs to Meilisearch (for AI Q&A bot)
67+
MEILI_ENDPOINT=... MEILI_API_KEY=... MEILI_INDEX=... bash scripts/upload.sh
6568
```
6669

6770
## Documentation Workflow

‎scripts/upload.sh‎

Lines changed: 261 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,261 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Upload documentation to Meilisearch for AI Q&A bot indexing.
4+
#
5+
# Reads markdown/mdx files from zh/ and en/, formats them into JSON,
6+
# and uploads to Meilisearch. Run from repository root, or the script
7+
# will cd to repo root automatically.
8+
#
9+
# Required env vars: MEILI_ENDPOINT, MEILI_API_KEY, MEILI_INDEX
10+
# Optional: BASE_URL (default https://docs.flashcat.cloud)
11+
#
12+
# Usage:
13+
# sh scripts/upload.sh [--dry-run] [--help]
14+
#
15+
# License: Same as the repository
16+
17+
set -euo pipefail
18+
19+
# Ensure we run from repo root (parent of scripts/)
20+
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
21+
REPO_ROOT=$(cd "$SCRIPT_DIR/.." && pwd)
22+
cd "$REPO_ROOT"
23+
24+
# --- Constants ---
25+
SCRIPT_NAME=$(basename "$0")
26+
BASE_URL="${BASE_URL:-https://docs.flashcat.cloud}"
27+
DRY_RUN=false
28+
29+
# --- Help ---
30+
usage() {
31+
cat <<EOF
32+
Usage: $SCRIPT_NAME [OPTIONS]
33+
34+
Upload documentation (zh/, en/) to Meilisearch for AI Q&A bot indexing.
35+
36+
Options:
37+
--dry-run Validate and list files without uploading
38+
-h, --help Show this help
39+
40+
Environment variables:
41+
MEILI_ENDPOINT Meilisearch instance URL (required)
42+
MEILI_API_KEY API key with documents write permission (required)
43+
MEILI_INDEX Target index name (required)
44+
BASE_URL Docs base URL for link construction (default: $BASE_URL)
45+
46+
Examples:
47+
MEILI_ENDPOINT=https://meilisearch.example.com \\
48+
MEILI_API_KEY=xxx MEILI_INDEX=docs sh scripts/upload.sh
49+
50+
sh scripts/upload.sh --dry-run
51+
EOF
52+
}
53+
54+
# --- Argument parsing ---
55+
for arg in "$@"; do
56+
case "$arg" in
57+
-h|--help)
58+
usage
59+
exit 0
60+
;;
61+
--dry-run)
62+
DRY_RUN=true
63+
;;
64+
*)
65+
echo "Unknown option: $arg" >&2
66+
usage >&2
67+
exit 1
68+
;;
69+
esac
70+
done
71+
72+
# --- Validation ---
73+
if [[ "$DRY_RUN" != true ]]; then
74+
if [[ -z "${MEILI_ENDPOINT:-}" || -z "${MEILI_API_KEY:-}" || -z "${MEILI_INDEX:-}" ]]; then
75+
echo "Error: MEILI_ENDPOINT, MEILI_API_KEY, and MEILI_INDEX must be set." >&2
76+
echo "Run with --help for usage." >&2
77+
exit 1
78+
fi
79+
fi
80+
81+
if ! command -v jq &> /dev/null; then
82+
echo "Error: jq is required. Install it first (e.g. brew install jq on macOS)." >&2
83+
exit 1
84+
fi
85+
86+
# --- Counters ---
87+
total_success=0
88+
total_files=0
89+
total_errors=0
90+
91+
# --- Upload a single document ---
92+
upload_document() {
93+
local json_payload=$1
94+
local title=$2
95+
96+
if [[ "$DRY_RUN" == true ]]; then
97+
echo "[dry-run] Would upload: $title"
98+
return 0
99+
fi
100+
101+
local response
102+
response=$(curl -sS --connect-timeout 30 --max-time 60 -X POST "$MEILI_ENDPOINT/indexes/$MEILI_INDEX/documents?primaryKey=id" \
103+
-H "Authorization: Bearer $MEILI_API_KEY" \
104+
-H "Content-Type: application/json" \
105+
--data-binary "$json_payload")
106+
107+
if [[ -z "$response" ]]; then
108+
echo "Upload failed: $title (empty response)" >&2
109+
return 1
110+
fi
111+
112+
if echo "$response" | jq -e '.taskUid' > /dev/null 2>&1; then
113+
echo "Uploaded: $title"
114+
return 0
115+
else
116+
echo "Upload failed: $title" >&2
117+
echo "Response: $response" >&2
118+
return 1
119+
fi
120+
}
121+
122+
# --- Extract title from frontmatter or filename ---
123+
extract_title() {
124+
local file=$1
125+
local title
126+
title=$(grep -m 1 '^title:' "$file" 2>/dev/null | sed -n 's/title: *"\(.*\)"/\1/p') || true
127+
if [[ -z "${title:-}" ]]; then
128+
title=$(grep -m 1 '^title:' "$file" 2>/dev/null | sed -n 's/title: *\(.*\)$/\1/p' | xargs) || true
129+
fi
130+
if [[ -z "${title:-}" ]]; then
131+
local base
132+
base=$(basename "$file")
133+
title="${base%.mdx}"
134+
title="${title%.md}"
135+
title=$(echo "$title" | sed 's/^[0-9.]*[[:space:]]*//')
136+
fi
137+
echo "$title"
138+
}
139+
140+
# --- Extract URL from frontmatter or construct from path ---
141+
extract_url() {
142+
local file=$1
143+
local dir=$2
144+
local locale=$3
145+
local doc_url
146+
doc_url=$(grep -m 1 '^url:' "$file" 2>/dev/null | sed -n 's/url: *"\(.*\)"/\1/p') || true
147+
if [[ -z "${doc_url:-}" ]]; then
148+
doc_url=$(grep -m 1 '^url:' "$file" 2>/dev/null | sed -n 's/url: *\(.*\)$/\1/p' | xargs) || true
149+
fi
150+
if [[ -z "${doc_url:-}" ]]; then
151+
local rel_path
152+
rel_path="${file#$dir/}"
153+
rel_path="${rel_path%.mdx}"
154+
rel_path="${rel_path%.md}"
155+
local locale_prefix
156+
[[ "$locale" == "zh-CN" ]] && locale_prefix="zh" || locale_prefix="en"
157+
doc_url="${BASE_URL}/${locale_prefix}/${rel_path}"
158+
fi
159+
echo "$doc_url"
160+
}
161+
162+
# --- Process a directory ---
163+
process_directory() {
164+
local dir=$1
165+
local locale=$2
166+
local temp_success temp_error temp_total
167+
168+
echo "Processing $dir/..."
169+
170+
temp_success=$(mktemp)
171+
temp_error=$(mktemp)
172+
temp_total=$(mktemp)
173+
echo "0" > "$temp_success"
174+
echo "0" > "$temp_error"
175+
echo "0" > "$temp_total"
176+
177+
temp_files=$(mktemp)
178+
find "$dir" -type f \( -name "*.md" -o -name "*.mdx" \) ! -name "index.md" ! -name "index.mdx" > "$temp_files"
179+
180+
while IFS= read -r file; do
181+
local title doc_url rel_path id json_payload
182+
title=$(extract_title "$file")
183+
doc_url=$(extract_url "$file" "$dir" "$locale")
184+
185+
rel_path="${file#$dir/}"
186+
rel_path="${rel_path%.mdx}"
187+
rel_path="${rel_path%.md}"
188+
id=$(echo -n "${dir}/${rel_path}" | openssl md5 | awk '{print $NF}')
189+
190+
if json_payload=$(jq -n \
191+
--arg id "$id" \
192+
--arg title "$title" \
193+
--rawfile content "$file" \
194+
--arg locale "$locale" \
195+
--arg url "$doc_url" \
196+
'{id: $id, title: $title, content: $content, locale: $locale, url: $url}' 2>/dev/null); then
197+
198+
if upload_document "$json_payload" "$title"; then
199+
echo $(($(cat "$temp_success") + 1)) > "$temp_success"
200+
else
201+
echo $(($(cat "$temp_error") + 1)) > "$temp_error"
202+
fi
203+
else
204+
echo "JSON error: $title ($file)" >&2
205+
echo $(($(cat "$temp_error") + 1)) > "$temp_error"
206+
fi
207+
208+
echo $(($(cat "$temp_total") + 1)) > "$temp_total"
209+
echo "Progress: $(cat "$temp_total") files"
210+
done < "$temp_files"
211+
rm -f "$temp_files"
212+
213+
local final_success final_error final_total
214+
final_success=$(cat "$temp_success")
215+
final_error=$(cat "$temp_error")
216+
final_total=$(cat "$temp_total")
217+
218+
total_success=$((total_success + final_success))
219+
total_errors=$((total_errors + final_error))
220+
total_files=$((total_files + final_total))
221+
222+
echo "Done $dir/: $final_success ok, $final_error failed"
223+
rm -f "$temp_success" "$temp_error" "$temp_total"
224+
}
225+
226+
# --- Main ---
227+
echo "Uploading docs to Meilisearch index: ${MEILI_INDEX:-<not set>}"
228+
echo "Base URL: $BASE_URL"
229+
echo "Working directory: $(pwd)"
230+
[[ "$DRY_RUN" == true ]] && echo "(dry-run mode - no uploads)"
231+
echo ""
232+
233+
if [[ -d "zh" ]]; then
234+
process_directory "zh" "zh-CN"
235+
else
236+
echo "Warning: zh/ directory not found"
237+
fi
238+
239+
if [[ -d "en" ]]; then
240+
process_directory "en" "en-US"
241+
else
242+
echo "Warning: en/ directory not found"
243+
fi
244+
245+
echo ""
246+
echo "=== Summary ==="
247+
echo "Total files: $total_files"
248+
echo "Uploaded: $total_success"
249+
echo "Failed: $total_errors"
250+
251+
if [[ $total_files -gt 0 ]]; then
252+
success_rate=$((total_success * 100 / total_files))
253+
echo "Success rate: ${success_rate}%"
254+
fi
255+
256+
if [[ $total_errors -gt 0 ]]; then
257+
echo "Some uploads failed. Check errors above."
258+
exit 1
259+
fi
260+
261+
echo "All done."

0 commit comments

Comments
 (0)