Docs update (#1375)

* allow nodes outside match

* ✨ Improve fetch docstrings and AST docs update logic
Updated fetch function documentation for clarity and added precision to AST docs update with adjusted model and parsing logic.

Author: pelikhan

packages/core/src/astgrep.ts

@@ -81,8 +81,6 @@ export async function astGrepFindFiles(
 
     const pending: Record<string, { root: SgRoot; edits: SgEdit[] }> = {}
     const replace = (node: SgNode, text: string) => {
-        if (!matches.includes(node))
-            throw new Error("node is not included in the matches")
         const edit = node.replace(text)
         const root = node.getRoot()
         const rootEdits =

packages/core/src/fetch.ts

@@ -27,12 +27,12 @@ export type FetchType = (
 /**
  * Creates a fetch function with retry logic.
  *
- * This function wraps the `crossFetch` with retry capabilities based
- * on provided options. It allows configuring the number of retries,
- * delay between retries, and specific HTTP status codes to retry on.
+ * Wraps `crossFetch` with retry capabilities based on provided options. 
+ * Configures the number of retries, delay between retries, and specific HTTP status codes to retry on. 
+ * Supports cancellation and proxy configuration.
  *
- * @param options - Options for retry configuration and tracing.
- * @returns A fetch function with retry capabilities.
+ * @param options - Configuration for retries, delays, status codes, cancellation, and tracing.
+ * @returns A fetch function with retry functionality.
  */
 export async function createFetch(
     options?: {
@@ -118,12 +118,12 @@ export async function fetch(
 /**
  * Fetches text content from a URL or file.
  *
- * This function attempts to fetch content from either a URL or a local file.
- * It supports HTTP(S) URLs and reads directly from the file system for local files.
+ * Attempts to fetch content from an HTTP(S) URL or read from the file system for local files.
+ * Handles retries, delays, and tracing if configured. Supports binary content via base64 encoding.
  *
- * @param urlOrFile - The URL or file to fetch from.
- * @param fetchOptions - Optional fetch configuration.
- * @returns An object containing fetch status and content.
+ * @param urlOrFile - URL or file path to fetch from.
+ * @param fetchOptions - Optional configuration for retries, delays, and other fetch settings.
+ * @returns Object containing fetch status, content, and metadata.
  */
 export async function fetchText(
     urlOrFile: string | WorkspaceFile,
@@ -195,16 +195,17 @@ export async function fetchText(
 }
 
 /**
- * Logs a POST request for tracing.
+ * Logs a POST request for tracing purposes.
  *
- * Constructs a curl command representing the POST request with appropriate headers
- * and body, optionally masking sensitive information like authorization headers.
+ * Constructs a curl command to represent the POST request, including headers 
+ * and body. Sensitive information in authorization headers can be masked 
+ * based on the provided options.
  *
- * @param trace - Markdown trace object for logging.
- * @param url - The URL of the request.
- * @param headers - The request headers.
- * @param body - The request body.
- * @param options - Options for displaying authorization header.
+ * @param trace - Object used for logging trace details.
+ * @param url - The target URL of the request.
+ * @param headers - The headers to include in the request.
+ * @param body - The request body, either as FormData or a raw object.
+ * @param options - Configuration options, including whether to mask authorization headers.
  */
 export function traceFetchPost(
     trace: MarkdownTrace,
@@ -245,13 +246,13 @@ ${Object.entries(headers)
 }
 
 /**
- * Converts a response status to a message.
+ * Converts the HTTP response status and status text into a string list.
  *
- * Converts the HTTP response status and status text into a string list
- * to facilitate logging and debugging.
+ * Facilitates logging and debugging by converting the status information 
+ * from the response object.
  *
- * @param res - The response object.
- * @returns A list of status and status text.
+ * @param res - The HTTP response object containing status and statusText.
+ * @returns A string list with status and status text.
  */
 export function statusToMessage(res?: {
     status?: number

packages/sample/genaisrc/ast-docs-update.genai.mts

@@ -0,0 +1,92 @@
+script({
+    title: "Generate TypeScript function documentation using AST insertion",
+    accept: ".ts",
+    files: "src/cowsay.ts",
+    parameters: {
+        applyEdits: {
+            type: "boolean",
+            default: false,
+            description: "If true, the script will not modify the files.",
+        },
+    },
+})
+const { output } = env
+const { applyEdits } = env.vars
+const file = env.files[0]
+
+// find all exported functions with comments
+const sg = await host.astGrep()
+const { matches, replace, commitEdits } = await sg.search("ts", file.filename, {
+    rule: {
+        kind: "export_statement",
+        follows: {
+            kind: "comment",
+            stopBy: "neighbor",
+        },
+        has: {
+            kind: "function_declaration",
+        },
+    },
+})
+
+// for each match, generate a docstring for functions not documented
+for (const match of matches) {
+    const comment = match.prev()
+
+    const res = await runPrompt(
+        (_) => {
+            _.def("FILE", match.getRoot().root().text(), { flex: 1 })
+            _.def("DOCSTRING", comment.text(), { flex: 10 })
+            _.def("FUNCTION", match.text(), { flex: 10 })
+            // this needs more eval-ing
+            _.$`Update the docstring <DOCSTRING> of function <FUNCTION>.
+            - If the docstring is up to date, return /NOP/.
+            - Make sure parameters are documented.
+            - Be concise. Use technical tone.
+            - do NOT include types, this is for TypeScript.
+            - Use docstring syntax. do not wrap in markdown code section.
+            - Minimize updates to the existing docstring.
+            
+            The full source of the file is in <FILE> for reference.
+            The source of the function is in <FUNCTION>.
+            The current docstring is <DOCSTRING>.
+            `
+        },
+        {
+            model: "large",
+            responseType: "text",
+            flexTokens: 12000,
+            label: match.child(0).text(),
+        }
+    )
+    // if generation is successful, insert the docs
+    if (res.error) {
+        output.warn(res.error.message)
+        continue
+    }
+
+    if (res.text.includes("/NOP/")) continue
+
+    const docs = docify(
+        parsers.unfence(res.text.trim(), ["", "typescript", "ts"])
+    )
+    replace(comment, docs)
+}
+
+// apply all edits and write to the file
+const modified = await commitEdits()
+if (applyEdits) {
+    await workspace.writeFiles(modified)
+} else if (modified.length) {
+    output.diff(file, modified[0])
+    output.warn(
+        `edit not applied, use --vars 'applyEdits=true' to apply the edits`
+    )
+}
+
+// normalizes the docstring in case the LLM decides not to generate proper comments
+function docify(docs: string) {
+    if (!/^\/\*\*.*.*\*\/$/s.test(docs))
+        docs = `/**\n* ${docs.split(/\r?\n/g).join("\n* ")}\n*/`
+    return docs.replace(/\n+$/, "")
+}