log tokens in convert

Author: pelikhan

packages/cli/src/convert.ts

@@ -1,8 +1,6 @@
 import {
-    CONVERTS_DIR_NAME,
     FILES_NOT_FOUND_ERROR_CODE,
     GENAI_ANY_REGEX,
-    GENAI_ANYTS_REGEX,
     HTTPS_REGEX,
     JSON5_REGEX,
     TRACE_FILENAME,
@@ -30,6 +28,8 @@ import { toSignal } from "../../core/src/cancellation"
 import { normalizeInt } from "../../core/src/cleaners"
 import { YAMLStringify } from "../../core/src/yaml"
 import { ensureDotGenaiscriptPath, getConvertDir } from "../../core/src/workdir"
+import { GenerationStats } from "../../core/src/usage"
+import { measure } from "../../core/src/performance"
 
 export async function convertFiles(
     scriptId: string,
@@ -134,6 +134,7 @@ export async function convertFiles(
         (filename) => ({ filename }) as WorkspaceFile
     )
 
+    const usage = new GenerationStats("")
     const results: Record<string, string> = {}
     const p = new PLimitPromiseQueue(normalizeInt(concurrency) || 1)
     await p.mapAll(files, async (file) => {
@@ -147,6 +148,7 @@ export async function convertFiles(
         const fileTrace = convertTrace.startTraceDetails(file.filename)
         convertTrace.item(link("trace", fileOutTrace))
         logVerbose(`trace: ${fileOutTrace}`)
+        const m = measure("convert")
         try {
             // apply AI transformation
             const result = await run(script.filename, file.filename, {
@@ -228,6 +230,8 @@ export async function convertFiles(
             }
 
             results[file.filename] = text
+            const end = m()
+            usage.addUsage(result.stats, end)
         } catch (error) {
             logError(error)
             fileTrace.error(undefined, error)
@@ -237,5 +241,6 @@ export async function convertFiles(
         }
     })
 
+    usage.log()
     logVerbose(`trace: ${outTraceFilename}`)
 }

packages/core/src/chat.ts

@@ -787,7 +787,7 @@ async function processChatMessage(
         choices,
     } = options
 
-    stats.addUsage(req, resp)
+    stats.addRequestUsage(req, resp)
     const assisantMessage = parseAssistantMessage(resp)
     if (assisantMessage) {
         messages.push(assisantMessage)

packages/core/src/promptrunner.ts

@@ -118,7 +118,7 @@ async function resolveExpansionVars(
 // Main function to run a template with given options
 /**
  * Executes a prompt template with specified options.
- * 
+ *
  * @param prj The project context providing runtime and configuration.
  * @param template The prompt script template to execute.
  * @param fragment Additional context such as files and metadata.

packages/core/src/usage.ts

@@ -301,24 +301,9 @@ export class GenerationStats {
             logVerbose(`missing pricing for ${[...unknowns].join(", ")}`)
     }
 
-    /**
-     * Adds usage statistics to the current instance.
-     *
-     * @param req - The request containing details about the chat completion.
-     * @param usage - The usage statistics to be added.
-     */
-    addUsage(req: CreateChatCompletionRequest, resp: ChatCompletionResponse) {
-        const {
-            usage = { completion_tokens: 0, prompt_tokens: 0, total_tokens: 0 },
-            model,
-            cached,
-            duration,
-        } = resp
-        const { messages } = req
-
-        if (!cached) {
-            this.usage.duration += duration ?? 0
-
+    addUsage(usage: ChatCompletionUsage, duration?: number) {
+        this.usage.duration += duration ?? 0
+        if (usage) {
             this.usage.completion_tokens += usage.completion_tokens ?? 0
             this.usage.prompt_tokens += usage.prompt_tokens ?? 0
             this.usage.total_tokens += usage.total_tokens ?? 0
@@ -336,6 +321,29 @@ export class GenerationStats {
             this.usage.completion_tokens_details.rejected_prediction_tokens +=
                 usage.completion_tokens_details?.rejected_prediction_tokens ?? 0
         }
+    }
+
+    /**
+     * Adds usage statistics to the current instance.
+     *
+     * @param req - The request containing details about the chat completion.
+     * @param usage - The usage statistics to be added.
+     */
+    addRequestUsage(
+        req: CreateChatCompletionRequest,
+        resp: ChatCompletionResponse
+    ) {
+        const {
+            usage = { completion_tokens: 0, prompt_tokens: 0, total_tokens: 0 },
+            model,
+            cached,
+            duration,
+        } = resp
+        const { messages } = req
+
+        if (!cached) {
+            this.addUsage(usage, duration)
+        }
 
         const { provider } = parseModelIdentifier(this.model)
         const chatTurn = {

packages/sample/genaisrc/ast-docs-update.genai.mts

@@ -14,6 +14,18 @@ const { output } = env
 const { applyEdits } = env.vars
 const file = env.files[0]
 
+const prettier = async (file) => {
+    // format
+    const res = await host.exec("prettier", [
+        "--write",
+        file.filename,
+    ])
+    if (!res.exitCode) throw new Error(res.stderr)
+}
+
+// normalize spacing
+await prettier(file)
+
 // find all exported functions with comments
 const sg = await host.astGrep()
 const { matches, replace, commitEdits } = await sg.search("ts", file.filename, {
@@ -79,6 +91,8 @@ for (const match of matches) {
 const modified = await commitEdits()
 if (applyEdits) {
     await workspace.writeFiles(modified)
+    // normalize spacing
+    await prettier(file)
 } else if (modified.length) {
     output.diff(file, modified[0])
     output.warn(