feat: inline simulation

app/aws-lsp-codewhisperer-runtimes/package.json

@@ -32,7 +32,6 @@
         "https-browserify": "^1.0.0",
         "os-browserify": "^0.3.0",
         "path-browserify": "^1.0.1",
-        "process": "^0.11.10",
         "stream-browserify": "^3.0.0",
         "stream-http": "^3.2.0",
         "url": "^0.11.4",
@@ -45,9 +44,11 @@
         "@wdio/local-runner": "^9.12.1",
         "@wdio/mocha-framework": "^9.12.1",
         "@wdio/spec-reporter": "^9.11.0",
+        "buffer": "^6.0.3",
         "html-loader": "^5.1.0",
         "html-webpack-plugin": "^5.6.3",
         "node-loader": "^2.1.0",
+        "process": "^0.11.10",
         "ts-loader": "^9.4.4",
         "webpack": "^5.94.0",
         "webpack-cli": "^6.0.1"

app/aws-lsp-codewhisperer-runtimes/scripts/local-build.js

@@ -71,13 +71,13 @@ function createServerArtifact() {
         }
 
         console.log('\nStep 1: Running clean in root directory...')
-        executeCommand('npm run clean', ROOT_DIR)
+        // executeCommand('npm run clean', ROOT_DIR)
 
         console.log('\nStep 2: Installing dependencies in root directory...')
-        executeCommand('npm i', ROOT_DIR)
+        // executeCommand('npm i', ROOT_DIR)
 
         console.log('\nStep 3: Running compile in root directory...')
-        executeCommand('npm run compile', ROOT_DIR)
+        // executeCommand('npm run compile', ROOT_DIR)
 
         console.log('\nStep 4: Running package in target directory...')
 

package.json

@@ -39,6 +39,7 @@
         "@aws/language-server-runtimes": "^0.3.12",
         "@smithy/types": "4.2.0",
         "clean": "^4.0.2",
+        "fuzzball": "^2.2.3",
         "typescript": "^5.8.2"
     },
     "devDependencies": {
@@ -48,6 +49,7 @@
         "@types/node": "^22.9.0",
         "@typescript-eslint/eslint-plugin": "^8.32.1",
         "@typescript-eslint/parser": "^8.32.1",
+        "buffer": "^6.0.3",
         "c8": "^10.1.2",
         "conventional-changelog-conventionalcommits": "^8.0.0",
         "eslint": "^8.42.0",
@@ -59,6 +61,7 @@
         "oss-attribution-generator": "^1.7.1",
         "prettier": "^3.3.3",
         "pretty-quick": "^4.0.0",
+        "process": "^0.11.10",
         "shx": "^0.3.4",
         "ts-node": "^10.9.1"
     },

server/aws-lsp-codewhisperer/src/language-server/inline-completion/handler/inlineCompletionHandler.ts

@@ -485,13 +485,24 @@
                 session,
                 this.getTimeSinceLastUserModification()
             )
-            return EMPTY_RESULT
+            return {
+                items: [],
+                sessionId: session.id,
+                metadata: {
+                    request: session.requestContext,
+                    response: suggestionResponse,
+                },
+            }
         }
 
         return {
             items: suggestionsWithRightContext,
             sessionId: session.id,
             partialResultToken: suggestionResponse.responseContext.nextToken,
+            metadata: {
+                request: session.requestContext,
+                response: suggestionResponse,
+            },
         }
     }
 

server/aws-lsp-codewhisperer/src/language-server/inline-completion/simulation/utils.ts

@@ -0,0 +1,69 @@
+import * as Fuzz from 'fuzzball'
+
+interface DetailedResult {
+    suggestionIndex: number
+    suggestion: string
+    groundTruth: string
+    editSimilarity: number
+    exactMatch: boolean
+}
+
+interface EditDistanceResult {
+    emRatio: number
+    editSimAvg: number
+    detailedResults: DetailedResult[]
+}
+
+function calEditSim(target: string, truth: string): number {
+    let editSim = 0.0
+
+    const pred = truth.trim()
+    const gt = target.trim()
+    editSim += Fuzz.ratio(pred, gt)
+
+    return editSim
+}
+
+function computeEditDistances(suggestions: string[], truth: string): EditDistanceResult {
+    if (suggestions.length === 0) {
+        throw new Error('empty suggestion')
+    }
+
+    if (!truth.length) {
+        throw new Error('empty ground truth')
+    }
+
+    const detailedResults: DetailedResult[] = []
+    let editSim = 0
+    let exactMatchCnt = 0
+
+    for (let idx = 0; idx < suggestions.length; idx++) {
+        const suggestion = suggestions[idx]
+
+        const es = calEditSim(suggestion, truth)
+        const em = suggestion === truth
+        if (em) {
+            exactMatchCnt++
+        }
+
+        editSim += es
+
+        detailedResults.push({
+            suggestionIndex: idx,
+            suggestion: suggestion,
+            groundTruth: truth,
+            editSimilarity: es,
+            exactMatch: em,
+        })
+    }
+
+    const totalSamples = suggestions.length
+    const editSimAvg = totalSamples > 0 ? Math.round((editSim / totalSamples) * 100) / 100 : -1
+    const emRatio = exactMatchCnt / totalSamples
+
+    return {
+        emRatio,
+        editSimAvg,
+        detailedResults,
+    }
+}

server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts

@@ -32,6 +32,7 @@ import { RecentEditTracker } from '../language-server/inline-completion/tracker/
 import { CodeWhispererSupplementalContext } from './models/model'
 import { fetchSupplementalContext } from './supplementalContextUtil/supplementalContextUtil'
 import * as path from 'path'
+import * as fs from 'fs'
 import {
     CONTEXT_CHARACTERS_LIMIT,
     FILE_URI_CHARS_LIMIT,
@@ -50,6 +51,8 @@ import {
     DeleteWorkspaceRequest,
     FeatureEvaluation,
     GenerateCompletionsCommand,
+    GenerateCompletionsCommandInput,
+    GenerateCompletionsCommandOutput,
     GenerateCompletionsRequest,
     GenerateCompletionsResponse,
     GetCodeAnalysisCommand,
@@ -415,6 +418,9 @@ export class CodeWhispererServiceToken extends CodeWhispererServiceBase {
     #abTestingFetchingTimeout: NodeJS.Timeout | undefined
     #features: FeatureEvaluation[] | undefined
 
+    // TODO: should read from IDE
+    private isDevMode: boolean = true
+
     constructor(
         private credentialsProvider: CredentialsProvider,
         workspace: Workspace,
@@ -590,8 +596,11 @@ export class CodeWhispererServiceToken extends CodeWhispererServiceBase {
         // add cancellation check
         // add error check
         let logstr = `GenerateCompletion activity:\n`
+        let simulationRequest: GenerateCompletionsCommandInput = {} as any
+        let simulationResponse: GenerateCompletionsCommandOutput | Error = {} as any
         try {
             const tokenRequest = request as GenerateTokenSuggestionsRequest
+            simulationRequest = this.withProfileArn(tokenRequest)
 
             // Add customizationArn if available
             if (this.customizationArn) {
@@ -628,6 +637,7 @@ export class CodeWhispererServiceToken extends CodeWhispererServiceBase {
             // "recentEdits": ${recentEditsLogStr}\n`
 
             const response = await this.client.send(new GenerateCompletionsCommand(this.withProfileArn(tokenRequest)))
+            simulationResponse = response
 
             const responseContext: ResponseContext = {
                 requestId: response?.$metadata?.requestId ?? 'unknown',
@@ -653,12 +663,35 @@ export class CodeWhispererServiceToken extends CodeWhispererServiceBase {
             return r
         } catch (e) {
             logstr += `error: ${(e as Error).message}`
+            simulationResponse = e as Error
             throw e
         } finally {
+            this.writeSimulationOuputIfNeeded(simulationRequest, simulationResponse)
             this.logging.info(logstr)
         }
     }
 
+    private writeSimulationOuputIfNeeded(
+        request: GenerateCompletionsCommandInput,
+        response: GenerateCompletionsCommandOutput | Error
+    ) {
+        if (this.isDevMode) {
+            const simulationOutputEntry = {
+                response: response,
+                request: request,
+            }
+
+            try {
+                const timestamp = new Date().toISOString().replace(/[:.]/g, '-')
+                const filename = path.join('/Users/xshaohua/workplace/ide/apex-sample-projects', `simulation-lsp.json`)
+
+                fs.appendFileSync(filename, JSON.stringify(simulationOutputEntry) + '\n')
+            } catch (e) {
+                this.logging.error(`simulation error : ${(e as Error).message}`)
+            }
+        }
+    }
+
     private mapCodeWhispererApiResponseToSuggestion(
         apiResponse: GenerateCompletionsResponse,
         responseContext: ResponseContext

server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/crossFileContextUtil.ts

@@ -429,7 +429,8 @@ export async function getCrossFileCandidates(
      * for a resumed IDE session, opened tabs are restored but this getAllTextDocuments function returns empty
      * in that case we manually create TextDocuments from it
      */
-    let unsortedCandidates: TextDocument[] = await workspace.getAllTextDocuments()
+    // TODO: experimental changes for vscode simulation implementation, should revert as vscode editors are closed via API, it will not fire `textDocument/didClose`
+    let unsortedCandidates: TextDocument[] = []
     if (openTabFiles && openTabFiles.length > 0) {
         for (const openTabFile of openTabFiles) {
             try {