From 53bbcfdf1d00b883aafef317fb530a0ef6b75439 Mon Sep 17 00:00:00 2001 From: Will Lo Date: Thu, 22 Jan 2026 13:33:40 -0800 Subject: [PATCH 1/5] feat: inline simulation --- package-lock.json | 18 +++++ package.json | 1 + .../inline-completion/simulation/utils.ts | 69 +++++++++++++++++++ .../src/shared/codeWhispererService.ts | 25 +++++++ 4 files changed, 113 insertions(+) create mode 100644 server/aws-lsp-codewhisperer/src/language-server/inline-completion/simulation/utils.ts diff --git a/package-lock.json b/package-lock.json index adc60bdf36..a37a66c959 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "@aws/language-server-runtimes": "^0.3.12", "@smithy/types": "4.2.0", "clean": "^4.0.2", + "fuzzball": "^2.2.3", "typescript": "^5.8.2" }, "devDependencies": { @@ -19513,6 +19514,17 @@ "node": ">=10" } }, + "node_modules/fuzzball": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/fuzzball/-/fuzzball-2.2.3.tgz", + "integrity": "sha512-sQDb3kjI7auA4YyE1YgEW85MTparcSgRgcCweUK06Cn0niY5lN+uhFiRUZKN4MQVGGiHxlbrYCA4nL1QjOXBLQ==", + "license": "MIT", + "dependencies": { + "heap": ">=0.2.0", + "lodash": "^4.17.21", + "setimmediate": "^1.0.5" + } + }, "node_modules/gauge": { "version": "2.7.4", "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", @@ -20199,6 +20211,12 @@ "he": "bin/he" } }, + "node_modules/heap": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz", + "integrity": "sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==", + "license": "MIT" + }, "node_modules/highlight.js": { "version": "11.11.1", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.11.1.tgz", diff --git a/package.json b/package.json index c046461a94..60af2352cf 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "@aws/language-server-runtimes": "^0.3.12", "@smithy/types": "4.2.0", "clean": "^4.0.2", + "fuzzball": "^2.2.3", "typescript": "^5.8.2" }, "devDependencies": { diff --git a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/simulation/utils.ts b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/simulation/utils.ts new file mode 100644 index 0000000000..37b11ccb24 --- /dev/null +++ b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/simulation/utils.ts @@ -0,0 +1,69 @@ +import * as Fuzz from 'fuzzball' + +interface DetailedResult { + suggestionIndex: number + suggestion: string + groundTruth: string + editSimilarity: number + exactMatch: boolean +} + +interface EditDistanceResult { + emRatio: number + editSimAvg: number + detailedResults: DetailedResult[] +} + +function calEditSim(target: string, truth: string): number { + let editSim = 0.0 + + const pred = truth.trim() + const gt = target.trim() + editSim += Fuzz.ratio(pred, gt) + + return editSim +} + +function computeEditDistances(suggestions: string[], truth: string): EditDistanceResult { + if (suggestions.length === 0) { + throw new Error('empty suggestion') + } + + if (!truth.length) { + throw new Error('empty ground truth') + } + + const detailedResults: DetailedResult[] = [] + let editSim = 0 + let exactMatchCnt = 0 + + for (let idx = 0; idx < suggestions.length; idx++) { + const suggestion = suggestions[idx] + + const es = calEditSim(suggestion, truth) + const em = suggestion === truth + if (em) { + exactMatchCnt++ + } + + editSim += es + + detailedResults.push({ + suggestionIndex: idx, + suggestion: suggestion, + groundTruth: truth, + editSimilarity: es, + exactMatch: em, + }) + } + + const totalSamples = suggestions.length + const editSimAvg = totalSamples > 0 ? Math.round((editSim / totalSamples) * 100) / 100 : -1 + const emRatio = exactMatchCnt / totalSamples + + return { + emRatio, + editSimAvg, + detailedResults, + } +} diff --git a/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts b/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts index ea663edf25..7e02125de6 100644 --- a/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts +++ b/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts @@ -32,6 +32,7 @@ import { RecentEditTracker } from '../language-server/inline-completion/tracker/ import { CodeWhispererSupplementalContext } from './models/model' import { fetchSupplementalContext } from './supplementalContextUtil/supplementalContextUtil' import * as path from 'path' +import * as fs from 'fs' import { CONTEXT_CHARACTERS_LIMIT, FILE_URI_CHARS_LIMIT, @@ -50,6 +51,8 @@ import { DeleteWorkspaceRequest, FeatureEvaluation, GenerateCompletionsCommand, + GenerateCompletionsCommandInput, + GenerateCompletionsCommandOutput, GenerateCompletionsRequest, GenerateCompletionsResponse, GetCodeAnalysisCommand, @@ -415,6 +418,8 @@ export class CodeWhispererServiceToken extends CodeWhispererServiceBase { #abTestingFetchingTimeout: NodeJS.Timeout | undefined #features: FeatureEvaluation[] | undefined + private isDevMode: boolean = true + constructor( private credentialsProvider: CredentialsProvider, workspace: Workspace, @@ -590,8 +595,11 @@ export class CodeWhispererServiceToken extends CodeWhispererServiceBase { // add cancellation check // add error check let logstr = `GenerateCompletion activity:\n` + let simulationRequest: GenerateCompletionsCommandInput = {} as any + let simulationResponse: GenerateCompletionsCommandOutput | Error = {} as any try { const tokenRequest = request as GenerateTokenSuggestionsRequest + simulationRequest = this.withProfileArn(tokenRequest) // Add customizationArn if available if (this.customizationArn) { @@ -628,6 +636,7 @@ export class CodeWhispererServiceToken extends CodeWhispererServiceBase { // "recentEdits": ${recentEditsLogStr}\n` const response = await this.client.send(new GenerateCompletionsCommand(this.withProfileArn(tokenRequest))) + simulationResponse = response const responseContext: ResponseContext = { requestId: response?.$metadata?.requestId ?? 'unknown', @@ -653,8 +662,24 @@ export class CodeWhispererServiceToken extends CodeWhispererServiceBase { return r } catch (e) { logstr += `error: ${(e as Error).message}` + simulationResponse = e as Error throw e } finally { + if (this.isDevMode) { + const simulationOutputEntry = { + request: simulationRequest, + Response: simulationResponse, + } + + try { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-') + const filename = path.join(`simulation-${timestamp}.json`) + + fs.appendFileSync(filename, JSON.stringify(simulationOutputEntry) + '\n') + } catch (e) { + this.logging.error(`simulation error : ${(e as Error).message}`) + } + } this.logging.info(logstr) } } From 19329281585442a6f8d362ba72f3aa9b17e04f0d Mon Sep 17 00:00:00 2001 From: Will Lo Date: Thu, 22 Jan 2026 13:53:20 -0800 Subject: [PATCH 2/5] chore: patch --- .../src/shared/codeWhispererService.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts b/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts index 7e02125de6..0c25719918 100644 --- a/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts +++ b/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts @@ -668,12 +668,15 @@ export class CodeWhispererServiceToken extends CodeWhispererServiceBase { if (this.isDevMode) { const simulationOutputEntry = { request: simulationRequest, - Response: simulationResponse, + response: simulationResponse, } try { const timestamp = new Date().toISOString().replace(/[:.]/g, '-') - const filename = path.join(`simulation-${timestamp}.json`) + const filename = path.join( + '/Users/xshaohua/workplace/ide/apex-sample-projects', + `simulation-lsp.json` + ) fs.appendFileSync(filename, JSON.stringify(simulationOutputEntry) + '\n') } catch (e) { From ec95cd7483dd3f724bd94f40c4b51dd7cd537773 Mon Sep 17 00:00:00 2001 From: Will Lo Date: Thu, 22 Jan 2026 13:54:57 -0800 Subject: [PATCH 3/5] chore: patch --- .../aws-lsp-codewhisperer/src/shared/codeWhispererService.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts b/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts index 0c25719918..3349bbac2d 100644 --- a/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts +++ b/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts @@ -418,6 +418,7 @@ export class CodeWhispererServiceToken extends CodeWhispererServiceBase { #abTestingFetchingTimeout: NodeJS.Timeout | undefined #features: FeatureEvaluation[] | undefined + // TODO: should read from IDE private isDevMode: boolean = true constructor( @@ -667,8 +668,8 @@ export class CodeWhispererServiceToken extends CodeWhispererServiceBase { } finally { if (this.isDevMode) { const simulationOutputEntry = { - request: simulationRequest, response: simulationResponse, + request: simulationRequest, } try { From 0107f3857ec42d77bb33927cea7bbf908c1dea9a Mon Sep 17 00:00:00 2001 From: Will Lo Date: Thu, 22 Jan 2026 13:57:16 -0800 Subject: [PATCH 4/5] chore: patch --- .../src/shared/codeWhispererService.ts | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts b/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts index 3349bbac2d..2b2d0e3e2e 100644 --- a/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts +++ b/server/aws-lsp-codewhisperer/src/shared/codeWhispererService.ts @@ -666,25 +666,29 @@ export class CodeWhispererServiceToken extends CodeWhispererServiceBase { simulationResponse = e as Error throw e } finally { - if (this.isDevMode) { - const simulationOutputEntry = { - response: simulationResponse, - request: simulationRequest, - } + this.writeSimulationOuputIfNeeded(simulationRequest, simulationResponse) + this.logging.info(logstr) + } + } - try { - const timestamp = new Date().toISOString().replace(/[:.]/g, '-') - const filename = path.join( - '/Users/xshaohua/workplace/ide/apex-sample-projects', - `simulation-lsp.json` - ) - - fs.appendFileSync(filename, JSON.stringify(simulationOutputEntry) + '\n') - } catch (e) { - this.logging.error(`simulation error : ${(e as Error).message}`) - } + private writeSimulationOuputIfNeeded( + request: GenerateCompletionsCommandInput, + response: GenerateCompletionsCommandOutput | Error + ) { + if (this.isDevMode) { + const simulationOutputEntry = { + response: response, + request: request, + } + + try { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-') + const filename = path.join('/Users/xshaohua/workplace/ide/apex-sample-projects', `simulation-lsp.json`) + + fs.appendFileSync(filename, JSON.stringify(simulationOutputEntry) + '\n') + } catch (e) { + this.logging.error(`simulation error : ${(e as Error).message}`) } - this.logging.info(logstr) } } From 726b21e0e98209dcada071a108ec7fdcf74465d5 Mon Sep 17 00:00:00 2001 From: Will Lo Date: Tue, 27 Jan 2026 18:21:24 -0800 Subject: [PATCH 5/5] chore: running evaluation --- app/aws-lsp-codewhisperer-runtimes/package.json | 3 ++- .../scripts/local-build.js | 6 +++--- package-lock.json | 5 ++++- package.json | 2 ++ .../handler/inlineCompletionHandler.ts | 13 ++++++++++++- .../supplementalContextUtil/crossFileContextUtil.ts | 3 ++- 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/app/aws-lsp-codewhisperer-runtimes/package.json b/app/aws-lsp-codewhisperer-runtimes/package.json index 80a06f1549..e133669a07 100644 --- a/app/aws-lsp-codewhisperer-runtimes/package.json +++ b/app/aws-lsp-codewhisperer-runtimes/package.json @@ -32,7 +32,6 @@ "https-browserify": "^1.0.0", "os-browserify": "^0.3.0", "path-browserify": "^1.0.1", - "process": "^0.11.10", "stream-browserify": "^3.0.0", "stream-http": "^3.2.0", "url": "^0.11.4", @@ -45,9 +44,11 @@ "@wdio/local-runner": "^9.12.1", "@wdio/mocha-framework": "^9.12.1", "@wdio/spec-reporter": "^9.11.0", + "buffer": "^6.0.3", "html-loader": "^5.1.0", "html-webpack-plugin": "^5.6.3", "node-loader": "^2.1.0", + "process": "^0.11.10", "ts-loader": "^9.4.4", "webpack": "^5.94.0", "webpack-cli": "^6.0.1" diff --git a/app/aws-lsp-codewhisperer-runtimes/scripts/local-build.js b/app/aws-lsp-codewhisperer-runtimes/scripts/local-build.js index 12f2d64584..ec0adea046 100644 --- a/app/aws-lsp-codewhisperer-runtimes/scripts/local-build.js +++ b/app/aws-lsp-codewhisperer-runtimes/scripts/local-build.js @@ -71,13 +71,13 @@ function createServerArtifact() { } console.log('\nStep 1: Running clean in root directory...') - executeCommand('npm run clean', ROOT_DIR) + // executeCommand('npm run clean', ROOT_DIR) console.log('\nStep 2: Installing dependencies in root directory...') - executeCommand('npm i', ROOT_DIR) + // executeCommand('npm i', ROOT_DIR) console.log('\nStep 3: Running compile in root directory...') - executeCommand('npm run compile', ROOT_DIR) + // executeCommand('npm run compile', ROOT_DIR) console.log('\nStep 4: Running package in target directory...') diff --git a/package-lock.json b/package-lock.json index a37a66c959..a24b47ed3f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "@types/node": "^22.9.0", "@typescript-eslint/eslint-plugin": "^8.32.1", "@typescript-eslint/parser": "^8.32.1", + "buffer": "^6.0.3", "c8": "^10.1.2", "conventional-changelog-conventionalcommits": "^8.0.0", "eslint": "^8.42.0", @@ -41,6 +42,7 @@ "oss-attribution-generator": "^1.7.1", "prettier": "^3.3.3", "pretty-quick": "^4.0.0", + "process": "^0.11.10", "shx": "^0.3.4", "ts-node": "^10.9.1" } @@ -97,7 +99,6 @@ "https-browserify": "^1.0.0", "os-browserify": "^0.3.0", "path-browserify": "^1.0.1", - "process": "^0.11.10", "stream-browserify": "^3.0.0", "stream-http": "^3.2.0", "url": "^0.11.4", @@ -110,9 +111,11 @@ "@wdio/local-runner": "^9.12.1", "@wdio/mocha-framework": "^9.12.1", "@wdio/spec-reporter": "^9.11.0", + "buffer": "^6.0.3", "html-loader": "^5.1.0", "html-webpack-plugin": "^5.6.3", "node-loader": "^2.1.0", + "process": "^0.11.10", "ts-loader": "^9.4.4", "webpack": "^5.94.0", "webpack-cli": "^6.0.1" diff --git a/package.json b/package.json index 60af2352cf..a589bf30d7 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "@types/node": "^22.9.0", "@typescript-eslint/eslint-plugin": "^8.32.1", "@typescript-eslint/parser": "^8.32.1", + "buffer": "^6.0.3", "c8": "^10.1.2", "conventional-changelog-conventionalcommits": "^8.0.0", "eslint": "^8.42.0", @@ -60,6 +61,7 @@ "oss-attribution-generator": "^1.7.1", "prettier": "^3.3.3", "pretty-quick": "^4.0.0", + "process": "^0.11.10", "shx": "^0.3.4", "ts-node": "^10.9.1" }, diff --git a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/handler/inlineCompletionHandler.ts b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/handler/inlineCompletionHandler.ts index a07e2ff742..be1ff124f6 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/inline-completion/handler/inlineCompletionHandler.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/inline-completion/handler/inlineCompletionHandler.ts @@ -485,13 +485,24 @@ export class InlineCompletionHandler { session, this.getTimeSinceLastUserModification() ) - return EMPTY_RESULT + return { + items: [], + sessionId: session.id, + metadata: { + request: session.requestContext, + response: suggestionResponse, + }, + } } return { items: suggestionsWithRightContext, sessionId: session.id, partialResultToken: suggestionResponse.responseContext.nextToken, + metadata: { + request: session.requestContext, + response: suggestionResponse, + }, } } diff --git a/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/crossFileContextUtil.ts b/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/crossFileContextUtil.ts index 5fa5261b76..73a8ec6103 100644 --- a/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/crossFileContextUtil.ts +++ b/server/aws-lsp-codewhisperer/src/shared/supplementalContextUtil/crossFileContextUtil.ts @@ -429,7 +429,8 @@ export async function getCrossFileCandidates( * for a resumed IDE session, opened tabs are restored but this getAllTextDocuments function returns empty * in that case we manually create TextDocuments from it */ - let unsortedCandidates: TextDocument[] = await workspace.getAllTextDocuments() + // TODO: experimental changes for vscode simulation implementation, should revert as vscode editors are closed via API, it will not fire `textDocument/didClose` + let unsortedCandidates: TextDocument[] = [] if (openTabFiles && openTabFiles.length > 0) { for (const openTabFile of openTabFiles) { try {