Skip to content

Commit 755fe9a

Browse files
authored
Enable Python Emitter in TypeSpec Playground (#10203)
Closes #10169 - Bundle and upload Python emitter, peer deps, and wheel file to Azure Storage Blob for playground consumption - update the Python emitter to create a browser-compatible Pyodide codepath which loads the python libraries from storage blob, and display it in the playground
1 parent 85592b0 commit 755fe9a

18 files changed

Lines changed: 776 additions & 335 deletions

File tree

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
changeKind: internal
3+
packages:
4+
- "@typespec/http-client-python"
5+
- "@typespec/bundle-uploader"
6+
---
7+
8+
Extend publish pipeline to upload emitter bundles to Playground storage account. Update Python emitter to be browser-compatible for use in the TypeSpec playground.

eng/emitters/pipelines/templates/stages/emitter-stages.yml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,11 @@ parameters:
8282
type: string
8383
default: "3.12"
8484

85+
# Whether to bundle and upload the emitter package to the playground package storage.
86+
- name: UploadPlaygroundBundle
87+
type: boolean
88+
default: false
89+
8590
stages:
8691
# Build stage
8792
# Responsible for building the autorest generator and typespec emitter packages
@@ -341,6 +346,29 @@ stages:
341346
ArtifactPath: $(buildArtifactsPath)
342347
LanguageShortName: ${{ parameters.LanguageShortName }}
343348

349+
- ${{ if parameters.UploadPlaygroundBundle }}:
350+
- script: npm ci
351+
displayName: Install emitter dependencies for playground bundle
352+
workingDirectory: $(Build.SourcesDirectory)/${{ parameters.PackagePath }}
353+
- script: npm run build
354+
displayName: Build emitter for playground bundle
355+
workingDirectory: $(Build.SourcesDirectory)/${{ parameters.PackagePath }}
356+
- script: npm install -g pnpm
357+
displayName: Install pnpm for playground bundle upload
358+
- script: pnpm install --filter "@typespec/bundle-uploader..."
359+
displayName: Install bundle-uploader dependencies
360+
workingDirectory: $(Build.SourcesDirectory)
361+
- script: pnpm --filter "@typespec/bundle-uploader..." build
362+
displayName: Build bundle-uploader
363+
workingDirectory: $(Build.SourcesDirectory)
364+
- task: AzureCLI@1
365+
displayName: Upload playground bundle
366+
inputs:
367+
azureSubscription: "Azure SDK Engineering System"
368+
scriptLocation: inlineScript
369+
inlineScript: node ./eng/emitters/scripts/upload-bundled-emitter.js ${{ parameters.PackagePath }}
370+
workingDirectory: $(Build.SourcesDirectory)
371+
344372
templateContext:
345373
outputs:
346374
- output: pipelineArtifact
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// @ts-check
2+
import { resolve } from "path";
3+
import { bundleAndUploadStandalonePackage } from "../../../packages/bundle-uploader/dist/src/index.js";
4+
import { repoRoot } from "../../common/scripts/helpers.js";
5+
6+
const packageRelativePath = process.argv[2];
7+
if (!packageRelativePath) {
8+
// eslint-disable-next-line no-console
9+
console.error("Usage: node upload-bundled-emitter.js <package-path>");
10+
// eslint-disable-next-line no-console
11+
console.error(" e.g. node upload-bundled-emitter.js packages/http-client-csharp");
12+
process.exit(1);
13+
}
14+
15+
// remove leading slash if exists, then resolve to absolute path
16+
const packagePath = resolve(repoRoot, packageRelativePath.replace(/^\//, ""));
17+
18+
await bundleAndUploadStandalonePackage({ packagePath });

packages/bundle-uploader/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
"@azure/storage-blob": "catalog:",
4242
"@pnpm/workspace.find-packages": "catalog:",
4343
"@typespec/bundler": "workspace:^",
44+
"globby": "catalog:",
4445
"json5": "catalog:",
4546
"picocolors": "catalog:",
4647
"semver": "catalog:"

packages/bundle-uploader/src/index.ts

Lines changed: 139 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
import { AzureCliCredential } from "@azure/identity";
22
import { findWorkspacePackagesNoCheck } from "@pnpm/workspace.find-packages";
33
import { createTypeSpecBundle } from "@typespec/bundler";
4-
import { resolve } from "path";
4+
import { readFile } from "fs/promises";
5+
import { globby } from "globby";
6+
import { relative, resolve } from "path";
57
import { join as joinUnix } from "path/posix";
68
import pc from "picocolors";
79
import { parse } from "semver";
@@ -16,6 +18,137 @@ function logSuccess(message: string) {
1618
logInfo(pc.green(`✔ ${message}`));
1719
}
1820

21+
export interface PlaygroundAssetConfig {
22+
/** Glob pattern relative to the package root (e.g. "generator/dist/pygen-*.whl"). */
23+
path: string;
24+
/** MIME content type for the blob upload. */
25+
contentType: string;
26+
}
27+
28+
export interface PlaygroundConfig {
29+
/** Static files to upload as binary blobs. Paths support simple glob patterns. */
30+
assets?: PlaygroundAssetConfig[];
31+
/** Peer dependencies that should be bundled and uploaded. */
32+
bundlePeerDependencies?: string[];
33+
}
34+
35+
export interface BundleAndUploadStandalonePackageOptions {
36+
/**
37+
* Absolute path to the package directory.
38+
*/
39+
packagePath: string;
40+
}
41+
42+
/**
43+
* Bundle and upload a standalone package that is not part of the pnpm workspace.
44+
* Uploads the bundle files and writes a `latest.json` under the package's blob directory
45+
* (e.g. `@typespec/http-client-csharp/latest.json`).
46+
*
47+
* If the package's `package.json` contains a `playgroundConfig` section, this function
48+
* will also upload static assets (resolved via glob patterns) and bundle peer dependencies.
49+
*/
50+
export async function bundleAndUploadStandalonePackage({
51+
packagePath,
52+
}: BundleAndUploadStandalonePackageOptions) {
53+
const pkgJsonPath = resolve(packagePath, "package.json");
54+
const pkgJson = JSON.parse(await readFile(pkgJsonPath, "utf-8"));
55+
const playgroundConfig: PlaygroundConfig | undefined = pkgJson.playgroundConfig;
56+
57+
const bundle = await createTypeSpecBundle(packagePath);
58+
const manifest = bundle.manifest;
59+
logInfo(`Bundling standalone package: ${manifest.name}@${manifest.version}`);
60+
61+
const uploader = new TypeSpecBundledPackageUploader(new AzureCliCredential());
62+
await uploader.createIfNotExists();
63+
64+
const result = await uploader.upload(bundle);
65+
if (result.status === "uploaded") {
66+
logSuccess(`Bundle for package ${manifest.name}@${manifest.version} uploaded.`);
67+
} else {
68+
logInfo(`Bundle for package ${manifest.name} already exists for version ${manifest.version}.`);
69+
}
70+
71+
const importMap: Record<string, string> = {};
72+
for (const [key, value] of Object.entries(result.imports)) {
73+
importMap[joinUnix(manifest.name, key)] = value;
74+
}
75+
76+
await uploadPlaygroundAssets(uploader, packagePath, manifest, importMap, playgroundConfig);
77+
78+
await uploader.updatePackageLatest(manifest.name, {
79+
version: manifest.version,
80+
imports: importMap,
81+
});
82+
logSuccess(`Updated ${manifest.name}/latest.json for version ${manifest.version}.`);
83+
}
84+
85+
/**
86+
* Upload playground assets and bundle peer dependencies based on the provided config.
87+
*/
88+
async function uploadPlaygroundAssets(
89+
uploader: TypeSpecBundledPackageUploader,
90+
packagePath: string,
91+
manifest: { name: string; version: string },
92+
importMap: Record<string, string>,
93+
config: PlaygroundConfig | undefined,
94+
) {
95+
if (!config) {
96+
return;
97+
}
98+
99+
// Upload static assets (e.g. .whl files)
100+
if (config.assets) {
101+
for (const asset of config.assets) {
102+
const matchedFiles = await globby(asset.path, { cwd: packagePath, absolute: true });
103+
if (matchedFiles.length === 0) {
104+
logInfo(pc.yellow(`⚠ No files matched asset pattern: ${asset.path}`));
105+
continue;
106+
}
107+
for (const filePath of matchedFiles) {
108+
const relativePath = relative(packagePath, filePath).replace(/\\/g, "/");
109+
const blobPath = joinUnix(manifest.name, manifest.version, relativePath);
110+
const content = await readFile(filePath);
111+
const assetResult = await uploader.uploadBinaryAsset(blobPath, content, asset.contentType);
112+
const importKey = joinUnix(manifest.name, relativePath);
113+
importMap[importKey] = assetResult.url;
114+
if (assetResult.status === "uploaded") {
115+
logSuccess(`Uploaded asset: ${relativePath}`);
116+
} else {
117+
logInfo(`Asset already exists: ${relativePath}`);
118+
}
119+
}
120+
}
121+
}
122+
123+
// Bundle and upload peer dependencies
124+
if (config.bundlePeerDependencies) {
125+
for (const depName of config.bundlePeerDependencies) {
126+
const depPath = resolve(packagePath, "node_modules", depName);
127+
try {
128+
const depBundle = await createTypeSpecBundle(depPath);
129+
const depResult = await uploader.upload(depBundle);
130+
if (depResult.status === "uploaded") {
131+
logSuccess(
132+
`Bundle for peer dep ${depBundle.manifest.name}@${depBundle.manifest.version} uploaded.`,
133+
);
134+
} else {
135+
logInfo(
136+
`Bundle for peer dep ${depBundle.manifest.name} already exists for version ${depBundle.manifest.version}.`,
137+
);
138+
}
139+
for (const [key, value] of Object.entries(depResult.imports)) {
140+
importMap[joinUnix(depBundle.manifest.name, key)] = value;
141+
}
142+
} catch (e: unknown) {
143+
throw new Error(
144+
`Failed to bundle peer dependency ${depName}: ${e instanceof Error ? e.message : e}`,
145+
{ cause: e },
146+
);
147+
}
148+
}
149+
}
150+
}
151+
19152
export interface BundleAndUploadPackagesOptions {
20153
repoRoot: string;
21154
/**
@@ -85,9 +218,12 @@ export async function bundleAndUploadPackages({
85218
}
86219
}
87220
logInfo(`Import map for ${indexVersion}:`, importMap);
88-
await uploader.updateIndex(indexName, {
221+
const index = {
89222
version: indexVersion,
90223
imports: importMap,
91-
});
224+
};
225+
await uploader.updateIndex(indexName, index);
92226
logSuccess(`Updated index for version ${indexVersion}.`);
227+
await uploader.updateLatestIndex(indexName, index);
228+
logSuccess(`Updated latest index for version ${indexVersion}.`);
93229
}

packages/bundle-uploader/src/upload-browser-package.ts

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,80 @@ export class TypeSpecBundledPackageUploader {
7575
});
7676
}
7777

78+
async uploadBinaryAsset(
79+
blobPath: string,
80+
content: Buffer,
81+
contentType: string,
82+
): Promise<{ status: "uploaded" | "already-exists"; url: string }> {
83+
const normalizedPath = normalizePath(blobPath);
84+
const blob = this.#container.getBlockBlobClient(normalizedPath);
85+
const url = `${this.#container.url}/${normalizedPath}`;
86+
try {
87+
await blob.uploadData(content, {
88+
blobHTTPHeaders: {
89+
blobContentType: contentType,
90+
},
91+
conditions: {
92+
ifNoneMatch: "*",
93+
},
94+
});
95+
return { status: "uploaded", url };
96+
} catch (e: any) {
97+
if (e.code === "BlobAlreadyExists") {
98+
return { status: "already-exists", url };
99+
}
100+
throw e;
101+
}
102+
}
103+
104+
async getLatestIndex(name: string): Promise<PackageIndex | undefined> {
105+
const blob = this.#container.getBlockBlobClient(`indexes/${name}/latest.json`);
106+
if (await blob.exists()) {
107+
const response = await blob.download();
108+
const body = await response.blobBody;
109+
const existingContent = await body?.text();
110+
if (existingContent) {
111+
return JSON.parse(existingContent);
112+
}
113+
}
114+
return undefined;
115+
}
116+
117+
async updateLatestIndex(name: string, index: PackageIndex) {
118+
const blob = this.#container.getBlockBlobClient(`indexes/${name}/latest.json`);
119+
const content = JSON.stringify(index);
120+
await blob.upload(content, content.length, {
121+
blobHTTPHeaders: {
122+
blobContentType: "application/json; charset=utf-8",
123+
},
124+
});
125+
}
126+
127+
/** Read the latest.json for a package from `{pkgName}/latest.json`. */
128+
async getPackageLatest(pkgName: string): Promise<PackageIndex | undefined> {
129+
const blob = this.#container.getBlockBlobClient(normalizePath(join(pkgName, "latest.json")));
130+
if (await blob.exists()) {
131+
const response = await blob.download();
132+
const body = await response.blobBody;
133+
const existingContent = await body?.text();
134+
if (existingContent) {
135+
return JSON.parse(existingContent);
136+
}
137+
}
138+
return undefined;
139+
}
140+
141+
/** Write the latest.json for a package at `{pkgName}/latest.json`. */
142+
async updatePackageLatest(pkgName: string, index: PackageIndex) {
143+
const blob = this.#container.getBlockBlobClient(normalizePath(join(pkgName, "latest.json")));
144+
const content = JSON.stringify(index);
145+
await blob.upload(content, content.length, {
146+
blobHTTPHeaders: {
147+
blobContentType: "application/json; charset=utf-8",
148+
},
149+
});
150+
}
151+
78152
async #uploadManifest(manifest: BundleManifest) {
79153
try {
80154
const blob = this.#container.getBlockBlobClient(

packages/http-client-python/.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,7 @@ generator/test/.diff-summary.md
1010

1111
# local folder for debug
1212
alpha
13+
14+
# pip component detection reports (generated during build)
15+
component-detection-pip-report.json
16+
generator/component-detection-pip-report.json
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
export const PYODIDE_VERSION = "0.26.2";
2+
export const PYGEN_WHEEL_FILENAME = "pygen-0.1.0-py3-none-any.whl";
3+
export const BLOB_STORAGE_BASE_URL = "https://typespec.blob.core.windows.net/pkgs";
4+
export const PACKAGE_NAME = "@typespec/http-client-python";
5+
6+
export const blackExcludeDirs = [
7+
"__pycache__/",
8+
"node_modules/",
9+
"venv/",
10+
"env/",
11+
".direnv",
12+
".eggs",
13+
".git",
14+
".hg",
15+
".tox",
16+
".venv",
17+
".eggs",
18+
".mypy_cache",
19+
".pytest_cache",
20+
".vscode",
21+
".*_build/",
22+
"/build/",
23+
"dist",
24+
".nox",
25+
".svn",
26+
"TempTypeSpecFiles/",
27+
];

0 commit comments

Comments
 (0)