feat: support remote specs with authentication (#230)

- new cli param `--remote-spec-request-headers` /
  `OPENAPI_REMOTE_SPEC_REQUEST_HEADERS`

- accepts a JSON object like `{[uri: string]: {name: string, value:
string} | {name: string, value: string}[]}`
  - I don't _love_ using JSON as a format for this, however at least
    it's standardized and escaping values etc is well understood. By
    using JSON I'm confident that there won't be any edge cases with
    spaces or other separaters that would need handling with a format
    like `<uri>:<name>:<value>`

- main use-case is for generating from remote specs that are behind some
  format of request header based authentication (eg: private github
  repos, GCP IAP proxy, etc)
  - Previously I was having to `curl` these to the local filesystem
    first, which is inconvenient

closes #229

Author: mnahkies

.husky/pre-commit

@@ -1,4 +1 @@
-#!/usr/bin/env sh
-. "$(dirname -- "$0")/_/husky.sh"
-
 yarn lint-staged

packages/documentation/src/pages/reference/cli-options.mdx

@@ -115,6 +115,72 @@ Default: `false` (use `unknown` rather than `any`)
 
 ### Misc
 
+#### `--remote-spec-request-headers` (authenticated remote specifications)
+As environment variable `OPENAPI_REMOTE_SPEC_REQUEST_HEADERS`
+
+Allows providing request headers to use when fetching remote specifications. This allows for running
+generation against remote sources that require authentication.
+
+Common examples include [private github repositories](https://docs.github.com/en/rest/authentication/authenticating-to-the-rest-api),
+urls secured by an authenticating proxy like [GCP IAP Proxy](https://cloud.google.com/iap/docs/concepts-overview),
+or just generally authenticated servers
+
+
+<Callout type="warning" emoji="⚠️">
+  We strongly recommend using the **environment variable** variant of this option
+  (`OPENAPI_REMOTE_SPEC_REQUEST_HEADERS`), as values for this option will likely include secrets, and it
+  is best to keep these out of your shell history.
+</Callout>
+
+The format of the value is a JSON object keyed by URI, with values being either an object,
+or array of `{name, value}` pairs. As a typescript type:
+```typescript
+type value = {
+  [uri: string]: { name: string, value: string }[] | { name: string, value: string }
+}
+```
+
+For example:
+```json
+{
+  "https://example.com": [
+    {"name": "X-Client-Id", "value": "my-client-id"},
+    {"name": "X-Client-Secret", "value": "some-secret-value"}
+  ],
+  "https://example.org/some/path": {"name": "Proxy-Authorization", "value": "some token"}
+}
+```
+
+A full match on the provided uri is required for the headers to be sent.
+Eg: given a uri of "https://exmaple.com:8080/openapi.yaml" the headers would **not**
+be sent for requests to other ports, resource paths, or protocols, but a less specific
+uri like "https://example.com" will send headers on any (`https`) request to that domain.
+
+<Callout emoji="💡">
+  Why JSON you ask? Simply put it has well defined semantics, and is easy to parse without fear of jumbling the pieces together.
+
+  Unfortunately it is a little annoying to formulate in shell scripts, so here's some examples to get you started
+
+  Using [jq](https://jqlang.github.io/jq/):
+  ```shell
+  jq --null-input --compact-output \
+    --arg domain "https://example.com" \
+    --arg name "authorization" \
+    --arg value "secret value" '{$domain: {$name, $value}}'
+  ```
+
+  Using [nodejs](https://nodejs.org/):
+  ```shell
+  node -p 'JSON.stringify({[process.argv[1]]: {name: process.argv[2], value: process.argv[3]}})' \
+    https://example.com \
+    authorization \
+    'some secret value'
+  ```
+
+  Where typically in either example the values would be coming from shell variables, eg: storing a short-lived
+  access token, etc.
+</Callout>
+
 #### `-h, --help`
 Displays help text for command
 

packages/openapi-code-generator/package.json

@@ -51,7 +51,8 @@
     "json5": "^2.2.3",
     "lodash": "^4.17.21",
     "source-map-support": "^0.5.21",
-    "tslib": "^2.6.3"
+    "tslib": "^2.6.3",
+    "zod": "^3.23.8"
   },
   "peerDependencies": {
     "@typespec/compiler": "^0.58.0",

packages/openapi-code-generator/src/cli.spec.ts

@@ -0,0 +1,66 @@
+import {describe, it} from "@jest/globals"
+import {boolParser, remoteSpecRequestHeadersParser} from "./cli"
+
+describe("cli", () => {
+  describe("boolParser", () => {
+    it.each([
+      ["true", true],
+      ["1", true],
+      ["on", true],
+      ["TRUE", true],
+      ["1", true],
+      ["ON", true],
+      ["false", false],
+      ["0", false],
+      ["off", false],
+      ["FALSE", false],
+      ["0", false],
+      ["OFF", false],
+    ])("%s -> %s", (input, expected) => {
+      expect(boolParser(input)).toBe(expected)
+    })
+  })
+
+  describe("remoteSpecRequestHeadersParser", () => {
+    it("accepts a single header", () => {
+      expect(
+        remoteSpecRequestHeadersParser(
+          JSON.stringify({
+            "https://example.com/": {
+              name: "some-header-name",
+              value: "some-header-value",
+            },
+          }),
+        ),
+      ).toEqual({
+        "https://example.com/": [
+          {name: "some-header-name", value: "some-header-value"},
+        ],
+      })
+    })
+
+    it("accepts multiple headers", () => {
+      expect(
+        remoteSpecRequestHeadersParser(
+          JSON.stringify({
+            "https://example.com/": [
+              {
+                name: "some-header-name",
+                value: "some-header-value",
+              },
+              {
+                name: "some-other-header-name",
+                value: "some-other-header-value",
+              },
+            ],
+          }),
+        ),
+      ).toEqual({
+        "https://example.com/": [
+          {name: "some-header-name", value: "some-header-value"},
+          {name: "some-other-header-name", value: "some-other-header-value"},
+        ],
+      })
+    })
+  })
+})

packages/openapi-code-generator/src/cli.ts

@@ -8,6 +8,7 @@ import {
   InvalidArgumentError,
   Option,
 } from "@commander-js/extra-typings"
+import {z} from "zod"
 import {promptContinue} from "./core/cli-utils"
 import {NodeFsAdaptor} from "./core/file-system/node-fs-adaptor"
 import type {OperationGroupStrategy} from "./core/input"
@@ -19,7 +20,7 @@ import {generate} from "./index"
 import type {templates} from "./templates"
 import {TypescriptFormatterBiome} from "./typescript/common/typescript-formatter.biome"
 
-const boolParser = (arg: string): boolean => {
+export const boolParser = (arg: string): boolean => {
   const TRUTHY_VALUES = ["true", "1", "on"]
   const FALSY_VALUES = ["false", "0", "off", ""]
 
@@ -39,17 +40,38 @@ const boolParser = (arg: string): boolean => {
   )
 }
 
+export const remoteSpecRequestHeadersParser = (arg: string) => {
+  return z
+    .preprocess(
+      (str) =>
+        z
+          .string()
+          .transform((it) => JSON.parse(it))
+          .parse(str),
+      z.record(
+        z.string(),
+        z.preprocess(
+          (it) => (!it || Array.isArray(it) ? it : [it]),
+          z.array(
+            z.object({
+              name: z.string(),
+              value: z.string(),
+            }),
+          ),
+        ),
+      ),
+    )
+    .parse(arg)
+}
+
 const program = new Command()
   .addOption(
-    new Option("-i --input <value>", "input file to generate from")
+    new Option("-i --input <value>", "input specification to generate from")
       .env("OPENAPI_INPUT")
       .makeOptionMandatory(),
   )
   .addOption(
-    new Option(
-      "--input-type <value>",
-      "type of input file. this can be openapi3 or typespec",
-    )
+    new Option("--input-type <value>", "type of input specification")
       .env("OPENAPI_INPUT_TYPE")
       .choices(["openapi3", "typespec"] as const)
       .default("openapi3" as const)
@@ -74,7 +96,7 @@ const program = new Command()
   .addOption(
     new Option(
       "-s --schema-builder <value>",
-      "runtime schema parsing library to use",
+      "(typescript) runtime schema parsing library to use",
     )
       .env("OPENAPI_SCHEMA_BUILDER")
       .choices(["zod", "joi"] as const)
@@ -84,7 +106,7 @@ const program = new Command()
   .addOption(
     new Option(
       "--ts-allow-any [bool]",
-      "(typescript) whether to use `any` or `unknown` for unspecified types",
+      `(typescript) whether to use "any" or "unknown" for unspecified types`,
     )
       .env("OPENAPI_TS_ALLOW_ANY")
       .argParser(boolParser)
@@ -93,7 +115,7 @@ const program = new Command()
   .addOption(
     new Option(
       "--enable-runtime-response-validation [bool]",
-      "(experimental) whether to validate response bodies using the chosen runtime schema library",
+      "(experimental) (client sdks only) whether to validate response bodies using the chosen runtime schema library",
     )
       .env("OPENAPI_ENABLE_RUNTIME_RESPONSE_VALIDATION")
       .argParser(boolParser)
@@ -111,7 +133,7 @@ const program = new Command()
   .addOption(
     new Option(
       "--allow-unused-imports [bool]",
-      "Keep unused imports. Especially useful if there is a bug in the unused-import elimination.",
+      "Keep unused imports. Primarily useful if a bug occurs in the unused-import elimination.",
     )
       .env("OPENAPI_ALLOW_UNUSED_IMPORTS")
       .argParser(boolParser)
@@ -120,7 +142,10 @@ const program = new Command()
   .addOption(
     new Option(
       "--grouping-strategy <value>",
-      "(experimental) Strategy to use for splitting output into separate files. Set to none for a single generated.ts",
+      `
+    (experimental) Strategy to use for splitting output into separate files.
+
+    Set to none for a single generated.ts`,
     )
       .env("OPENAPI_GROUPING_STRATEGY")
       .choices([
@@ -131,12 +156,32 @@ const program = new Command()
       .default("none" as const)
       .makeOptionMandatory(),
   )
-  .showHelpAfterError()
-  .parse()
+  .addOption(
+    new Option(
+      "--remote-spec-request-headers <value>",
+      `
+    Request headers to use when fetching remote specifications.
+
+    Format is a JSON object keyed by domain name/uri, with values {name, value}[].
+    Eg: '{"https://example.com": [{"name": "Authorization", "value": "some arbitrary value"}]}'.
 
-const config = program.opts()
+    Use this if you're generating from a uri that requires authentication.
+
+    A full match on the provided domain/uri is required for the headers to be sent.
+    Eg: given a uri of "https://exmaple.com:8080/openapi.yaml" the headers would not
+    be sent for requests to other ports, resource paths, or protocols, but a less specific
+    uri like "https://example.com" will send headers on any request to that domain.
+
+    Using the environment variable variant is recommended to keep secrets out of your shell history`,
+    )
+      .env("OPENAPI_REMOTE_SPEC_REQUEST_HEADERS")
+      .argParser(remoteSpecRequestHeadersParser),
+  )
+  .showHelpAfterError()
 
 async function main() {
+  const config = program.parse().opts()
+
   const fsAdaptor = new NodeFsAdaptor()
   // TODO: make switchable with prettier / auto-detect from project?
   const formatter = await TypescriptFormatterBiome.createNodeFormatter()
@@ -173,15 +218,17 @@ async function main() {
   )
 }
 
-main()
-  .then(() => {
-    logger.info("generation complete!")
-    logger.info("elapsed", logger.toJSON())
+if (require.main === module) {
+  main()
+    .then(() => {
+      logger.info("generation complete!")
+      logger.info("elapsed", logger.toJSON())
 
-    process.exit(0)
-  })
-  .catch((err) => {
-    logger.error("unhandled error", err)
+      process.exit(0)
+    })
+    .catch((err) => {
+      logger.error("unhandled error", err)
 
-    process.exit(1)
-  })
+      process.exit(1)
+    })
+}