Skip to content

Commit bc42d51

Browse files
author
Daniel Chen
authored
Merge pull request buildpacks-community#1616 from buildpacks-community/blob-creds
Support authentication methods for blobs
2 parents ae5eb15 + 9d9bc14 commit bc42d51

23 files changed

Lines changed: 712 additions & 31 deletions

api/openapi-spec/swagger.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5721,6 +5721,9 @@
57215721
"x-kubernetes-patch-merge-key": "type",
57225722
"x-kubernetes-patch-strategy": "merge"
57235723
},
5724+
"latestAttestationImage": {
5725+
"type": "string"
5726+
},
57245727
"latestCacheImage": {
57255728
"type": "string"
57265729
},
@@ -6997,6 +7000,9 @@
69977000
"url"
69987001
],
69997002
"properties": {
7003+
"auth": {
7004+
"type": "string"
7005+
},
70007006
"stripComponents": {
70017007
"type": "integer",
70027008
"format": "int64"

cmd/build-init/main.go

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ var (
3636
gitRevision = flag.String("git-revision", os.Getenv("GIT_REVISION"), "The Git revision to make the repository HEAD.")
3737
gitInitializeSubmodules = flag.Bool("git-initialize-submodules", getenvBool("GIT_INITIALIZE_SUBMODULES"), "Initialize submodules during git clone")
3838
blobURL = flag.String("blob-url", os.Getenv("BLOB_URL"), "The url of the source code blob.")
39+
blobAuth = flag.Bool("blob-auth", getenvBool("BLOB_AUTH"), "If authentication should be used for blobs")
3940
stripComponents = flag.Int("strip-components", getenvInt("BLOB_STRIP_COMPONENTS", 0), "The number of directory components to strip from the blobs content when extracting.")
4041
registryImage = flag.String("registry-image", os.Getenv("REGISTRY_IMAGE"), "The registry location of the source code image.")
4142
hostName = flag.String("dns-probe-hostname", os.Getenv("DNS_PROBE_HOSTNAME"), "hostname to dns poll")
@@ -49,6 +50,7 @@ var (
4950

5051
basicGitCredentials flaghelpers.CredentialsFlags
5152
sshGitCredentials flaghelpers.CredentialsFlags
53+
blobCredentials flaghelpers.CredentialsFlags
5254
basicDockerCredentials flaghelpers.CredentialsFlags
5355
dockerCfgCredentials flaghelpers.CredentialsFlags
5456
dockerConfigCredentials flaghelpers.CredentialsFlags
@@ -60,6 +62,7 @@ var (
6062
func init() {
6163
flag.Var(&basicGitCredentials, "basic-git", "Basic authentication for git of the form 'secretname=git.domain.com'")
6264
flag.Var(&sshGitCredentials, "ssh-git", "SSH authentication for git of the form 'secretname=git.domain.com'")
65+
flag.Var(&blobCredentials, "blob", "Authentication for blob of the form 'secretname=git.domain.com'")
6366
flag.Var(&basicDockerCredentials, "basic-docker", "Basic authentication for docker of the form 'secretname=git.domain.com'")
6467
flag.Var(&dockerCfgCredentials, "dockercfg", "Docker Cfg credentials in the form of the path to the credential")
6568
flag.Var(&dockerConfigCredentials, "dockerconfig", "Docker Config JSON credentials in the form of the path to the credential")
@@ -220,8 +223,27 @@ func fetchSource(logger *log.Logger, keychain authn.Keychain) error {
220223
}
221224
return fetcher.Fetch(appDir, *gitURL, *gitRevision, projectMetadataDir)
222225
case *blobURL != "":
226+
var (
227+
blobKeychain blob.Keychain
228+
err error
229+
)
230+
if *blobAuth {
231+
if len(blobCredentials) == 0 {
232+
logger.Println("Loading blob credentials from helpers")
233+
blobKeychain = blob.DefaultKeychain
234+
} else {
235+
logger.Println("Loading blob credentials from service account secrets")
236+
logLoadingSecrets(logger, blobCredentials)
237+
blobKeychain, err = blob.NewMountedSecretBlobKeychain(buildSecretsDir, blobCredentials)
238+
if err != nil {
239+
return err
240+
}
241+
}
242+
}
243+
223244
fetcher := blob.Fetcher{
224-
Logger: logger,
245+
Logger: logger,
246+
Keychain: blobKeychain,
225247
}
226248
return fetcher.Fetch(appDir, *blobURL, *stripComponents, projectMetadataDir)
227249
case *registryImage != "":

cmd/completion/main.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ var (
4848
cosignDockerMediaTypes flaghelpers.CredentialsFlags
4949
basicGitCredentials flaghelpers.CredentialsFlags
5050
sshGitCredentials flaghelpers.CredentialsFlags
51+
blobCredentials flaghelpers.CredentialsFlags
5152
logger *log.Logger
5253
)
5354

@@ -60,6 +61,7 @@ func init() {
6061
flag.Var(&dockerConfigCredentials, "dockerconfig", "Docker Config JSON credentials in the form of the path to the credential")
6162
flag.Var(&basicGitCredentials, "basic-git", "Basic authentication for git of the form 'secretname=git.domain.com'")
6263
flag.Var(&sshGitCredentials, "ssh-git", "SSH authentication for git of the form 'secretname=git.domain.com'")
64+
flag.Var(&blobCredentials, "blob", "Authentication for blob of the form 'secretname=git.domain.com'")
6365

6466
flag.Var(&cosignAnnotations, "cosign-annotations", "Cosign custom signing annotations")
6567
flag.Var(&cosignRepositories, "cosign-repositories", "Cosign signing repository of the form 'secretname=registry.example.com/project'")

docs/build.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,10 @@ The `source` field is a composition of a source code location and a `subpath`. I
9595
- `git`: (Source Code is a git repository)
9696
- `url`: The git repository url. Both https and ssh formats are supported; with ssh format requiring a [ssh secret](secrets.md#git-secrets).
9797
- `revision`: The git revision to use. This value may be a commit sha, branch name, or tag.
98+
- `auth`: Optional auth to use with blob source. Leave empty for no auth, "secret" for providing auth [via Secret](secrets.md#blob-secrets), or "helper" to use service account IAM (specific to each IaaS).
99+
> Note: Only [Microsoft Azure](https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview)
100+
> and [Google Cloud Platform](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#kubernetes-sa-to-iam)
101+
> helpers are currently implemented, contributions are welcome to `pkg/blob/<iaas>_keychain.go`.
98102
- `subPath`: A subdirectory within the source folder where application code resides. Can be ignored if the source code resides at the `root` level.
99103

100104
* Blob

docs/image.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,11 +99,16 @@ The `source` field is a composition of a source code location and a `subpath`. I
9999
blob:
100100
url: ""
101101
stripComponents: 0
102+
auth: "" | "secret" | "helper"
102103
subPath: ""
103104
```
104105
- `blob`: (Source Code is a blob/jar in a blobstore)
105106
- `url`: The URL of the source code blob. This blob needs to either be publicly accessible or have the access token in the URL
106107
- `stripComponents`: Optional number of directory components to strip from the blobs content when extracting.
108+
- `auth`: Optional auth to use with blob source. Leave empty for no auth, "secret" for providing auth [via Secret](secrets.md#blob-secrets), or "helper" to use service account IAM (specific to each IaaS).
109+
> Note: Only [Microsoft Azure](https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview)
110+
> and [Google Cloud Platform](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#kubernetes-sa-to-iam)
111+
> helpers are currently implemented, contributions are welcome to `pkg/blob/<iaas>_keychain.go`.
107112
- `subPath`: A subdirectory within the source folder where application code resides. Can be ignored if the source code resides at the `root` level.
108113

109114
* Registry

docs/secrets.md

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,26 @@ stringData:
126126
password: <generated-token>
127127
```
128128

129+
### Blob Secrets
130+
131+
Secrets are used with a `kpack.io/blob` annotation that references a hostname for a blob location. Only one of username/password, bearer, or authorization is allowed.
132+
133+
```yaml
134+
apiVersion: v1
135+
kind: Secret
136+
metadata:
137+
name: blob-secret
138+
annotations:
139+
kpack.io/blob: my-blob-store.com
140+
stringData:
141+
username: <username>
142+
password: <password>
143+
144+
bearer: <oauth2 token>
145+
146+
authorization: <third-party-auth-header>
147+
```
148+
129149
### Service Account
130150

131151
To use these secrets with kpack create a service account and reference the service account in image and build resources. When configuring the image resource, reference the `name` of your registry credential and the `name` of your git credential.

go.mod

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ module github.com/pivotal/kpack
33
go 1.21
44

55
require (
6+
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0
7+
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0
8+
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0
69
github.com/BurntSushi/toml v1.3.2
710
github.com/Masterminds/semver/v3 v3.2.1
811
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a
@@ -30,6 +33,7 @@ require (
3033
go.uber.org/zap v1.26.0
3134
golang.org/x/crypto v0.19.0
3235
golang.org/x/net v0.21.0
36+
golang.org/x/oauth2 v0.15.0
3337
golang.org/x/sync v0.6.0
3438
k8s.io/api v0.29.0
3539
k8s.io/apimachinery v0.29.0
@@ -50,6 +54,7 @@ require (
5054
filippo.io/edwards25519 v1.0.0 // indirect
5155
github.com/AliyunContainerService/ack-ram-tool/pkg/credentials/alibabacloudsdkgo/helper v0.2.0 // indirect
5256
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
57+
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect
5358
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
5459
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
5560
github.com/Azure/go-autorest/autorest v0.11.29 // indirect
@@ -61,6 +66,7 @@ require (
6166
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
6267
github.com/Azure/go-autorest/logger v0.2.1 // indirect
6368
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
69+
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect
6470
github.com/Microsoft/go-winio v0.6.1 // indirect
6571
github.com/OneOfOne/xxhash v1.2.8 // indirect
6672
github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect
@@ -158,6 +164,7 @@ require (
158164
github.com/gobwas/glob v0.2.3 // indirect
159165
github.com/gogo/protobuf v1.3.2 // indirect
160166
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
167+
github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
161168
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
162169
github.com/golang/protobuf v1.5.3 // indirect
163170
github.com/golang/snappy v0.0.4 // indirect
@@ -188,6 +195,7 @@ require (
188195
github.com/kelseyhightower/envconfig v1.4.0 // indirect
189196
github.com/kevinburke/ssh_config v1.2.0 // indirect
190197
github.com/klauspost/compress v1.17.2 // indirect
198+
github.com/kylelemons/godebug v1.1.0 // indirect
191199
github.com/letsencrypt/boulder v0.0.0-20231026200631-000cd05d5491 // indirect
192200
github.com/magiconair/properties v1.8.7 // indirect
193201
github.com/mailru/easyjson v0.7.7 // indirect
@@ -216,6 +224,7 @@ require (
216224
github.com/pborman/uuid v1.2.1 // indirect
217225
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
218226
github.com/pjbgf/sha1cd v0.3.0 // indirect
227+
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
219228
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
220229
github.com/prometheus/client_golang v1.17.0 // indirect
221230
github.com/prometheus/client_model v0.5.0 // indirect
@@ -269,7 +278,6 @@ require (
269278
go.uber.org/multierr v1.11.0 // indirect
270279
golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 // indirect
271280
golang.org/x/mod v0.14.0 // indirect
272-
golang.org/x/oauth2 v0.15.0 // indirect
273281
golang.org/x/sys v0.17.0 // indirect
274282
golang.org/x/term v0.17.0 // indirect
275283
golang.org/x/text v0.14.0 // indirect

go.sum

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,8 @@ github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1 h1:MyVTgWR
7777
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.1/go.mod h1:GpPjLhVR9dnUoJMyHWSPy71xY9/lcmpzIPZXmF0FCVY=
7878
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 h1:D3occbWoio4EBLkbkevetNMAVX197GkzbUMtqjGWn80=
7979
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0/go.mod h1:bTSOgj05NGRuHHhQwAdPnYr9TOdNmKlZTgGLL6nyAdI=
80+
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY=
81+
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4=
8082
github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
8183
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
8284
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
@@ -387,6 +389,8 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/
387389
github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
388390
github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
389391
github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
392+
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
393+
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
390394
github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
391395
github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
392396
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
@@ -1548,6 +1552,7 @@ golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7w
15481552
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
15491553
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
15501554
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1555+
golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
15511556
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
15521557
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
15531558
golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

pkg/apis/build/v1alpha2/build_pod.go

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ const (
4343
cosignRespositoryAnnotationPrefix = "kpack.io/cosign.repository"
4444
DOCKERSecretAnnotationPrefix = "kpack.io/docker"
4545
GITSecretAnnotationPrefix = "kpack.io/git"
46+
BlobSecretAnnotationPrefix = "kpack.io/blob"
4647
IstioInject = "sidecar.istio.io/inject"
4748
BuildReadyAnnotation = "build.kpack.io/ready"
4849

@@ -234,7 +235,9 @@ func (b *Build) BuildPod(images BuildPodImages, buildContext BuildContext) (*cor
234235
buildEnv = append(buildEnv, envVar)
235236
}
236237

237-
secretVolumes, secretVolumeMounts, secretArgs := b.setupSecretVolumesAndArgs(buildContext.Secrets, gitAndDockerSecrets)
238+
blobAuthUseSecrets := b.Spec.Source.Blob != nil && b.Spec.Source.Blob.Auth == string(corev1alpha1.BlobAuthSecret)
239+
240+
secretVolumes, secretVolumeMounts, secretArgs := b.setupSecretVolumesAndArgs(buildContext.Secrets, buildSecrets(blobAuthUseSecrets))
238241
cosignVolumes, cosignVolumeMounts, cosignSecretArgs := b.setupCosignVolumes(buildContext.Secrets)
239242
imagePullVolumes, imagePullVolumeMounts, imagePullArgs := b.setupImagePullVolumes(buildContext.ImagePullSecrets)
240243

@@ -978,8 +981,18 @@ func (b *Build) cacheVolume(os string) []corev1.Volume {
978981
}}
979982
}
980983

981-
func gitAndDockerSecrets(secret corev1.Secret) bool {
982-
return secret.Annotations[GITSecretAnnotationPrefix] != "" || dockerSecrets(secret)
984+
func buildSecrets(includeBlobSecrets bool) func(corev1.Secret) bool {
985+
return func(secret corev1.Secret) bool {
986+
return gitSecrets(secret) || blobSecrets(includeBlobSecrets, secret) || dockerSecrets(secret)
987+
}
988+
}
989+
990+
func gitSecrets(secret corev1.Secret) bool {
991+
return secret.Annotations[GITSecretAnnotationPrefix] != ""
992+
}
993+
994+
func blobSecrets(includeBlobSecret bool, secret corev1.Secret) bool {
995+
return includeBlobSecret && secret.Annotations[BlobSecretAnnotationPrefix] != ""
983996
}
984997

985998
func dockerSecrets(secret corev1.Secret) bool {
@@ -1009,6 +1022,9 @@ func (b *Build) setupSecretVolumesAndArgs(secrets []corev1.Secret, filter func(s
10091022
case secret.Type == corev1.SecretTypeSSHAuth:
10101023
annotatedUrl := secret.Annotations[GITSecretAnnotationPrefix]
10111024
args = append(args, fmt.Sprintf("-ssh-%s=%s=%s", "git", secret.Name, annotatedUrl))
1025+
case secret.Annotations[BlobSecretAnnotationPrefix] != "":
1026+
annotatedUrl := secret.Annotations[BlobSecretAnnotationPrefix]
1027+
args = append(args, fmt.Sprintf("-blob=%s=%s", secret.Name, annotatedUrl))
10121028
default:
10131029
//ignoring secret
10141030
continue

0 commit comments

Comments
 (0)