Skip to content

Commit a1f3f86

Browse files
emilymyeemilymye
authored
Merge pull request #16585 from emilymye/release-2.36.0
2 parents d5d0bc3 + 41f775d commit a1f3f86

913 files changed

Lines changed: 21235 additions & 6639 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/java_tests.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,13 @@ on:
3838

3939

4040
jobs:
41+
setup-go:
42+
runs-on: [ubuntu-latest, macos-latest, windows-latest]
43+
steps:
44+
- uses: actions/checkout@v2
45+
- uses: actions/setup-go@v2
46+
with:
47+
go-version: '1.16.12'
4148

4249
check_gcp_variables:
4350
timeout-minutes: 5
@@ -108,6 +115,7 @@ jobs:
108115
path: runners/core-java/build/reports/tests/test
109116

110117
java_wordcount_direct_runner:
118+
needs: setup-go
111119
name: 'Java Wordcount Direct Runner'
112120
runs-on: ${{ matrix.os }}
113121
strategy:
@@ -137,6 +145,7 @@ jobs:
137145
name: 'Java Wordcount Dataflow'
138146
needs:
139147
- check_gcp_variables
148+
- setup-go
140149
runs-on: ${{ matrix.os }}
141150
if: |
142151
needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && (

‎.github/workflows/local_env_tests.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,7 @@ jobs:
3333
timeout-minutes: 25
3434
name: "Ubuntu run local environment shell script"
3535
runs-on: ubuntu-latest
36-
steps:steps:
37-
- uses: actions/checkout@v2
36+
steps:
3837
- uses: actions/checkout@v2
3938
- uses: actions/setup-go@v2
4039
with:

‎.github/workflows/playground_deploy_examples.yml‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: Collect And Deploy Playground Examples
17+
18+
on:
19+
push:
20+
tags: 'v*'
21+
workflow_dispatch:
22+
23+
jobs:
24+
deploy_examples:
25+
name: Deploy examples
26+
runs-on: ubuntu-latest
27+
env:
28+
BEAM_ROOT_DIR: ../../
29+
BEAM_EXAMPLE_CATEGORIES: ../categories.yaml
30+
BEAM_VERSION: 2.33.0
31+
SERVER_ADDRESS: backend-python-dot-datatokenization.uc.r.appspot.com
32+
K8S_NAMESPACE: playground-backend
33+
HELM_APP_NAME: playground-backend
34+
steps:
35+
- name: Check out the repo
36+
uses: actions/checkout@v2
37+
- uses: actions/setup-python@v2
38+
with:
39+
python-version: '3.8'
40+
- uses: actions/setup-java@v2
41+
with:
42+
distribution: 'zulu'
43+
java-version: '8'
44+
- name: Install kubectl
45+
run: |
46+
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" &&\
47+
chmod +x kubectl &&\
48+
mv kubectl /usr/local/bin/
49+
- name: Install helm
50+
run: |
51+
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 &&\
52+
chmod 700 get_helm.sh &&\
53+
./get_helm.sh
54+
- name: Set up Cloud SDK
55+
uses: google-github-actions/setup-gcloud@v0.3.0
56+
- name: install deps
57+
run: pip install -r requirements.txt
58+
working-directory: playground/infrastructure
59+
- name: Setup GCP account
60+
run: |
61+
echo "${{ secrets.GCP_ACCESS_KEY }}" | base64 -d > /tmp/gcp_access.json
62+
gcloud auth activate-service-account --project=datatokenization --key-file=/tmp/gcp_access.json
63+
- name: Get K8s Config
64+
run: gcloud container clusters get-credentials --region us-central1-a playground-examples
65+
- name: Login to Docker Registry
66+
run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.REGISTRY_NAME }}
67+
- name: Build And Push Java Backend
68+
run: ./gradlew playground:backend:containers:java:dockerTagPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' -Pdocker-tag="$GITHUB_SHA"
69+
- name: Build And Push Go Backend
70+
run: ./gradlew playground:backend:containers:go:dockerTagPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pdocker-tag="$GITHUB_SHA"
71+
- name: Build And Push Python Backend
72+
run: ./gradlew playground:backend:containers:python:dockerTagPush -Pdocker-repository-root='${{ secrets.REGISTRY_NAME}}/${{ secrets.PROJECT_ID }}/playground-repository' -Pdocker-tag="$GITHUB_SHA"
73+
- name: Install helm chart
74+
run: |
75+
kubectl create namespace $K8S_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - &&\
76+
helm install --namespace $K8S_NAMESPACE $HELM_APP_NAME . --set global.registry="${{ secrets.REGISTRY_NAME }}/${{ secrets.PROJECT_ID }}/playground-repository" --set global.tag="$GITHUB_SHA" &&\
77+
sleep 120
78+
working-directory: playground/infrastructure/helm
79+
- name: Run Python Examples CI
80+
run: |
81+
K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-python" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
82+
BEAM_ROOT_DIR="../../"
83+
BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
84+
export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8081" && python3 ci_cd.py --step CI --sdk SDK_PYTHON
85+
working-directory: playground/infrastructure
86+
- name: Run Python Examples CD
87+
run: |
88+
K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-python" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
89+
BEAM_ROOT_DIR="../../"
90+
BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
91+
export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8081" && python3 ci_cd.py --step CD --sdk SDK_PYTHON
92+
working-directory: playground/infrastructure
93+
env:
94+
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
95+
- name: Run Go Examples CI
96+
run: |
97+
K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-go" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
98+
BEAM_ROOT_DIR="../../"
99+
BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
100+
export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8082" && python3 ci_cd.py --step CI --sdk SDK_GO
101+
working-directory: playground/infrastructure
102+
- name: Run Go Examples CD
103+
run: |
104+
K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-go" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
105+
BEAM_ROOT_DIR="../../"
106+
BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
107+
export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8082" && python3 ci_cd.py --step CD --sdk SDK_GO
108+
working-directory: playground/infrastructure
109+
env:
110+
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
111+
- name: Run Java Examples CI
112+
run: |
113+
K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-java" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
114+
BEAM_ROOT_DIR="../../"
115+
BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
116+
export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8080" && python3 ci_cd.py --step CI --sdk SDK_JAVA
117+
working-directory: playground/infrastructure
118+
- name: Run Java Examples CD
119+
run: |
120+
K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-java" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
121+
BEAM_ROOT_DIR="../../"
122+
BEAM_EXAMPLE_CATEGORIES="../../playground/categories.yaml"
123+
export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8080" && python3 ci_cd.py --step CD --sdk SDK_JAVA
124+
working-directory: playground/infrastructure
125+
env:
126+
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
127+
- name: Delete Helm Chart
128+
run: |
129+
helm del --namespace $K8S_NAMESPACE $HELM_APP_NAME

‎.github/workflows/python_tests.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,14 @@ on:
3838

3939
jobs:
4040

41+
setup-go:
42+
runs-on: [ubuntu-latest, macos-latest, windows-latest]
43+
steps:
44+
- uses: actions/checkout@v2
45+
- uses: actions/setup-go@v2
46+
with:
47+
go-version: '1.16.12'
48+
4149
check_gcp_variables:
4250
timeout-minutes: 5
4351
name: "Check GCP variables"
@@ -168,6 +176,7 @@ jobs:
168176
name: 'Python Wordcount Dataflow'
169177
needs:
170178
- build_python_sdk_source
179+
- setup-go
171180
runs-on: ${{ matrix.os }}
172181
strategy:
173182
fail-fast: false

‎.test-infra/jenkins/job_Inventory.groovy‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,6 @@ nums.each {
6464
ALL_SUPPORTED_VERSIONS.each { version ->
6565
shell("python${version} --version || echo \"python${version} not found\"")
6666
}
67-
shell('/home/jenkins/tools/maven/latest/mvn -v || echo "mvn not found"')
68-
shell('/home/jenkins/tools/gradle4.3/gradle -v || echo "gradle not found"')
6967
shell('gcloud -v || echo "gcloud not found"')
7068
shell('kubectl version || echo "kubectl not found"')
7169
ALL_SUPPORTED_VERSIONS.each { version ->

‎.test-infra/jenkins/job_PostCommit_Java_Examples_Direct.groovy‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
import CommonJobProperties as commonJobProperties
20+
21+
// This job runs the Java examples tests with DirectRunner.
22+
job('beam_PostCommit_Java_Examples_Direct') {
23+
description('Run Java Examples on Direct Runner')
24+
25+
// Set common parameters.
26+
commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
27+
28+
// Allows triggering this build against pull requests.
29+
commonJobProperties.enablePhraseTriggeringFromPullRequest(
30+
delegate,
31+
'Java Direct Runner Examples',
32+
'Run Java Examples_Direct')
33+
34+
publishers {
35+
archiveJunit('**/build/test-results/**/*.xml')
36+
}
37+
38+
// Execute shell command to run examples.
39+
steps {
40+
gradle {
41+
rootBuildScriptDir(commonJobProperties.checkoutDir)
42+
tasks(':runners:direct:examplesIntegrationTest')
43+
commonJobProperties.setGradleSwitches(delegate)
44+
}
45+
}
46+
}

‎.test-infra/jenkins/job_PostCommit_Java_Examples_Flink.groovy‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
import CommonJobProperties as commonJobProperties
20+
21+
// This job runs the Java examples tests with FlinkRunner.
22+
job('beam_PostCommit_Java_Examples_Flink') {
23+
description('Run Java Examples on Flink Runner')
24+
25+
// Set common parameters.
26+
commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
27+
28+
// Allows triggering this build against pull requests.
29+
commonJobProperties.enablePhraseTriggeringFromPullRequest(
30+
delegate,
31+
'Java Flink Runner Examples',
32+
'Run Java Examples_Flink')
33+
34+
publishers {
35+
archiveJunit('**/build/test-results/**/*.xml')
36+
}
37+
38+
// Execute shell command to run examples.
39+
steps {
40+
gradle {
41+
rootBuildScriptDir(commonJobProperties.checkoutDir)
42+
tasks(":runners:flink:${CommonTestProperties.getFlinkVersion()}:examplesIntegrationTest")
43+
commonJobProperties.setGradleSwitches(delegate)
44+
}
45+
}
46+
}

‎.test-infra/jenkins/job_PostCommit_Java_Examples_Spark.groovy‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
import CommonJobProperties as commonJobProperties
20+
21+
// This job runs the Java examples tests with SparkRunner.
22+
job('beam_PostCommit_Java_Examples_Spark') {
23+
description('Run Java Examples on Spark Runner')
24+
25+
// Set common parameters.
26+
commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
27+
28+
// Allows triggering this build against pull requests.
29+
commonJobProperties.enablePhraseTriggeringFromPullRequest(
30+
delegate,
31+
'Java Spark Runner Examples',
32+
'Run Java Examples_Spark')
33+
34+
publishers {
35+
archiveJunit('**/build/test-results/**/*.xml')
36+
}
37+
38+
// Execute shell command to run examples.
39+
steps {
40+
gradle {
41+
rootBuildScriptDir(commonJobProperties.checkoutDir)
42+
tasks(':runners:spark:2:examplesIntegrationTest')
43+
tasks(':runners:spark:3:examplesIntegrationTest')
44+
commonJobProperties.setGradleSwitches(delegate)
45+
}
46+
}
47+
}

‎.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Spark_Batch.groovy‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Spark_Batch',
2525
description('Runs the Java PortableValidatesRunner suite on the Spark runner in batch mode.')
2626

2727
// Set common parameters.
28-
commonJobProperties.setTopLevelMainJobProperties(delegate)
28+
commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
2929

3030
// Publish all test results to Jenkins
3131
publishers {
@@ -38,6 +38,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Spark_Batch',
3838
rootBuildScriptDir(commonJobProperties.checkoutDir)
3939
tasks(':runners:spark:2:job-server:validatesPortableRunnerBatch')
4040
tasks(':runners:spark:3:job-server:validatesPortableRunnerBatch')
41+
tasks(':runners:spark:2:job-server:validatesPortableRunnerDocker')
42+
tasks(':runners:spark:3:job-server:validatesPortableRunnerDocker')
4143
commonJobProperties.setGradleSwitches(delegate)
4244
}
4345
}

‎.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Dataflo
2727

2828
description('Runs the ValidatesRunner suite on the Dataflow runner forcing streaming mode.')
2929

30-
commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 540)
30+
commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 720)
3131

3232
// Publish all test results to Jenkins
3333
publishers {

0 commit comments

Comments
 (0)