Skip to content

Commit 51abf3f

Browse files
serkan-ozalsvrnm
serkan-ozal
and
svrnm
authored
feat(collector): introduce new workflow to publish collector Lambda layer (#1692)
* get architecture from input * get aws region from input * release layer * Add README * Rename workflow * Update collector/README.md Co-authored-by: Severin Neumann <[email protected]> * Update collector/README.md Co-authored-by: Severin Neumann <[email protected]> * Update collector/README.md Co-authored-by: Severin Neumann <[email protected]> * Update collector/README.md Co-authored-by: Severin Neumann <[email protected]> --------- Co-authored-by: Severin Neumann <[email protected]>
1 parent 6e8e746 commit 51abf3f

File tree

5 files changed

+240
-35
lines changed

5 files changed

+240
-35
lines changed

‎.github/workflows/build-collector.yml

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎.github/workflows/layer-publish.yml

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,14 @@ on:
3232
description: 'Publish to which AWS region?'
3333
required: true
3434
type: string
35-
35+
role-arn:
36+
description: '(optional) AWS IAM Role ARN to be assumed for publishing layer. If no input is given, defaults to `PROD_LAMBDA_ROLE_ARN` secret.'
37+
required: false
38+
type: string
39+
layer-version:
40+
description: '(optional) Layer version to be used in the layer name. If no input is given, its value is tried to be extracted from the `GITHUB_REF_NAME` variable'
41+
required: false
42+
type: string
3643

3744
permissions:
3845
id-token: write
@@ -58,18 +65,24 @@ jobs:
5865
echo "ARCH=$ARCH" >> $GITHUB_ENV
5966
6067
if [[ -n "${{ inputs.runtimes }}" ]]; then
61-
RUNTIMES="--compatible-runtimes ${{ inputs.runtimes }}"
68+
COMPATIBLE_RUNTIMES="--compatible-runtimes ${{ inputs.runtimes }}"
6269
fi
63-
echo "RUNTIMES=$RUNTIMES" >> $GITHUB_ENV
70+
echo "COMPATIBLE_RUNTIMES=$COMPATIBLE_RUNTIMES" >> $GITHUB_ENV
6471
6572
if [[ "${{ inputs.release-group }}" != "prod" ]]; then
6673
LAYER_NAME=$LAYER_NAME-${{ inputs.release-group }}
6774
fi
6875
69-
LAYER_VERSION=$(echo "$GITHUB_REF_NAME" | sed -r 's/.*\/[^0-9\.]*//g')
70-
LAYER_VERSION_CLEANED=$(echo "$LAYER_VERSION" | sed -r 's/\./_/g')
76+
if [[ -n "${{ inputs.layer-version }}" ]]; then
77+
LAYER_VERSION="${{ inputs.layer-version }}"
78+
else
79+
LAYER_VERSION=$(echo "$GITHUB_REF_NAME" | sed -r 's/.*\/[^0-9\.]*//g')
80+
fi
81+
LAYER_VERSION_CLEANED=$(echo "$LAYER_VERSION" | sed -r 's/\./_/g')
7182
72-
LAYER_NAME=$LAYER_NAME-$LAYER_VERSION_CLEANED
83+
if [[ -n "$LAYER_VERSION_CLEANED" ]]; then
84+
LAYER_NAME=$LAYER_NAME-$LAYER_VERSION_CLEANED
85+
fi
7386
echo "LAYER_NAME=$LAYER_NAME" >> $GITHUB_ENV
7487
7588
echo GITHUB_ENV:
@@ -82,7 +95,7 @@ jobs:
8295

8396
- uses: aws-actions/configure-aws-credentials@v4
8497
with:
85-
role-to-assume: ${{ secrets.PROD_LAMBDA_ROLE_ARN }}
98+
role-to-assume: ${{ inputs.role-arn || secrets.OTEL_LAMBDA_LAYER_PUBLISH_ROLE_ARN || secrets.PROD_LAMBDA_ROLE_ARN }}
8699
role-duration-seconds: 1200
87100
aws-region: ${{ inputs.aws_region }}
88101
mask-aws-account-id: false
@@ -93,7 +106,8 @@ jobs:
93106
aws lambda publish-layer-version \
94107
--layer-name $LAYER_NAME \
95108
--license-info "Apache 2.0" \
96-
--compatible-architectures $ARCH $RUNTIMES \
109+
--compatible-architectures $ARCH \
110+
$COMPATIBLE_RUNTIMES \
97111
--zip-file fileb://${{ inputs.artifact-name }} \
98112
--query 'LayerVersionArn' \
99113
--output text

‎.github/workflows/publish-layer-collector.yml

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
name: "Publish Collector Lambda layer"
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
architecture:
7+
description: 'Architecture of the layer to be published'
8+
required: true
9+
type: choice
10+
options:
11+
- all
12+
- amd64
13+
- arm64
14+
default: all
15+
aws-region:
16+
description: 'AWS Region(s) where layer will be published'
17+
required: true
18+
type: choice
19+
options:
20+
- all
21+
- ap-northeast-1
22+
- ap-northeast-2
23+
- ap-south-1
24+
- ap-southeast-1
25+
- ap-southeast-2
26+
- ca-central-1
27+
- eu-central-1
28+
- eu-north-1
29+
- eu-west-1
30+
- eu-west-2
31+
- eu-west-3
32+
- sa-east-1
33+
- us-east-1
34+
- us-east-2
35+
- us-west-1
36+
- us-west-2
37+
default: all
38+
role-arn:
39+
description: 'AWS IAM Role ARN to be assumed for publishing layer'
40+
required: false
41+
type: string
42+
layer-version:
43+
description: 'Layer version to be appended into the layer name'
44+
required: false
45+
type: string
46+
build-tags:
47+
description: 'Build tags to customize collector build'
48+
required: false
49+
type: string
50+
51+
jobs:
52+
prepare-build-jobs:
53+
runs-on: ubuntu-latest
54+
outputs:
55+
build_jobs: ${{ steps.prepare-build-jobs.outputs.build_jobs }}
56+
steps:
57+
- id: prepare-build-jobs
58+
name: Prepare Build Jobs
59+
run: |
60+
architectures=''
61+
if [ ${{ github.event.inputs.architecture }} == 'all' ]; then
62+
architectures='["amd64", "arm64"]'
63+
else
64+
architectures='["${{ github.event.inputs.architecture }}"]'
65+
fi
66+
echo "build_jobs={"architecture": ${architectures}}" | tr -d '[:space:]' >> $GITHUB_OUTPUT
67+
build-layer:
68+
needs: prepare-build-jobs
69+
runs-on: ubuntu-latest
70+
strategy:
71+
matrix: ${{ fromJSON(needs.prepare-build-jobs.outputs.build_jobs) }}
72+
steps:
73+
- name: Checkout Repo
74+
uses: actions/checkout@v4
75+
- name: Setup Go
76+
uses: actions/setup-go@v5
77+
with:
78+
go-version: '~1.21.9'
79+
- name: Build Collector
80+
run: |
81+
if [[ -n "${{ inputs.build-tags }}" ]]; then
82+
BUILDTAGS="${{ inputs.build-tags }}"
83+
if [[ "$BUILDTAGS" != "lambdacomponents.custom"* ]]; then
84+
BUILDTAGS="lambdacomponents.custom,$BUILDTAGS"
85+
fi
86+
fi
87+
echo "Build tags: $BUILDTAGS"
88+
make -C collector package GOARCH=${{ matrix.architecture }} BUILDTAGS=$BUILDTAGS
89+
- name: Upload Collector Artifact
90+
uses: actions/upload-artifact@v4
91+
with:
92+
name: opentelemetry-collector-layer-${{ matrix.architecture }}.zip
93+
path: ${{ github.workspace }}/collector/build/opentelemetry-collector-layer-${{ matrix.architecture }}.zip
94+
prepare-release-jobs:
95+
needs: build-layer
96+
runs-on: ubuntu-latest
97+
outputs:
98+
release_jobs: ${{ steps.prepare-release-jobs.outputs.release_jobs }}
99+
steps:
100+
- id: prepare-release-jobs
101+
name: Prepare Release Jobs
102+
run: |
103+
architectures=''
104+
if [ ${{ github.event.inputs.architecture }} == 'all' ]; then
105+
architectures='["amd64", "arm64"]'
106+
else
107+
architectures='["${{ github.event.inputs.architecture }}"]'
108+
fi
109+
aws_regions=''
110+
if [ ${{ github.event.inputs.aws-region }} == 'all' ]; then
111+
aws_regions='["ap-northeast-1", "ap-northeast-2", "ap-south-1", "ap-southeast-1", "ap-southeast-2", "ca-central-1", "eu-central-1", "eu-north-1", "eu-west-1", "eu-west-2", "eu-west-3", "sa-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2"]'
112+
else
113+
aws_regions='["${{ github.event.inputs.aws-region }}"]'
114+
fi
115+
echo "release_jobs={"architecture": ${architectures}, "aws_region": ${aws_regions}}" | tr -d '[:space:]' >> $GITHUB_OUTPUT
116+
release-layer:
117+
uses: ./.github/workflows/layer-publish.yml
118+
needs: prepare-release-jobs
119+
strategy:
120+
matrix: ${{ fromJSON(needs.prepare-release-jobs.outputs.release_jobs) }}
121+
with:
122+
artifact-name: opentelemetry-collector-layer-${{ matrix.architecture }}.zip
123+
layer-name: opentelemetry-collector
124+
architecture: ${{ matrix.architecture }}
125+
runtimes: "nodejs16.x nodejs18.x nodejs20.x nodejs22.x java11 java17 java21 python3.8 python3.9 python3.10 python3.11 python3.12"
126+
release-group: prod
127+
aws_region: ${{ matrix.aws_region }}
128+
role-arn: ${{ github.event.inputs.role-arn }}
129+
component-version: 'NA'
130+
layer-version: ${{ github.event.inputs.layer-version }}
131+
secrets: inherit

‎collector/README.md

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,40 @@ For example, if you want to add the extension `foo`, the file providing this ext
5252

5353
You can provide your addition as a pull-request to this repository. Before doing so, please also read through the details of [Contributing](#contributing) to this project.
5454

55+
## Build and publish your own OpenTelemetry Collector Lambda layer
56+
57+
To build and publish collector Lambda layer from your own fork into your own AWS account,
58+
you can use the `Publish Collector Lambda Layer` workflow which can only be triggered manually.
59+
60+
61+
To do that, first you need to
62+
- Create Github's OIDC provider in your (or target) AWS account (for more details, you can check [here](https://github.com/aws-actions/configure-aws-credentials?oidc))
63+
- Create an AWS IAM Role in the AWS account to be assumed by the `Publish Collector Lambda Layer` workflow from your forked OpenTelemetry Lambda repository.
64+
65+
To setup those, you can use (copy or load) the AWS CloudFormation template [here](../utils/aws-cloudformation/aws-cf-stack-for-layer-publish.yml).
66+
Once AWS CloudFormation stack is created from the given template,
67+
ARN of the created AWS IAM Role to be assumed will be shown as `RoleARN` in the output of the stack, so note it to be used later.
68+
69+
After that, you can run the `Publish Collector Lambda Layer` workflow to build the Lambda collector and publish it to the target AWS account as Lambda layer:
70+
- Specify the architecture of the collector Lambda layer to be published via the `Architecture of the layer to be published` input.
71+
Available options are `all`, `amd64` and `arm64`.
72+
The default value is `all` which builds and publishes layer for both of the `amd64` and `arm64` architectures.
73+
- Specify the AWS region(s) where the collector Lambda layer will be published to via the `AWS Region(s) where layer will be published` input.
74+
Available options are `all`, `ap-northeast-1`, `ap-northeast-2`, `ap-south-1`, `ap-southeast-1`, `ap-southeast-2`, `ca-central-1`, `eu-central-1`, `eu-north-1`, `eu-west-1`, `eu-west-2`, `eu-west-3`, `sa-east-1`, `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`.
75+
The default value is `all` which publishes layer to all the defined AWS regions mentioned above.
76+
- Specify the AWS IAM Role ARN to be assumed for publishing layer via the `AWS IAM Role ARN to be assumed for publishing layer` input.
77+
This is the ARN of the AWS IAM Role you have taken from the `RoleARN` output variable of the created AWS CloudFormation stack above.
78+
This input is **optional** and if not specified, AWS IAM Role ARN to be assumed is tried to be resolved from `OTEL_LAMBDA_LAYER_PUBLISH_ROLE_ARN` secret.
79+
If it is still not able to resolved (neither this input is specified, nor `OTEL_LAMBDA_LAYER_PUBLISH_ROLE_ARN` secret is defined),
80+
layer publish job will fail due to missing AWS credentials.
81+
- Specify the layer version to be appended into layer name via the `Layer version to be appended into the layer name` input
82+
to be used in the following format: `opentelemetry-lambda-collector-${architecture}-${layer-version}`.
83+
This input is **optional** and if not specified, layer name is generated in the `opentelemetry-lambda-collector-${architecture}` format without layer version postfix.
84+
- Specify the build tags to build the collector with a customized set of connectors/exporters/receivers/processors
85+
via the `Build tags to customize collector build` input.
86+
This input is **optional** and if not specified, collector is built with the default set of connectors/exporters/receivers/processors.
87+
Check the [Build Tags](#build-tags) section for the details.
88+
5589
## Installing
5690
To install the OpenTelemetry Collector Lambda layer to an existing Lambda function using the `aws` CLI:
5791

‎utils/aws-cloudformation/aws-cf-stack-for-layer-publish.yml

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
Parameters:
2+
GitHubOrgName:
3+
Description: Name of the GitHub organization/user
4+
Type: String
5+
RepositoryName:
6+
Description: Name of the GitHub repository
7+
Type: String
8+
Default: "opentelemetry-lambda"
9+
10+
Resources:
11+
Role:
12+
Type: AWS::IAM::Role
13+
Properties:
14+
RoleName: "github-otel-lambda-layer-publish-role"
15+
AssumeRolePolicyDocument:
16+
Statement:
17+
- Effect: Allow
18+
Action: sts:AssumeRoleWithWebIdentity
19+
Principal:
20+
Federated: !Ref GithubOIDC
21+
Condition:
22+
StringEquals:
23+
token.actions.githubusercontent.com:aud: "sts.amazonaws.com"
24+
StringLike:
25+
token.actions.githubusercontent.com:sub: !Sub "repo:${GitHubOrgName}/${RepositoryName}:*"
26+
Policies:
27+
- PolicyName: "github-otel-lambda-layer-publish-policy"
28+
PolicyDocument:
29+
Version: "2012-10-17"
30+
Statement:
31+
- Effect: Allow
32+
Action:
33+
- "lambda:GetLayer*"
34+
- "lambda:ListLayer*"
35+
- "lambda:AddLayer*"
36+
- "lambda:PublishLayer*"
37+
Resource:
38+
- !Sub "arn:aws:lambda:*:${AWS::AccountId}:layer:opentelemetry-*"
39+
- !Sub "arn:aws:lambda:*:${AWS::AccountId}:layer:opentelemetry-*:*"
40+
41+
GithubOIDC:
42+
Type: AWS::IAM::OIDCProvider
43+
Properties:
44+
Url: "https://token.actions.githubusercontent.com"
45+
ClientIdList:
46+
- "sts.amazonaws.com"
47+
ThumbprintList:
48+
- "ffffffffffffffffffffffffffffffffffffffff"
49+
50+
Outputs:
51+
RoleARN:
52+
Description: "ARN of the AWS IAM role to be assumed by Github for the OpenTelemetry Layer publishing"
53+
Value: !GetAtt Role.Arn

0 commit comments

Comments
 (0)