Skip to content

Commit c68b346

Browse files
knizhnikknizhnik
authored
Merge pull request #10467 from neondatabase/compute-rc-2025-01-21
Compute release 2025-01-21
2 parents 045b05c + 19bf7b7 commit c68b346

File tree

74 files changed

+3461
-873
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

74 files changed

+3461
-873
lines changed

Diff for: ‎.github/workflows/_check-codestyle-rust.yml

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
name: Check Codestyle Rust
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
build-tools-image:
7+
description: "build-tools image"
8+
required: true
9+
type: string
10+
archs:
11+
description: "Json array of architectures to run on"
12+
type: string
13+
14+
15+
defaults:
16+
run:
17+
shell: bash -euxo pipefail {0}
18+
19+
jobs:
20+
check-codestyle-rust:
21+
strategy:
22+
matrix:
23+
arch: ${{ fromJson(inputs.archs) }}
24+
runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' || 'small')) }}
25+
26+
container:
27+
image: ${{ inputs.build-tools-image }}
28+
credentials:
29+
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
30+
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
31+
options: --init
32+
33+
steps:
34+
- name: Checkout
35+
uses: actions/checkout@v4
36+
with:
37+
submodules: true
38+
39+
- name: Cache cargo deps
40+
uses: actions/cache@v4
41+
with:
42+
path: |
43+
~/.cargo/registry
44+
!~/.cargo/registry/src
45+
~/.cargo/git
46+
target
47+
key: v1-${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./rust-toolchain.toml') }}-rust
48+
49+
# Some of our rust modules use FFI and need those to be checked
50+
- name: Get postgres headers
51+
run: make postgres-headers -j$(nproc)
52+
53+
# cargo hack runs the given cargo subcommand (clippy in this case) for all feature combinations.
54+
# This will catch compiler & clippy warnings in all feature combinations.
55+
# TODO: use cargo hack for build and test as well, but, that's quite expensive.
56+
# NB: keep clippy args in sync with ./run_clippy.sh
57+
#
58+
# The only difference between "clippy --debug" and "clippy --release" is that in --release mode,
59+
# #[cfg(debug_assertions)] blocks are not built. It's not worth building everything for second
60+
# time just for that, so skip "clippy --release".
61+
- run: |
62+
CLIPPY_COMMON_ARGS="$( source .neon_clippy_args; echo "$CLIPPY_COMMON_ARGS")"
63+
if [ "$CLIPPY_COMMON_ARGS" = "" ]; then
64+
echo "No clippy args found in .neon_clippy_args"
65+
exit 1
66+
fi
67+
echo "CLIPPY_COMMON_ARGS=${CLIPPY_COMMON_ARGS}" >> $GITHUB_ENV
68+
- name: Run cargo clippy (debug)
69+
run: cargo hack --features default --ignore-unknown-features --feature-powerset clippy $CLIPPY_COMMON_ARGS
70+
71+
- name: Check documentation generation
72+
run: cargo doc --workspace --no-deps --document-private-items
73+
env:
74+
RUSTDOCFLAGS: "-Dwarnings -Arustdoc::private_intra_doc_links"
75+
76+
# Use `${{ !cancelled() }}` to run quck tests after the longer clippy run
77+
- name: Check formatting
78+
if: ${{ !cancelled() }}
79+
run: cargo fmt --all -- --check
80+
81+
# https://github.com/facebookincubator/cargo-guppy/tree/bec4e0eb29dcd1faac70b1b5360267fc02bf830e/tools/cargo-hakari#2-keep-the-workspace-hack-up-to-date-in-ci
82+
- name: Check rust dependencies
83+
if: ${{ !cancelled() }}
84+
run: |
85+
cargo hakari generate --diff # workspace-hack Cargo.toml is up-to-date
86+
cargo hakari manage-deps --dry-run # all workspace crates depend on workspace-hack
87+
88+
# https://github.com/EmbarkStudios/cargo-deny
89+
- name: Check rust licenses/bans/advisories/sources
90+
if: ${{ !cancelled() }}
91+
run: cargo deny check --hide-inclusion-graph

Diff for: ‎.github/workflows/build_and_test.yml

+8-79
Original file line numberDiff line numberDiff line change
@@ -164,77 +164,11 @@ jobs:
164164

165165
check-codestyle-rust:
166166
needs: [ check-permissions, build-build-tools-image ]
167-
strategy:
168-
matrix:
169-
arch: [ x64, arm64 ]
170-
runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' || 'small')) }}
171-
172-
container:
173-
image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
174-
credentials:
175-
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
176-
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
177-
options: --init
178-
179-
steps:
180-
- name: Checkout
181-
uses: actions/checkout@v4
182-
with:
183-
submodules: true
184-
185-
- name: Cache cargo deps
186-
uses: actions/cache@v4
187-
with:
188-
path: |
189-
~/.cargo/registry
190-
!~/.cargo/registry/src
191-
~/.cargo/git
192-
target
193-
key: v1-${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./rust-toolchain.toml') }}-rust
194-
195-
# Some of our rust modules use FFI and need those to be checked
196-
- name: Get postgres headers
197-
run: make postgres-headers -j$(nproc)
198-
199-
# cargo hack runs the given cargo subcommand (clippy in this case) for all feature combinations.
200-
# This will catch compiler & clippy warnings in all feature combinations.
201-
# TODO: use cargo hack for build and test as well, but, that's quite expensive.
202-
# NB: keep clippy args in sync with ./run_clippy.sh
203-
#
204-
# The only difference between "clippy --debug" and "clippy --release" is that in --release mode,
205-
# #[cfg(debug_assertions)] blocks are not built. It's not worth building everything for second
206-
# time just for that, so skip "clippy --release".
207-
- run: |
208-
CLIPPY_COMMON_ARGS="$( source .neon_clippy_args; echo "$CLIPPY_COMMON_ARGS")"
209-
if [ "$CLIPPY_COMMON_ARGS" = "" ]; then
210-
echo "No clippy args found in .neon_clippy_args"
211-
exit 1
212-
fi
213-
echo "CLIPPY_COMMON_ARGS=${CLIPPY_COMMON_ARGS}" >> $GITHUB_ENV
214-
- name: Run cargo clippy (debug)
215-
run: cargo hack --features default --ignore-unknown-features --feature-powerset clippy $CLIPPY_COMMON_ARGS
216-
217-
- name: Check documentation generation
218-
run: cargo doc --workspace --no-deps --document-private-items
219-
env:
220-
RUSTDOCFLAGS: "-Dwarnings -Arustdoc::private_intra_doc_links"
221-
222-
# Use `${{ !cancelled() }}` to run quck tests after the longer clippy run
223-
- name: Check formatting
224-
if: ${{ !cancelled() }}
225-
run: cargo fmt --all -- --check
226-
227-
# https://github.com/facebookincubator/cargo-guppy/tree/bec4e0eb29dcd1faac70b1b5360267fc02bf830e/tools/cargo-hakari#2-keep-the-workspace-hack-up-to-date-in-ci
228-
- name: Check rust dependencies
229-
if: ${{ !cancelled() }}
230-
run: |
231-
cargo hakari generate --diff # workspace-hack Cargo.toml is up-to-date
232-
cargo hakari manage-deps --dry-run # all workspace crates depend on workspace-hack
233-
234-
# https://github.com/EmbarkStudios/cargo-deny
235-
- name: Check rust licenses/bans/advisories/sources
236-
if: ${{ !cancelled() }}
237-
run: cargo deny check --hide-inclusion-graph
167+
uses: ./.github/workflows/_check-codestyle-rust.yml
168+
with:
169+
build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
170+
archs: '["x64", "arm64"]'
171+
secrets: inherit
238172

239173
build-and-test-locally:
240174
needs: [ tag, build-build-tools-image ]
@@ -890,7 +824,7 @@ jobs:
890824
docker compose -f ./docker-compose/docker-compose.yml down
891825
892826
promote-images-dev:
893-
needs: [ check-permissions, tag, vm-compute-node-image ]
827+
needs: [ check-permissions, tag, vm-compute-node-image, neon-image ]
894828
runs-on: ubuntu-22.04
895829

896830
permissions:
@@ -1144,12 +1078,6 @@ jobs:
11441078
console.log(`Tag ${tag} created successfully.`);
11451079
}
11461080
1147-
// TODO: check how GitHub releases looks for proxy/compute releases and enable them if they're ok
1148-
if (context.ref !== 'refs/heads/release') {
1149-
console.log(`GitHub release skipped for ${context.ref}.`);
1150-
return;
1151-
}
1152-
11531081
try {
11541082
const existingRelease = await github.rest.repos.getReleaseByTag({
11551083
owner: context.repo.owner,
@@ -1168,7 +1096,8 @@ jobs:
11681096
owner: context.repo.owner,
11691097
repo: context.repo.repo,
11701098
tag_name: tag,
1171-
generate_release_notes: true,
1099+
// TODO: Automate release notes properly
1100+
generate_release_notes: false,
11721101
});
11731102
console.log(`Release for tag ${tag} created successfully.`);
11741103
}

Diff for: ‎.github/workflows/pre-merge-checks.yml

+33
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
name: Pre-merge checks
22

33
on:
4+
pull_request:
5+
paths:
6+
- .github/workflows/_check-codestyle-python.yml
7+
- .github/workflows/_check-codestyle-rust.yml
8+
- .github/workflows/build-build-tools-image.yml
9+
- .github/workflows/pre-merge-checks.yml
410
merge_group:
511
branches:
612
- main
@@ -17,8 +23,10 @@ jobs:
1723
runs-on: ubuntu-22.04
1824
outputs:
1925
python-changed: ${{ steps.python-src.outputs.any_changed }}
26+
rust-changed: ${{ steps.rust-src.outputs.any_changed }}
2027
steps:
2128
- uses: actions/checkout@v4
29+
2230
- uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45.0.4
2331
id: python-src
2432
with:
@@ -30,11 +38,25 @@ jobs:
3038
poetry.lock
3139
pyproject.toml
3240
41+
- uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45.0.4
42+
id: rust-src
43+
with:
44+
files: |
45+
.github/workflows/_check-codestyle-rust.yml
46+
.github/workflows/build-build-tools-image.yml
47+
.github/workflows/pre-merge-checks.yml
48+
**/**.rs
49+
**/Cargo.toml
50+
Cargo.toml
51+
Cargo.lock
52+
3353
- name: PRINT ALL CHANGED FILES FOR DEBUG PURPOSES
3454
env:
3555
PYTHON_CHANGED_FILES: ${{ steps.python-src.outputs.all_changed_files }}
56+
RUST_CHANGED_FILES: ${{ steps.rust-src.outputs.all_changed_files }}
3657
run: |
3758
echo "${PYTHON_CHANGED_FILES}"
59+
echo "${RUST_CHANGED_FILES}"
3860
3961
build-build-tools-image:
4062
if: needs.get-changed-files.outputs.python-changed == 'true'
@@ -55,6 +77,16 @@ jobs:
5577
build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm-x64
5678
secrets: inherit
5779

80+
check-codestyle-rust:
81+
if: needs.get-changed-files.outputs.rust-changed == 'true'
82+
needs: [ get-changed-files, build-build-tools-image ]
83+
uses: ./.github/workflows/_check-codestyle-rust.yml
84+
with:
85+
# `-bookworm-x64` suffix should match the combination in `build-build-tools-image`
86+
build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm-x64
87+
archs: '["x64"]'
88+
secrets: inherit
89+
5890
# To get items from the merge queue merged into main we need to satisfy "Status checks that are required".
5991
# Currently we require 2 jobs (checks with exact name):
6092
# - conclusion
@@ -67,6 +99,7 @@ jobs:
6799
needs:
68100
- get-changed-files
69101
- check-codestyle-python
102+
- check-codestyle-rust
70103
runs-on: ubuntu-22.04
71104
steps:
72105
- name: Create fake `neon-cloud-e2e` check

Diff for: ‎Cargo.lock

+4
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎compute/compute-node.Dockerfile

+36-8
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ RUN cd postgres && \
6666
make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install && \
6767
# Enable some of contrib extensions
6868
echo 'trusted = true' >> /usr/local/pgsql/share/extension/autoinc.control && \
69+
echo 'trusted = true' >> /usr/local/pgsql/share/extension/dblink.control && \
6970
echo 'trusted = true' >> /usr/local/pgsql/share/extension/bloom.control && \
7071
echo 'trusted = true' >> /usr/local/pgsql/share/extension/earthdistance.control && \
7172
echo 'trusted = true' >> /usr/local/pgsql/share/extension/insert_username.control && \
@@ -994,24 +995,50 @@ RUN wget https://github.com/kelvich/pg_tiktoken/archive/9118dd4549b7d8c0bbc98e04
994995
#########################################################################################
995996
#
996997
# Layer "pg-pgx-ulid-build"
997-
# Compile "pgx_ulid" extension
998+
# Compile "pgx_ulid" extension for v16 and below
998999
#
9991000
#########################################################################################
10001001

10011002
FROM rust-extensions-build AS pg-pgx-ulid-build
10021003
ARG PG_VERSION
10031004

1004-
# doesn't support v17 yet
1005-
# https://github.com/pksunkara/pgx_ulid/pull/52
1006-
RUN case "${PG_VERSION}" in "v17") \
1007-
echo "pgx_ulid does not support pg17 as of the latest version (0.1.5)" && exit 0;; \
1005+
RUN case "${PG_VERSION}" in \
1006+
"v14" | "v15" | "v16") \
1007+
;; \
1008+
*) \
1009+
echo "skipping the version of pgx_ulid for $PG_VERSION" && exit 0 \
1010+
;; \
10081011
esac && \
10091012
wget https://github.com/pksunkara/pgx_ulid/archive/refs/tags/v0.1.5.tar.gz -O pgx_ulid.tar.gz && \
1010-
echo "9d1659a2da65af0133d5451c454de31b37364e3502087dadf579f790bc8bef17 pgx_ulid.tar.gz" | sha256sum --check && \
1013+
echo "9d1659a2da65af0133d5451c454de31b37364e3502087dadf579f790bc8bef17 pgx_ulid.tar.gz" | sha256sum --check && \
1014+
mkdir pgx_ulid-src && cd pgx_ulid-src && tar xzf ../pgx_ulid.tar.gz --strip-components=1 -C . && \
1015+
sed -i 's/pgrx = "^0.11.2"/pgrx = { version = "0.11.3", features = [ "unsafe-postgres" ] }/g' Cargo.toml && \
1016+
cargo pgrx install --release && \
1017+
echo 'trusted = true' >> /usr/local/pgsql/share/extension/ulid.control
1018+
1019+
#########################################################################################
1020+
#
1021+
# Layer "pg-pgx-ulid-pgrx12-build"
1022+
# Compile "pgx_ulid" extension for v17 and up
1023+
#
1024+
#########################################################################################
1025+
1026+
FROM rust-extensions-build-pgrx12 AS pg-pgx-ulid-pgrx12-build
1027+
ARG PG_VERSION
1028+
1029+
RUN case "${PG_VERSION}" in \
1030+
"v17") \
1031+
;; \
1032+
*) \
1033+
echo "skipping the version of pgx_ulid for $PG_VERSION" && exit 0 \
1034+
;; \
1035+
esac && \
1036+
wget https://github.com/pksunkara/pgx_ulid/archive/refs/tags/v0.2.0.tar.gz -O pgx_ulid.tar.gz && \
1037+
echo "cef6a9a2e5e7bd1a10a18989286586ee9e6c1c06005a4055cff190de41bf3e9f pgx_ulid.tar.gz" | sha256sum --check && \
10111038
mkdir pgx_ulid-src && cd pgx_ulid-src && tar xzf ../pgx_ulid.tar.gz --strip-components=1 -C . && \
1012-
sed -i 's/pgrx = "^0.11.2"/pgrx = { version = "=0.11.3", features = [ "unsafe-postgres" ] }/g' Cargo.toml && \
1039+
sed -i 's/pgrx = "^0.12.7"/pgrx = { version = "0.12.9", features = [ "unsafe-postgres" ] }/g' Cargo.toml && \
10131040
cargo pgrx install --release && \
1014-
echo "trusted = true" >> /usr/local/pgsql/share/extension/ulid.control
1041+
echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgx_ulid.control
10151042

10161043
#########################################################################################
10171044
#
@@ -1156,6 +1183,7 @@ COPY --from=timescaledb-pg-build /usr/local/pgsql/ /usr/local/pgsql/
11561183
COPY --from=pg-hint-plan-pg-build /usr/local/pgsql/ /usr/local/pgsql/
11571184
COPY --from=pg-cron-pg-build /usr/local/pgsql/ /usr/local/pgsql/
11581185
COPY --from=pg-pgx-ulid-build /usr/local/pgsql/ /usr/local/pgsql/
1186+
COPY --from=pg-pgx-ulid-pgrx12-build /usr/local/pgsql/ /usr/local/pgsql/
11591187
COPY --from=pg-session-jwt-build /usr/local/pgsql/ /usr/local/pgsql/
11601188
COPY --from=rdkit-pg-build /usr/local/pgsql/ /usr/local/pgsql/
11611189
COPY --from=pg-uuidv7-pg-build /usr/local/pgsql/ /usr/local/pgsql/

Diff for: ‎compute_tools/Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ tracing-subscriber.workspace = true
5151
tracing-utils.workspace = true
5252
thiserror.workspace = true
5353
url.workspace = true
54+
uuid.workspace = true
5455
prometheus.workspace = true
5556

5657
postgres_initdb.workspace = true

0 commit comments

Comments
 (0)