Skip to content

Commit cf8646d

Browse files
yliang412yliang412
authored
Merge pull request #9528 from neondatabase/rc/2024-10-25
Storage & Compute release 2024-10-25
2 parents c4e5693 + 46e9a47 commit cf8646d

File tree

143 files changed

+5502
-2510
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

143 files changed

+5502
-2510
lines changed

Diff for: ‎.github/workflows/_benchmarking_preparation.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ jobs:
2727

2828
runs-on: [ self-hosted, us-east-2, x64 ]
2929
container:
30-
image: neondatabase/build-tools:pinned
30+
image: neondatabase/build-tools:pinned-bookworm
3131
credentials:
3232
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
3333
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

Diff for: ‎.github/workflows/_build-and-test-locally.yml

+4-18
Original file line numberDiff line numberDiff line change
@@ -53,20 +53,6 @@ jobs:
5353
BUILD_TAG: ${{ inputs.build-tag }}
5454

5555
steps:
56-
- name: Fix git ownership
57-
run: |
58-
# Workaround for `fatal: detected dubious ownership in repository at ...`
59-
#
60-
# Use both ${{ github.workspace }} and ${GITHUB_WORKSPACE} because they're different on host and in containers
61-
# Ref https://github.com/actions/checkout/issues/785
62-
#
63-
git config --global --add safe.directory ${{ github.workspace }}
64-
git config --global --add safe.directory ${GITHUB_WORKSPACE}
65-
for r in 14 15 16 17; do
66-
git config --global --add safe.directory "${{ github.workspace }}/vendor/postgres-v$r"
67-
git config --global --add safe.directory "${GITHUB_WORKSPACE}/vendor/postgres-v$r"
68-
done
69-
7056
- uses: actions/checkout@v4
7157
with:
7258
submodules: true
@@ -124,28 +110,28 @@ jobs:
124110
uses: actions/cache@v4
125111
with:
126112
path: pg_install/v14
127-
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }}
113+
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }}
128114

129115
- name: Cache postgres v15 build
130116
id: cache_pg_15
131117
uses: actions/cache@v4
132118
with:
133119
path: pg_install/v15
134-
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }}
120+
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }}
135121

136122
- name: Cache postgres v16 build
137123
id: cache_pg_16
138124
uses: actions/cache@v4
139125
with:
140126
path: pg_install/v16
141-
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }}
127+
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }}
142128

143129
- name: Cache postgres v17 build
144130
id: cache_pg_17
145131
uses: actions/cache@v4
146132
with:
147133
path: pg_install/v17
148-
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v17_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }}
134+
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v17_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }}
149135

150136
- name: Build postgres v14
151137
if: steps.cache_pg_14.outputs.cache-hit != 'true'

Diff for: ‎.github/workflows/benchmarking.yml

+9-9
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ jobs:
8383

8484
runs-on: ${{ matrix.RUNNER }}
8585
container:
86-
image: neondatabase/build-tools:pinned
86+
image: neondatabase/build-tools:pinned-bookworm
8787
credentials:
8888
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
8989
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
@@ -178,7 +178,7 @@ jobs:
178178

179179
runs-on: [ self-hosted, us-east-2, x64 ]
180180
container:
181-
image: neondatabase/build-tools:pinned
181+
image: neondatabase/build-tools:pinned-bookworm
182182
credentials:
183183
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
184184
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
@@ -280,7 +280,7 @@ jobs:
280280
region_id_default=${{ env.DEFAULT_REGION_ID }}
281281
runner_default='["self-hosted", "us-east-2", "x64"]'
282282
runner_azure='["self-hosted", "eastus2", "x64"]'
283-
image_default="neondatabase/build-tools:pinned"
283+
image_default="neondatabase/build-tools:pinned-bookworm"
284284
matrix='{
285285
"pg_version" : [
286286
16
@@ -299,9 +299,9 @@ jobs:
299299
"include": [{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_default"', "image": "'"$image_default"'" },
300300
{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new", "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
301301
{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new", "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
302-
{ "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned" },
303-
{ "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "10gb","runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned" },
304-
{ "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "50gb","runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned" },
302+
{ "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned-bookworm" },
303+
{ "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "10gb","runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned-bookworm" },
304+
{ "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "50gb","runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned-bookworm" },
305305
{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-sharding-reuse", "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" }]
306306
}'
307307
@@ -665,7 +665,7 @@ jobs:
665665

666666
runs-on: [ self-hosted, us-east-2, x64 ]
667667
container:
668-
image: neondatabase/build-tools:pinned
668+
image: neondatabase/build-tools:pinned-bookworm
669669
credentials:
670670
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
671671
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
@@ -772,7 +772,7 @@ jobs:
772772

773773
runs-on: [ self-hosted, us-east-2, x64 ]
774774
container:
775-
image: neondatabase/build-tools:pinned
775+
image: neondatabase/build-tools:pinned-bookworm
776776
credentials:
777777
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
778778
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
@@ -877,7 +877,7 @@ jobs:
877877

878878
runs-on: [ self-hosted, us-east-2, x64 ]
879879
container:
880-
image: neondatabase/build-tools:pinned
880+
image: neondatabase/build-tools:pinned-bookworm
881881
credentials:
882882
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
883883
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

Diff for: ‎.github/workflows/build-build-tools-image.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ jobs:
8282

8383
- uses: docker/build-push-action@v6
8484
with:
85-
file: Dockerfile.build-tools
85+
file: build-tools.Dockerfile
8686
context: .
8787
provenance: false
8888
push: true

Diff for: ‎.github/workflows/build_and_test.yml

+3-17
Original file line numberDiff line numberDiff line change
@@ -683,7 +683,7 @@ jobs:
683683
provenance: false
684684
push: true
685685
pull: true
686-
file: compute/Dockerfile.compute-node
686+
file: compute/compute-node.Dockerfile
687687
cache-from: type=registry,ref=cache.neon.build/compute-node-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }}
688688
cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/compute-node-{0}:cache-{1}-{2},mode=max', matrix.version.pg, matrix.version.debian, matrix.arch) || '' }}
689689
tags: |
@@ -703,7 +703,7 @@ jobs:
703703
provenance: false
704704
push: true
705705
pull: true
706-
file: compute/Dockerfile.compute-node
706+
file: compute/compute-node.Dockerfile
707707
target: neon-pg-ext-test
708708
cache-from: type=registry,ref=cache.neon.build/neon-test-extensions-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }}
709709
cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/neon-test-extensions-{0}:cache-{1}-{2},mode=max', matrix.version.pg, matrix.version.debian, matrix.arch) || '' }}
@@ -728,7 +728,7 @@ jobs:
728728
provenance: false
729729
push: true
730730
pull: true
731-
file: compute/Dockerfile.compute-node
731+
file: compute/compute-node.Dockerfile
732732
cache-from: type=registry,ref=cache.neon.build/neon-test-extensions-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }}
733733
cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/compute-tools-{0}:cache-{1}-{2},mode=max', matrix.version.pg, matrix.version.debian, matrix.arch) || '' }}
734734
tags: |
@@ -1078,20 +1078,6 @@ jobs:
10781078
runs-on: [ self-hosted, small ]
10791079
container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
10801080
steps:
1081-
- name: Fix git ownership
1082-
run: |
1083-
# Workaround for `fatal: detected dubious ownership in repository at ...`
1084-
#
1085-
# Use both ${{ github.workspace }} and ${GITHUB_WORKSPACE} because they're different on host and in containers
1086-
# Ref https://github.com/actions/checkout/issues/785
1087-
#
1088-
git config --global --add safe.directory ${{ github.workspace }}
1089-
git config --global --add safe.directory ${GITHUB_WORKSPACE}
1090-
for r in 14 15 16 17; do
1091-
git config --global --add safe.directory "${{ github.workspace }}/vendor/postgres-v$r"
1092-
git config --global --add safe.directory "${GITHUB_WORKSPACE}/vendor/postgres-v$r"
1093-
done
1094-
10951081
- uses: actions/checkout@v4
10961082

10971083
- name: Trigger deploy workflow

Diff for: ‎.github/workflows/check-build-tools-image.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131
id: get-build-tools-tag
3232
env:
3333
IMAGE_TAG: |
34-
${{ hashFiles('Dockerfile.build-tools',
34+
${{ hashFiles('build-tools.Dockerfile',
3535
'.github/workflows/check-build-tools-image.yml',
3636
'.github/workflows/build-build-tools-image.yml') }}
3737
run: |

Diff for: ‎.github/workflows/cloud-regress.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131

3232
runs-on: us-east-2
3333
container:
34-
image: neondatabase/build-tools:pinned
34+
image: neondatabase/build-tools:pinned-bookworm
3535
options: --init
3636

3737
steps:

Diff for: ‎.github/workflows/trigger-e2e-tests.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ jobs:
112112
# This isn't exhaustive, just the paths that are most directly compute-related.
113113
# For example, compute_ctl also depends on libs/utils, but we don't trigger
114114
# an e2e run on that.
115-
vendor/*|pgxn/*|compute_tools/*|libs/vm_monitor/*|compute/Dockerfile.compute-node)
115+
vendor/*|pgxn/*|compute_tools/*|libs/vm_monitor/*|compute/compute-node.Dockerfile)
116116
platforms=$(echo "${platforms}" | jq --compact-output '. += ["k8s-neonvm"] | unique')
117117
;;
118118
*)

Diff for: ‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ __pycache__/
66
test_output/
77
.vscode
88
.idea
9+
*.swp
10+
tags
911
neon.iml
1012
/.neon
1113
/integration_tests/.neon

Diff for: ‎Cargo.lock

+2-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎Makefile

+2-2
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,7 @@ clean: postgres-clean neon-pg-clean-ext
297297
# This removes everything
298298
.PHONY: distclean
299299
distclean:
300-
rm -rf $(POSTGRES_INSTALL_DIR)
300+
$(RM) -r $(POSTGRES_INSTALL_DIR)
301301
$(CARGO_CMD_PREFIX) cargo clean
302302

303303
.PHONY: fmt
@@ -329,7 +329,7 @@ postgres-%-pgindent: postgres-%-pg-bsd-indent postgres-%-typedefs.list
329329
$(ROOT_PROJECT_DIR)/vendor/postgres-$*/src/tools/pgindent/pgindent --typedefs postgres-$*-typedefs-full.list \
330330
$(ROOT_PROJECT_DIR)/vendor/postgres-$*/src/ \
331331
--excludes $(ROOT_PROJECT_DIR)/vendor/postgres-$*/src/tools/pgindent/exclude_file_patterns
332-
rm -f pg*.BAK
332+
$(RM) pg*.BAK
333333

334334
# Indent pxgn/neon.
335335
.PHONY: neon-pgindent

Diff for: ‎Dockerfile.build-tools renamed to ‎build-tools.Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ RUN wget -O /tmp/openssl-${OPENSSL_VERSION}.tar.gz https://www.openssl.org/sourc
142142
# Use the same version of libicu as the compute nodes so that
143143
# clusters created using inidb on pageserver can be used by computes.
144144
#
145-
# TODO: at this time, Dockerfile.compute-node uses the debian bullseye libicu
145+
# TODO: at this time, compute-node.Dockerfile uses the debian bullseye libicu
146146
# package, which is 67.1. We're duplicating that knowledge here, and also, technically,
147147
# Debian has a few patches on top of 67.1 that we're not adding here.
148148
ENV ICU_VERSION=67.1

Diff for: ‎compute/Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ sql_exporter_autoscaling.yml: $(jsonnet_files)
3434

3535
.PHONY: clean
3636
clean:
37-
rm --force \
37+
$(RM) \
3838
etc/neon_collector.yml \
3939
etc/neon_collector_autoscaling.yml \
4040
etc/sql_exporter.yml \

Diff for: ‎compute/README.md

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
This directory contains files that are needed to build the compute
22
images, or included in the compute images.
33

4-
Dockerfile.compute-node
4+
compute-node.Dockerfile
55
To build the compute image
66

77
vm-image-spec.yaml
@@ -14,8 +14,8 @@ etc/
1414
patches/
1515
Some extensions need to be patched to work with Neon. This
1616
directory contains such patches. They are applied to the extension
17-
sources in Dockerfile.compute-node
17+
sources in compute-node.Dockerfile
1818

1919
In addition to these, postgres itself, the neon postgres extension,
2020
and compute_ctl are built and copied into the compute image by
21-
Dockerfile.compute-node.
21+
compute-node.Dockerfile.

Diff for: ‎compute/Dockerfile.compute-node renamed to ‎compute/compute-node.Dockerfile

File renamed without changes.

Diff for: ‎compute/etc/sql_exporter/checkpoints_timed.libsonnet

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
local neon = import 'neon.libsonnet';
22

3-
local pg_stat_bgwriter = importstr 'sql_exporter/checkpoints_req.sql';
4-
local pg_stat_checkpointer = importstr 'sql_exporter/checkpoints_req.17.sql';
3+
local pg_stat_bgwriter = importstr 'sql_exporter/checkpoints_timed.sql';
4+
local pg_stat_checkpointer = importstr 'sql_exporter/checkpoints_timed.17.sql';
55

66
{
77
metric_name: 'checkpoints_timed',

Diff for: ‎compute/etc/sql_exporter/retained_wal.sql

+6-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
SELECT
22
slot_name,
3-
pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)::FLOAT8 AS retained_wal
3+
pg_wal_lsn_diff(
4+
CASE
5+
WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn()
6+
ELSE pg_current_wal_lsn()
7+
END,
8+
restart_lsn)::FLOAT8 AS retained_wal
49
FROM pg_replication_slots
510
WHERE active = false;

Diff for: ‎control_plane/src/bin/neon_local.rs

+13-12
Original file line numberDiff line numberDiff line change
@@ -1073,10 +1073,10 @@ async fn handle_tenant(subcmd: &TenantCmd, env: &mut local_env::LocalEnv) -> any
10731073
tenant_id,
10741074
TimelineCreateRequest {
10751075
new_timeline_id,
1076-
ancestor_timeline_id: None,
1077-
ancestor_start_lsn: None,
1078-
existing_initdb_timeline_id: None,
1079-
pg_version: Some(args.pg_version),
1076+
mode: pageserver_api::models::TimelineCreateRequestMode::Bootstrap {
1077+
existing_initdb_timeline_id: None,
1078+
pg_version: Some(args.pg_version),
1079+
},
10801080
},
10811081
)
10821082
.await?;
@@ -1133,10 +1133,10 @@ async fn handle_timeline(cmd: &TimelineCmd, env: &mut local_env::LocalEnv) -> Re
11331133
let storage_controller = StorageController::from_env(env);
11341134
let create_req = TimelineCreateRequest {
11351135
new_timeline_id,
1136-
ancestor_timeline_id: None,
1137-
existing_initdb_timeline_id: None,
1138-
ancestor_start_lsn: None,
1139-
pg_version: Some(args.pg_version),
1136+
mode: pageserver_api::models::TimelineCreateRequestMode::Bootstrap {
1137+
existing_initdb_timeline_id: None,
1138+
pg_version: Some(args.pg_version),
1139+
},
11401140
};
11411141
let timeline_info = storage_controller
11421142
.tenant_timeline_create(tenant_id, create_req)
@@ -1189,10 +1189,11 @@ async fn handle_timeline(cmd: &TimelineCmd, env: &mut local_env::LocalEnv) -> Re
11891189
let storage_controller = StorageController::from_env(env);
11901190
let create_req = TimelineCreateRequest {
11911191
new_timeline_id,
1192-
ancestor_timeline_id: Some(ancestor_timeline_id),
1193-
existing_initdb_timeline_id: None,
1194-
ancestor_start_lsn: start_lsn,
1195-
pg_version: None,
1192+
mode: pageserver_api::models::TimelineCreateRequestMode::Branch {
1193+
ancestor_timeline_id,
1194+
ancestor_start_lsn: start_lsn,
1195+
pg_version: None,
1196+
},
11961197
};
11971198
let timeline_info = storage_controller
11981199
.tenant_timeline_create(tenant_id, create_req)

Diff for: ‎control_plane/src/pageserver.rs

-22
Original file line numberDiff line numberDiff line change
@@ -529,28 +529,6 @@ impl PageServerNode {
529529
Ok(self.http_client.list_timelines(*tenant_shard_id).await?)
530530
}
531531

532-
pub async fn timeline_create(
533-
&self,
534-
tenant_shard_id: TenantShardId,
535-
new_timeline_id: TimelineId,
536-
ancestor_start_lsn: Option<Lsn>,
537-
ancestor_timeline_id: Option<TimelineId>,
538-
pg_version: Option<u32>,
539-
existing_initdb_timeline_id: Option<TimelineId>,
540-
) -> anyhow::Result<TimelineInfo> {
541-
let req = models::TimelineCreateRequest {
542-
new_timeline_id,
543-
ancestor_start_lsn,
544-
ancestor_timeline_id,
545-
pg_version,
546-
existing_initdb_timeline_id,
547-
};
548-
Ok(self
549-
.http_client
550-
.timeline_create(tenant_shard_id, &req)
551-
.await?)
552-
}
553-
554532
/// Import a basebackup prepared using either:
555533
/// a) `pg_basebackup -F tar`, or
556534
/// b) The `fullbackup` pageserver endpoint

0 commit comments

Comments
 (0)