Skip to content

Commit fe1b181

Browse files
ololobusololobus
authored
Merge pull request #9459 from neondatabase/compute-rc-2024-10-20
Compute release 2024-10-20
2 parents 7f080da + cc25ef7 commit fe1b181

File tree

108 files changed

+2117
-1515
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

108 files changed

+2117
-1515
lines changed

Diff for: ‎Cargo.lock

+163-57
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎Cargo.toml

+6-6
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ reqwest-retry = "0.5"
142142
routerify = "3"
143143
rpds = "0.13"
144144
rustc-hash = "1.1.0"
145-
rustls = "0.22"
145+
rustls = "0.23"
146146
rustls-pemfile = "2"
147147
scopeguard = "1.1"
148148
sysinfo = "0.29.2"
@@ -172,8 +172,8 @@ tikv-jemalloc-ctl = "0.5"
172172
tokio = { version = "1.17", features = ["macros"] }
173173
tokio-epoll-uring = { git = "https://github.com/neondatabase/tokio-epoll-uring.git" , branch = "main" }
174174
tokio-io-timeout = "1.2.0"
175-
tokio-postgres-rustls = "0.11.0"
176-
tokio-rustls = "0.25"
175+
tokio-postgres-rustls = "0.12.0"
176+
tokio-rustls = "0.26"
177177
tokio-stream = "0.1"
178178
tokio-tar = "0.3"
179179
tokio-util = { version = "0.7.10", features = ["io", "rt"] }
@@ -192,8 +192,8 @@ url = "2.2"
192192
urlencoding = "2.1"
193193
uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] }
194194
walkdir = "2.3.2"
195-
rustls-native-certs = "0.7"
196-
x509-parser = "0.15"
195+
rustls-native-certs = "0.8"
196+
x509-parser = "0.16"
197197
whoami = "1.5.1"
198198

199199
## TODO replace this with tracing
@@ -244,7 +244,7 @@ workspace_hack = { version = "0.1", path = "./workspace_hack/" }
244244

245245
## Build dependencies
246246
criterion = "0.5.1"
247-
rcgen = "0.12"
247+
rcgen = "0.13"
248248
rstest = "0.18"
249249
camino-tempfile = "1.0.2"
250250
tonic-build = "0.12"

Diff for: ‎Dockerfile.build-tools

+3-3
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ RUN curl -sL "https://github.com/peak/s5cmd/releases/download/v${S5CMD_VERSION}/
7272
&& mv s5cmd /usr/local/bin/s5cmd
7373

7474
# LLVM
75-
ENV LLVM_VERSION=18
75+
ENV LLVM_VERSION=19
7676
RUN curl -fsSL 'https://apt.llvm.org/llvm-snapshot.gpg.key' | apt-key add - \
7777
&& echo "deb http://apt.llvm.org/${DEBIAN_VERSION}/ llvm-toolchain-${DEBIAN_VERSION}-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.stable.list \
7878
&& apt update \
@@ -99,7 +99,7 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "aws
9999
&& rm awscliv2.zip
100100

101101
# Mold: A Modern Linker
102-
ENV MOLD_VERSION=v2.33.0
102+
ENV MOLD_VERSION=v2.34.1
103103
RUN set -e \
104104
&& git clone https://github.com/rui314/mold.git \
105105
&& mkdir mold/build \
@@ -192,7 +192,7 @@ WORKDIR /home/nonroot
192192

193193
# Rust
194194
# Please keep the version of llvm (installed above) in sync with rust llvm (`rustc --version --verbose | grep LLVM`)
195-
ENV RUSTC_VERSION=1.81.0
195+
ENV RUSTC_VERSION=1.82.0
196196
ENV RUSTUP_HOME="/home/nonroot/.rustup"
197197
ENV PATH="/home/nonroot/.cargo/bin:${PATH}"
198198
ARG RUSTFILT_VERSION=0.2.1

Diff for: ‎compute/Dockerfile.compute-node

+2-2
Original file line numberDiff line numberDiff line change
@@ -975,8 +975,8 @@ ARG PG_VERSION
975975
RUN case "${PG_VERSION}" in "v17") \
976976
echo "pg_session_jwt does not yet have a release that supports pg17" && exit 0;; \
977977
esac && \
978-
wget https://github.com/neondatabase/pg_session_jwt/archive/5aee2625af38213650e1a07ae038fdc427250ee4.tar.gz -O pg_session_jwt.tar.gz && \
979-
echo "5d91b10bc1347d36cffc456cb87bec25047935d6503dc652ca046f04760828e7 pg_session_jwt.tar.gz" | sha256sum --check && \
978+
wget https://github.com/neondatabase/pg_session_jwt/archive/e642528f429dd3f5403845a50191b78d434b84a6.tar.gz -O pg_session_jwt.tar.gz && \
979+
echo "1a69210703cc91224785e59a0a67562dd9eed9a0914ac84b11447582ca0d5b93 pg_session_jwt.tar.gz" | sha256sum --check && \
980980
mkdir pg_session_jwt-src && cd pg_session_jwt-src && tar xzf ../pg_session_jwt.tar.gz --strip-components=1 -C . && \
981981
sed -i 's/pgrx = "=0.11.3"/pgrx = { version = "=0.11.3", features = [ "unsafe-postgres" ] }/g' Cargo.toml && \
982982
cargo pgrx install --release

Diff for: ‎compute/Makefile

+2
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,14 @@ neon_collector_autoscaling.yml: $(jsonnet_files)
2020
sql_exporter.yml: $(jsonnet_files)
2121
JSONNET_PATH=etc jsonnet \
2222
--output-file etc/$@ \
23+
--tla-str collector_name=neon_collector \
2324
--tla-str collector_file=neon_collector.yml \
2425
etc/sql_exporter.jsonnet
2526

2627
sql_exporter_autoscaling.yml: $(jsonnet_files)
2728
JSONNET_PATH=etc jsonnet \
2829
--output-file etc/$@ \
30+
--tla-str collector_name=neon_collector_autoscaling \
2931
--tla-str collector_file=neon_collector_autoscaling.yml \
3032
--tla-str application_name=sql_exporter_autoscaling \
3133
etc/sql_exporter.jsonnet

Diff for: ‎compute/etc/sql_exporter.jsonnet

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
function(collector_file, application_name='sql_exporter') {
1+
function(collector_name, collector_file, application_name='sql_exporter') {
22
// Configuration for sql_exporter for autoscaling-agent
33
// Global defaults.
44
global: {
@@ -28,7 +28,7 @@ function(collector_file, application_name='sql_exporter') {
2828
// Collectors (referenced by name) to execute on the target.
2929
// Glob patterns are supported (see <https://pkg.go.dev/path/filepath#Match> for syntax).
3030
collectors: [
31-
'neon_collector',
31+
collector_name,
3232
],
3333
},
3434

Diff for: ‎compute_tools/src/compute.rs

+100-23
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ use std::time::Instant;
1515

1616
use anyhow::{Context, Result};
1717
use chrono::{DateTime, Utc};
18+
use compute_api::spec::PgIdent;
1819
use futures::future::join_all;
1920
use futures::stream::FuturesUnordered;
2021
use futures::StreamExt;
@@ -25,15 +26,17 @@ use tracing::{debug, error, info, instrument, warn};
2526
use utils::id::{TenantId, TimelineId};
2627
use utils::lsn::Lsn;
2728

29+
use compute_api::privilege::Privilege;
2830
use compute_api::responses::{ComputeMetrics, ComputeStatus};
29-
use compute_api::spec::{ComputeFeature, ComputeMode, ComputeSpec};
31+
use compute_api::spec::{ComputeFeature, ComputeMode, ComputeSpec, ExtVersion};
3032
use utils::measured_stream::MeasuredReader;
3133

3234
use nix::sys::signal::{kill, Signal};
3335

3436
use remote_storage::{DownloadError, RemotePath};
3537

3638
use crate::checker::create_availability_check_data;
39+
use crate::installed_extensions::get_installed_extensions_sync;
3740
use crate::local_proxy;
3841
use crate::logger::inlinify;
3942
use crate::pg_helpers::*;
@@ -1121,6 +1124,11 @@ impl ComputeNode {
11211124
self.pg_reload_conf()?;
11221125
}
11231126
self.post_apply_config()?;
1127+
1128+
let connstr = self.connstr.clone();
1129+
thread::spawn(move || {
1130+
get_installed_extensions_sync(connstr).context("get_installed_extensions")
1131+
});
11241132
}
11251133

11261134
let startup_end_time = Utc::now();
@@ -1367,6 +1375,97 @@ LIMIT 100",
13671375
download_size
13681376
}
13691377

1378+
pub async fn set_role_grants(
1379+
&self,
1380+
db_name: &PgIdent,
1381+
schema_name: &PgIdent,
1382+
privileges: &[Privilege],
1383+
role_name: &PgIdent,
1384+
) -> Result<()> {
1385+
use tokio_postgres::config::Config;
1386+
use tokio_postgres::NoTls;
1387+
1388+
let mut conf = Config::from_str(self.connstr.as_str()).unwrap();
1389+
conf.dbname(db_name);
1390+
1391+
let (db_client, conn) = conf
1392+
.connect(NoTls)
1393+
.await
1394+
.context("Failed to connect to the database")?;
1395+
tokio::spawn(conn);
1396+
1397+
// TODO: support other types of grants apart from schemas?
1398+
let query = format!(
1399+
"GRANT {} ON SCHEMA {} TO {}",
1400+
privileges
1401+
.iter()
1402+
// should not be quoted as it's part of the command.
1403+
// is already sanitized so it's ok
1404+
.map(|p| p.as_str())
1405+
.collect::<Vec<&'static str>>()
1406+
.join(", "),
1407+
// quote the schema and role name as identifiers to sanitize them.
1408+
schema_name.pg_quote(),
1409+
role_name.pg_quote(),
1410+
);
1411+
db_client
1412+
.simple_query(&query)
1413+
.await
1414+
.with_context(|| format!("Failed to execute query: {}", query))?;
1415+
1416+
Ok(())
1417+
}
1418+
1419+
pub async fn install_extension(
1420+
&self,
1421+
ext_name: &PgIdent,
1422+
db_name: &PgIdent,
1423+
ext_version: ExtVersion,
1424+
) -> Result<ExtVersion> {
1425+
use tokio_postgres::config::Config;
1426+
use tokio_postgres::NoTls;
1427+
1428+
let mut conf = Config::from_str(self.connstr.as_str()).unwrap();
1429+
conf.dbname(db_name);
1430+
1431+
let (db_client, conn) = conf
1432+
.connect(NoTls)
1433+
.await
1434+
.context("Failed to connect to the database")?;
1435+
tokio::spawn(conn);
1436+
1437+
let version_query = "SELECT extversion FROM pg_extension WHERE extname = $1";
1438+
let version: Option<ExtVersion> = db_client
1439+
.query_opt(version_query, &[&ext_name])
1440+
.await
1441+
.with_context(|| format!("Failed to execute query: {}", version_query))?
1442+
.map(|row| row.get(0));
1443+
1444+
// sanitize the inputs as postgres idents.
1445+
let ext_name: String = ext_name.pg_quote();
1446+
let quoted_version: String = ext_version.pg_quote();
1447+
1448+
if let Some(installed_version) = version {
1449+
if installed_version == ext_version {
1450+
return Ok(installed_version);
1451+
}
1452+
let query = format!("ALTER EXTENSION {ext_name} UPDATE TO {quoted_version}");
1453+
db_client
1454+
.simple_query(&query)
1455+
.await
1456+
.with_context(|| format!("Failed to execute query: {}", query))?;
1457+
} else {
1458+
let query =
1459+
format!("CREATE EXTENSION IF NOT EXISTS {ext_name} WITH VERSION {quoted_version}");
1460+
db_client
1461+
.simple_query(&query)
1462+
.await
1463+
.with_context(|| format!("Failed to execute query: {}", query))?;
1464+
}
1465+
1466+
Ok(ext_version)
1467+
}
1468+
13701469
#[tokio::main]
13711470
pub async fn prepare_preload_libraries(
13721471
&self,
@@ -1484,28 +1583,6 @@ LIMIT 100",
14841583
info!("Pageserver config changed");
14851584
}
14861585
}
1487-
1488-
// Gather info about installed extensions
1489-
pub fn get_installed_extensions(&self) -> Result<()> {
1490-
let connstr = self.connstr.clone();
1491-
1492-
let rt = tokio::runtime::Builder::new_current_thread()
1493-
.enable_all()
1494-
.build()
1495-
.expect("failed to create runtime");
1496-
let result = rt
1497-
.block_on(crate::installed_extensions::get_installed_extensions(
1498-
connstr,
1499-
))
1500-
.expect("failed to get installed extensions");
1501-
1502-
info!(
1503-
"{}",
1504-
serde_json::to_string(&result).expect("failed to serialize extensions list")
1505-
);
1506-
1507-
Ok(())
1508-
}
15091586
}
15101587

15111588
pub fn forward_termination_signal() {

Diff for: ‎compute_tools/src/extension_server.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ pub fn get_pg_version(pgbin: &str) -> String {
107107
// pg_config --version returns a (platform specific) human readable string
108108
// such as "PostgreSQL 15.4". We parse this to v14/v15/v16 etc.
109109
let human_version = get_pg_config("--version", pgbin);
110-
return parse_pg_version(&human_version).to_string();
110+
parse_pg_version(&human_version).to_string()
111111
}
112112

113113
fn parse_pg_version(human_version: &str) -> &str {

Diff for: ‎compute_tools/src/http/api.rs

+79-2
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,11 @@ use crate::catalog::SchemaDumpError;
99
use crate::catalog::{get_database_schema, get_dbs_and_roles};
1010
use crate::compute::forward_termination_signal;
1111
use crate::compute::{ComputeNode, ComputeState, ParsedSpec};
12-
use compute_api::requests::ConfigurationRequest;
13-
use compute_api::responses::{ComputeStatus, ComputeStatusResponse, GenericAPIError};
12+
use compute_api::requests::{ConfigurationRequest, ExtensionInstallRequest, SetRoleGrantsRequest};
13+
use compute_api::responses::{
14+
ComputeStatus, ComputeStatusResponse, ExtensionInstallResult, GenericAPIError,
15+
SetRoleGrantsResponse,
16+
};
1417

1518
use anyhow::Result;
1619
use hyper::header::CONTENT_TYPE;
@@ -98,6 +101,38 @@ async fn routes(req: Request<Body>, compute: &Arc<ComputeNode>) -> Response<Body
98101
}
99102
}
100103

104+
(&Method::POST, "/extensions") => {
105+
info!("serving /extensions POST request");
106+
let status = compute.get_status();
107+
if status != ComputeStatus::Running {
108+
let msg = format!(
109+
"invalid compute status for extensions request: {:?}",
110+
status
111+
);
112+
error!(msg);
113+
return render_json_error(&msg, StatusCode::PRECONDITION_FAILED);
114+
}
115+
116+
let request = hyper::body::to_bytes(req.into_body()).await.unwrap();
117+
let request = serde_json::from_slice::<ExtensionInstallRequest>(&request).unwrap();
118+
let res = compute
119+
.install_extension(&request.extension, &request.database, request.version)
120+
.await;
121+
match res {
122+
Ok(version) => render_json(Body::from(
123+
serde_json::to_string(&ExtensionInstallResult {
124+
extension: request.extension,
125+
version,
126+
})
127+
.unwrap(),
128+
)),
129+
Err(e) => {
130+
error!("install_extension failed: {}", e);
131+
render_json_error(&e.to_string(), StatusCode::INTERNAL_SERVER_ERROR)
132+
}
133+
}
134+
}
135+
101136
(&Method::GET, "/info") => {
102137
let num_cpus = num_cpus::get_physical();
103138
info!("serving /info GET request. num_cpus: {}", num_cpus);
@@ -165,6 +200,48 @@ async fn routes(req: Request<Body>, compute: &Arc<ComputeNode>) -> Response<Body
165200
}
166201
}
167202

203+
(&Method::POST, "/grants") => {
204+
info!("serving /grants POST request");
205+
let status = compute.get_status();
206+
if status != ComputeStatus::Running {
207+
let msg = format!(
208+
"invalid compute status for set_role_grants request: {:?}",
209+
status
210+
);
211+
error!(msg);
212+
return render_json_error(&msg, StatusCode::PRECONDITION_FAILED);
213+
}
214+
215+
let request = hyper::body::to_bytes(req.into_body()).await.unwrap();
216+
let request = serde_json::from_slice::<SetRoleGrantsRequest>(&request).unwrap();
217+
218+
let res = compute
219+
.set_role_grants(
220+
&request.database,
221+
&request.schema,
222+
&request.privileges,
223+
&request.role,
224+
)
225+
.await;
226+
match res {
227+
Ok(()) => render_json(Body::from(
228+
serde_json::to_string(&SetRoleGrantsResponse {
229+
database: request.database,
230+
schema: request.schema,
231+
role: request.role,
232+
privileges: request.privileges,
233+
})
234+
.unwrap(),
235+
)),
236+
Err(e) => render_json_error(
237+
&format!("could not grant role privileges to the schema: {e}"),
238+
// TODO: can we filter on role/schema not found errors
239+
// and return appropriate error code?
240+
StatusCode::INTERNAL_SERVER_ERROR,
241+
),
242+
}
243+
}
244+
168245
// get the list of installed extensions
169246
// currently only used in python tests
170247
// TODO: call it from cplane

0 commit comments

Comments
 (0)