Skip to content

Commit 7366f84

Browse files
gurinderunahsifolex
authored
feat(cores)!: core manager [fixes NET-739 NET-755 NET-740] (#2069)
* feat(cores): cpu_range set * feat(cores): core manager * feat(cores): core manager * small improvements * small improvements * small improvements * small improvements * small improvements * small improvements * small improvements * small improvements * small improvements * small improvements * small improvements * fix dep * fix dep * fix dep * small improvements * small improvements * small improvements * pin system threads * pin worker threads * pin worker threads * compilation fix * fix tests * Fix dockerfile * Fix dockerfile * Fixes * Fix tests * fix order * fix order * Fix explore_services_fixed_heavy * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * Review fixes * switch to cpu-utils * fix clippy * use CUID * fix hex * fix hex * fix clippy * fix(deps): decider 0.6.0 --------- Co-authored-by: Anatolios Laskaris <[email protected]> Co-authored-by: folex <[email protected]>
1 parent 26cdc6e commit 7366f84

File tree

39 files changed

+1593
-148
lines changed

39 files changed

+1593
-148
lines changed

Cargo.lock

+287-40
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

+11-6
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ members = [
4646
"crates/chain-listener",
4747
"crates/hex-utils",
4848
"crates/chain-data",
49-
"crates/types"
50-
]
49+
"crates/types",
50+
"crates/core-manager"]
5151
exclude = [
5252
"nox/tests/tetraplets",
5353
]
@@ -94,10 +94,11 @@ particle-execution = { path = "particle-execution" }
9494
system-services = { path = "crates/system-services" }
9595
health = { path = "crates/health" }
9696
subnet-resolver = { path = "crates/subnet-resolver" }
97-
hex-utils = { path = "crates/hex-utils"}
98-
chain-data = { path = "crates/chain-data"}
99-
chain-listener = { path = "crates/chain-listener"}
100-
types = { path = "crates/types"}
97+
hex-utils = { path = "crates/hex-utils" }
98+
chain-data = { path = "crates/chain-data" }
99+
chain-listener = { path = "crates/chain-listener" }
100+
types = { path = "crates/types" }
101+
core-manager = { path = "crates/core-manager" }
101102

102103
# spell
103104
fluence-spell-dtos = "=0.7.0"
@@ -131,6 +132,7 @@ bs58 = "0.5.0"
131132
fluence-keypair = "0.10.4"
132133
parking_lot = "0.12.1"
133134
tokio = "1.36.0"
135+
async-trait = "0.1.77"
134136
tokio-stream = "0.1.14"
135137
tokio-util = "0.7.10"
136138
uuid = { version = "1.7.0", features = ["v4"] }
@@ -160,6 +162,9 @@ jsonrpsee = "0.21.0"
160162
blake3 = "1.5.0"
161163
rand = "0.8.5"
162164
futures-util = "0.3.30"
165+
num_cpus = "1.16.0"
166+
enum_dispatch = "0.3.12"
167+
serde_with = "3.6.0"
163168

164169
# Enable a small amount of optimization in debug mode
165170
[profile.dev]

aquamarine/Cargo.toml

+2-2
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,10 @@ humantime = "2.1.0"
4040
anyhow = "1.0.79"
4141
eyre = { workspace = true }
4242
bytesize = "1.3.0"
43-
async-trait = "0.1.77"
43+
async-trait = { workspace = true }
4444
health = { workspace = true }
4545
config = { version = "0.13.4", features = [] }
46-
enum_dispatch = "0.3.12"
46+
enum_dispatch = { workspace = true }
4747

4848
[dev-dependencies]
4949
tempfile = { workspace = true }

aquamarine/src/plumber.rs

+4-2
Original file line numberDiff line numberDiff line change
@@ -436,7 +436,7 @@ mod tests {
436436
use fluence_keypair::KeyPair;
437437
use fluence_libp2p::RandomPeerId;
438438
use futures::task::noop_waker_ref;
439-
use workers::{KeyStorage, PeerScopes, Workers};
439+
use workers::{DummyCoreManager, KeyStorage, PeerScopes, Workers};
440440

441441
use particle_args::Args;
442442
use particle_execution::{FunctionOutcome, ParticleFunction, ParticleParams, ServiceFunction};
@@ -537,14 +537,16 @@ mod tests {
537537

538538
let key_storage = Arc::new(key_storage);
539539

540+
let core_manager = Arc::new(DummyCoreManager::default().into());
541+
540542
let scope = PeerScopes::new(
541543
root_key_pair.get_peer_id(),
542544
RandomPeerId::random(),
543545
RandomPeerId::random(),
544546
key_storage.clone(),
545547
);
546548

547-
let workers = Workers::from_path(workers_path.clone(), key_storage.clone())
549+
let workers = Workers::from_path(workers_path.clone(), key_storage.clone(), core_manager)
548550
.await
549551
.expect("Could not load worker registry");
550552

crates/connected-client/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ libp2p = { workspace = true, features = ["identify"] }
1515
libp2p-swarm = { workspace = true }
1616
tokio = { workspace = true }
1717
futures = { workspace = true }
18-
serde = { version = "1.0.196", features = ["derive"] }
18+
serde = { workspace = true, features = ["derive"] }
1919
serde_json = { workspace = true }
2020
log = { workspace = true }
2121
derivative = { workspace = true }

crates/core-manager/Cargo.toml

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
[package]
2+
name = "core-manager"
3+
version = "0.1.0"
4+
edition = "2021"
5+
6+
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
7+
8+
[dependencies]
9+
fxhash = "0.2.1"
10+
range-set-blaze = "0.1.14"
11+
cpu-utils = { git = "https://github.com/fluencelabs/capacity-commitment-prover/", branch = "main" }
12+
ccp-shared = { git = "https://github.com/fluencelabs/capacity-commitment-prover/", branch = "main" }
13+
multimap = { version = "0.10.0", features = ["serde"] }
14+
bimap = { version = "0.6.3", features = ["serde"] }
15+
toml = { version = "0.8.10" }
16+
newtype_derive = "0.1.6"
17+
18+
tokio = { workspace = true, features = ["fs", "rt", "sync", "macros", "tracing"] }
19+
async-trait.workspace = true
20+
enum_dispatch.workspace = true
21+
num_cpus.workspace = true
22+
parking_lot.workspace = true
23+
thiserror.workspace = true
24+
serde = { workspace = true, features = ["derive"] }
25+
types.workspace = true
26+
tracing.workspace = true
27+
serde_with.workspace = true
28+
29+
30+
[dev-dependencies]
31+
tempfile = { workspace = true }
32+
hex.workspace = true

crates/core-manager/src/core_range.rs

+266
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,266 @@
1+
use range_set_blaze::RangeSetBlaze;
2+
use serde::{Deserialize, Deserializer, Serialize, Serializer};
3+
use std::fmt::{Debug, Display, Formatter};
4+
use std::str::FromStr;
5+
use thiserror::Error;
6+
7+
#[derive(Clone, PartialEq)]
8+
pub struct CoreRange(pub(crate) RangeSetBlaze<usize>);
9+
10+
impl Debug for CoreRange {
11+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
12+
f.write_str(self.0.to_string().as_str())
13+
}
14+
}
15+
16+
impl Default for CoreRange {
17+
fn default() -> Self {
18+
CoreRange(RangeSetBlaze::from_iter(0..num_cpus::get_physical()))
19+
}
20+
}
21+
22+
impl TryFrom<&[usize]> for CoreRange {
23+
type Error = ParseError;
24+
25+
fn try_from(value: &[usize]) -> Result<Self, Self::Error> {
26+
if value.is_empty() {
27+
return Err(ParseError::EmptyRange);
28+
}
29+
Ok(CoreRange(RangeSetBlaze::from_iter(value)))
30+
}
31+
}
32+
33+
impl FromStr for CoreRange {
34+
type Err = ParseError;
35+
36+
/// Parse CoreRange from string like "1,2-30,31"
37+
fn from_str(s: &str) -> Result<Self, Self::Err> {
38+
let mut result: RangeSetBlaze<usize> = RangeSetBlaze::new();
39+
let trimmed = s.trim();
40+
if trimmed.is_empty() {
41+
return Err(ParseError::EmptyRange);
42+
}
43+
44+
for part in trimmed.split(',') {
45+
let trimmed = part.trim();
46+
// either a single number or a dash range
47+
let range: Vec<&str> = trimmed.split('-').collect();
48+
match range[..] {
49+
[l, r] => {
50+
let l = l
51+
.parse::<usize>()
52+
.map_err(|_| ParseError::WrongRangeFormat {
53+
raw_str: trimmed.to_string(),
54+
})?;
55+
let r = r
56+
.parse::<usize>()
57+
.map_err(|_| ParseError::WrongRangeFormat {
58+
raw_str: trimmed.to_string(),
59+
})?;
60+
// insert the inclusive range
61+
result.ranges_insert(l..=r);
62+
}
63+
[value] => {
64+
let value =
65+
value
66+
.parse::<usize>()
67+
.map_err(|_| ParseError::WrongRangeFormat {
68+
raw_str: trimmed.to_string(),
69+
})?;
70+
result.insert(value);
71+
}
72+
_ => {
73+
return Err(ParseError::WrongRangeFormat {
74+
raw_str: trimmed.to_string(),
75+
});
76+
}
77+
}
78+
}
79+
80+
Ok(CoreRange(result))
81+
}
82+
}
83+
84+
impl Display for CoreRange {
85+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
86+
for (index, range) in self.0.ranges().enumerate() {
87+
if index != 0 {
88+
write!(f, ",")?;
89+
};
90+
let start = range.start();
91+
let end = range.end();
92+
if start == end {
93+
write!(f, "{}", start)?;
94+
} else {
95+
write!(f, "{}-{}", start, end)?;
96+
}
97+
}
98+
Ok(())
99+
}
100+
}
101+
102+
impl Serialize for CoreRange {
103+
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
104+
where
105+
S: Serializer,
106+
{
107+
serializer.serialize_str(self.to_string().as_str())
108+
}
109+
}
110+
111+
impl<'de> Deserialize<'de> for CoreRange {
112+
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
113+
where
114+
D: Deserializer<'de>,
115+
{
116+
let raw_str = String::deserialize(deserializer)?;
117+
CoreRange::from_str(raw_str.as_str()).map_err(|e| {
118+
serde::de::Error::custom(format!("failed to deserialize core range {raw_str} {e:?}"))
119+
})
120+
}
121+
}
122+
123+
#[derive(Debug, Error, PartialEq)]
124+
pub enum ParseError {
125+
#[error("Range can't be an empty")]
126+
EmptyRange,
127+
#[error("Failed to parse: {raw_str}")]
128+
WrongRangeFormat { raw_str: String },
129+
}
130+
131+
#[cfg(test)]
132+
mod tests {
133+
use crate::core_range::{CoreRange, ParseError};
134+
135+
#[test]
136+
fn range_parsing_test() {
137+
let core_range: CoreRange = "0-2".parse().unwrap();
138+
assert!(core_range.0.contains(0));
139+
assert!(core_range.0.contains(1));
140+
assert!(core_range.0.contains(2));
141+
assert!(!core_range.0.contains(3));
142+
}
143+
144+
#[test]
145+
fn values_parsing_test() {
146+
let core_range: CoreRange = "0,1,3".parse().unwrap();
147+
assert!(core_range.0.contains(0));
148+
assert!(core_range.0.contains(1));
149+
assert!(!core_range.0.contains(2));
150+
assert!(core_range.0.contains(3));
151+
}
152+
153+
#[test]
154+
fn wrong_parsing_test() {
155+
let result = "aaaa".parse::<CoreRange>();
156+
assert!(result.is_err());
157+
if let Err(err) = result {
158+
assert_eq!(
159+
err,
160+
ParseError::WrongRangeFormat {
161+
raw_str: "aaaa".to_string()
162+
}
163+
);
164+
assert_eq!(err.to_string(), "Failed to parse: aaaa")
165+
}
166+
}
167+
168+
#[test]
169+
fn wrong_parsing_test_2() {
170+
let result = "1-a".parse::<CoreRange>();
171+
assert!(result.is_err());
172+
if let Err(err) = result {
173+
assert_eq!(
174+
err,
175+
ParseError::WrongRangeFormat {
176+
raw_str: "1-a".to_string()
177+
}
178+
);
179+
assert_eq!(err.to_string(), "Failed to parse: 1-a")
180+
}
181+
}
182+
183+
#[test]
184+
fn wrong_parsing_test_3() {
185+
let result = "a-1".parse::<CoreRange>();
186+
assert!(result.is_err());
187+
if let Err(err) = result {
188+
assert_eq!(
189+
err,
190+
ParseError::WrongRangeFormat {
191+
raw_str: "a-1".to_string()
192+
}
193+
);
194+
assert_eq!(err.to_string(), "Failed to parse: a-1")
195+
}
196+
}
197+
198+
#[test]
199+
fn wrong_parsing_test_4() {
200+
let result = "a-1-2,3".parse::<CoreRange>();
201+
assert!(result.is_err());
202+
if let Err(err) = result {
203+
assert_eq!(
204+
err,
205+
ParseError::WrongRangeFormat {
206+
raw_str: "a-1-2".to_string()
207+
}
208+
);
209+
assert_eq!(err.to_string(), "Failed to parse: a-1-2")
210+
}
211+
}
212+
213+
#[test]
214+
fn empty_parsing_test_3() {
215+
let result = "".parse::<CoreRange>();
216+
assert!(result.is_err());
217+
if let Err(err) = result {
218+
assert_eq!(err, ParseError::EmptyRange);
219+
}
220+
}
221+
222+
#[test]
223+
fn slice_convert() {
224+
let core_range = CoreRange::try_from(&vec![1, 2, 3, 10][..]).unwrap();
225+
226+
assert!(!core_range.0.contains(0));
227+
assert!(core_range.0.contains(1));
228+
assert!(core_range.0.contains(2));
229+
assert!(core_range.0.contains(3));
230+
assert!(core_range.0.contains(10));
231+
assert!(!core_range.0.contains(11));
232+
}
233+
234+
#[test]
235+
fn empty_slice_convert() {
236+
let result = CoreRange::try_from(&vec![][..]);
237+
238+
assert!(result.is_err());
239+
if let Err(err) = result {
240+
assert_eq!(err, ParseError::EmptyRange);
241+
assert_eq!(err.to_string(), "Range can't be an empty")
242+
}
243+
}
244+
245+
#[test]
246+
fn last() {
247+
let core_range: CoreRange = "0-2".parse().unwrap();
248+
assert!(core_range.0.contains(0));
249+
assert!(core_range.0.contains(1));
250+
assert!(core_range.0.contains(2));
251+
assert!(!core_range.0.contains(3));
252+
}
253+
254+
#[test]
255+
fn compare_ranges() {
256+
let core_range_1: CoreRange = "0-2".parse().unwrap();
257+
let core_range_2: CoreRange = "0,1,2".parse().unwrap();
258+
assert_eq!(core_range_1, core_range_2);
259+
}
260+
261+
#[test]
262+
fn fmt() {
263+
let core_range_1: CoreRange = "0-2,5,7-9".parse().unwrap();
264+
assert_eq!(format!("{}", core_range_1), "0-2,5,7-9");
265+
}
266+
}

0 commit comments

Comments
 (0)