Skip to content

Commit 81af21f

Browse files
fmassotfmassot
authored
Disabled toggling of ingest api source and protect againt toggle/delete on server side. (#2914)
1 parent add9820 commit 81af21f

File tree

4 files changed

+65
-181
lines changed

4 files changed

+65
-181
lines changed

‎quickwit/quickwit-cli/src/source.rs

+1-124
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,7 @@ use colored::Colorize;
2626
use itertools::Itertools;
2727
use quickwit_common::uri::Uri;
2828
use quickwit_common::GREEN_COLOR;
29-
use quickwit_config::{
30-
validate_identifier, ConfigFormat, SourceConfig, CLI_INGEST_SOURCE_ID, INGEST_API_SOURCE_ID,
31-
};
29+
use quickwit_config::{validate_identifier, ConfigFormat, SourceConfig};
3230
use quickwit_metastore::checkpoint::SourceCheckpoint;
3331
use quickwit_rest_client::rest_client::{QuickwitClient, Transport};
3432
use quickwit_storage::load_file;
@@ -143,13 +141,6 @@ pub struct ToggleSourceArgs {
143141
pub enable: bool,
144142
}
145143

146-
#[derive(Debug, Eq, PartialEq)]
147-
pub struct ToggleIngestApiArgs {
148-
pub cluster_endpoint: Url,
149-
pub index_id: String,
150-
pub enable: bool,
151-
}
152-
153144
#[derive(Debug, Eq, PartialEq)]
154145
pub struct DeleteSourceArgs {
155146
pub cluster_endpoint: Url,
@@ -183,7 +174,6 @@ pub struct ResetCheckpointArgs {
183174
pub enum SourceCliCommand {
184175
CreateSource(CreateSourceArgs),
185176
ToggleSource(ToggleSourceArgs),
186-
ToggleIngestApi(ToggleIngestApiArgs),
187177
DeleteSource(DeleteSourceArgs),
188178
DescribeSource(DescribeSourceArgs),
189179
ListSources(ListSourcesArgs),
@@ -195,7 +185,6 @@ impl SourceCliCommand {
195185
match self {
196186
Self::CreateSource(args) => create_source_cli(args).await,
197187
Self::ToggleSource(args) => toggle_source_cli(args).await,
198-
Self::ToggleIngestApi(args) => toggle_ingest_api_cli(args).await,
199188
Self::DeleteSource(args) => delete_source_cli(args).await,
200189
Self::DescribeSource(args) => describe_source_cli(args).await,
201190
Self::ListSources(args) => list_sources_cli(args).await,
@@ -215,9 +204,6 @@ impl SourceCliCommand {
215204
"disable" => {
216205
Self::parse_toggle_source_args(subcommand, submatches).map(Self::ToggleSource)
217206
}
218-
"ingest-api" => {
219-
Self::parse_toggle_ingest_api_args(submatches).map(Self::ToggleIngestApi)
220-
}
221207
"delete" => Self::parse_delete_args(submatches).map(Self::DeleteSource),
222208
"describe" => Self::parse_describe_args(submatches).map(Self::DescribeSource),
223209
"list" => Self::parse_list_args(submatches).map(Self::ListSources),
@@ -273,23 +259,6 @@ impl SourceCliCommand {
273259
})
274260
}
275261

276-
fn parse_toggle_ingest_api_args(matches: &ArgMatches) -> anyhow::Result<ToggleIngestApiArgs> {
277-
let cluster_endpoint = matches
278-
.value_of("endpoint")
279-
.map(Url::from_str)
280-
.expect("`endpoint` is a required arg.")?;
281-
let index_id = matches
282-
.value_of("index")
283-
.expect("`index` is a required arg.")
284-
.to_string();
285-
let enable = matches.is_present("enable");
286-
Ok(ToggleIngestApiArgs {
287-
cluster_endpoint,
288-
index_id,
289-
enable,
290-
})
291-
}
292-
293262
fn parse_delete_args(matches: &ArgMatches) -> anyhow::Result<DeleteSourceArgs> {
294263
let cluster_endpoint = matches
295264
.value_of("endpoint")
@@ -388,13 +357,6 @@ async fn create_source_cli(args: CreateSourceArgs) -> anyhow::Result<()> {
388357
async fn toggle_source_cli(args: ToggleSourceArgs) -> anyhow::Result<()> {
389358
debug!(args=?args, "toggle-source");
390359
println!("❯ Toggling source...");
391-
if args.source_id == CLI_INGEST_SOURCE_ID {
392-
bail!(
393-
"Source `{}` is managed by Quickwit, you cannot enable or disable a source managed by \
394-
Quickwit.",
395-
args.source_id
396-
);
397-
}
398360
let transport = Transport::new(args.cluster_endpoint);
399361
let qw_client = QuickwitClient::new(transport);
400362
qw_client
@@ -412,37 +374,9 @@ async fn toggle_source_cli(args: ToggleSourceArgs) -> anyhow::Result<()> {
412374
Ok(())
413375
}
414376

415-
pub async fn toggle_ingest_api_cli(args: ToggleIngestApiArgs) -> anyhow::Result<()> {
416-
debug!(args=?args, "toggle-ingest-api");
417-
println!(
418-
"❯ {}abling ingest API...",
419-
if args.enable { "En" } else { "Dis" }
420-
);
421-
let transport = Transport::new(args.cluster_endpoint);
422-
let qw_client = QuickwitClient::new(transport);
423-
qw_client
424-
.sources(&args.index_id)
425-
.toggle(INGEST_API_SOURCE_ID, args.enable)
426-
.await
427-
.context("Failed to update source")?;
428-
let toggled_state_name = if args.enable { "enabled" } else { "disabled" };
429-
println!(
430-
"{} Source successfully {}.",
431-
toggled_state_name,
432-
"✔".color(GREEN_COLOR)
433-
);
434-
Ok(())
435-
}
436-
437377
async fn delete_source_cli(args: DeleteSourceArgs) -> anyhow::Result<()> {
438378
debug!(args=?args, "delete-source");
439379
println!("❯ Deleting source...");
440-
if args.source_id == INGEST_API_SOURCE_ID || args.source_id == CLI_INGEST_SOURCE_ID {
441-
bail!(
442-
"Source `{}` is managed by Quickwit, you cannot delete a source managed by Quickwit.",
443-
args.source_id
444-
);
445-
}
446380
validate_identifier("Source ID", &args.source_id)?;
447381

448382
if !args.assume_yes {
@@ -735,63 +669,6 @@ mod tests {
735669
}
736670
}
737671

738-
#[test]
739-
fn test_parse_toggle_ingest_api_args() {
740-
{
741-
let app = build_cli().no_binary_name(true);
742-
let matches = app
743-
.try_get_matches_from(vec![
744-
"source",
745-
"ingest-api",
746-
"--endpoint",
747-
"https://quickwit-cluster.io",
748-
"--index",
749-
"foo",
750-
"--enable",
751-
])
752-
.unwrap();
753-
let command = CliCommand::parse_cli_args(&matches).unwrap();
754-
let expected_command =
755-
CliCommand::Source(SourceCliCommand::ToggleIngestApi(ToggleIngestApiArgs {
756-
cluster_endpoint: Url::from_str("https://quickwit-cluster.io").unwrap(),
757-
index_id: "foo".to_string(),
758-
enable: true,
759-
}));
760-
assert_eq!(command, expected_command);
761-
}
762-
{
763-
let app = build_cli().no_binary_name(true);
764-
let matches = app
765-
.try_get_matches_from(vec!["source", "ingest-api", "--index", "foo", "--disable"])
766-
.unwrap();
767-
let command = CliCommand::parse_cli_args(&matches).unwrap();
768-
let expected_command =
769-
CliCommand::Source(SourceCliCommand::ToggleIngestApi(ToggleIngestApiArgs {
770-
cluster_endpoint: Url::from_str("http://127.0.0.1:7280").unwrap(),
771-
index_id: "foo".to_string(),
772-
enable: false,
773-
}));
774-
assert_eq!(command, expected_command);
775-
}
776-
{
777-
let app = build_cli().no_binary_name(true);
778-
let matches = app.try_get_matches_from(vec![
779-
"source",
780-
"ingest-api",
781-
"--index",
782-
"foo",
783-
"--enable",
784-
"--disable",
785-
]);
786-
assert!(matches.is_err());
787-
}
788-
{
789-
let app = build_cli().no_binary_name(true);
790-
let matches = app.try_get_matches_from(vec!["source", "ingest-api", "--index", "foo"]);
791-
assert!(matches.is_err());
792-
}
793-
}
794-
795672
#[test]
796673
fn test_parse_delete_source_args() {
797674
let app = build_cli().no_binary_name(true);

‎quickwit/quickwit-cli/tests/cli.rs

+1-55
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ use quickwit_cli::index::{
3434
SearchIndexArgs,
3535
};
3636
use quickwit_cli::service::RunCliCommand;
37-
use quickwit_cli::source::{toggle_ingest_api_cli, ToggleIngestApiArgs};
3837
use quickwit_cli::tool::{
3938
garbage_collect_index_cli, local_ingest_docs_cli, GarbageCollectIndexArgs, LocalIngestDocsArgs,
4039
};
@@ -43,7 +42,7 @@ use quickwit_common::rand::append_random_suffix;
4342
use quickwit_common::uri::Uri;
4443
use quickwit_common::ChecklistError;
4544
use quickwit_config::service::QuickwitService;
46-
use quickwit_config::{CLI_INGEST_SOURCE_ID, INGEST_API_SOURCE_ID};
45+
use quickwit_config::CLI_INGEST_SOURCE_ID;
4746
use quickwit_metastore::{quickwit_metastore_uri_resolver, Metastore, MetastoreError, SplitState};
4847
use serde_json::{json, Number, Value};
4948
use tokio::time::{sleep, Duration};
@@ -157,59 +156,6 @@ fn test_cmd_create_with_ill_formed_command() {
157156
);
158157
}
159158

160-
#[tokio::test]
161-
async fn test_cmd_toggle_ingest_api_source() {
162-
quickwit_common::setup_logging_for_tests();
163-
let index_id = append_random_suffix("test-create-cmd");
164-
let test_env = create_test_env(index_id.clone(), TestStorageType::LocalFileSystem).unwrap();
165-
test_env.start_server().await.unwrap();
166-
create_logs_index(&test_env).await.unwrap();
167-
168-
// Disable
169-
let toggle_args = ToggleIngestApiArgs {
170-
cluster_endpoint: test_env.cluster_endpoint.clone(),
171-
enable: false,
172-
index_id: index_id.to_string(),
173-
};
174-
toggle_ingest_api_cli(toggle_args).await.unwrap();
175-
let index_metadata = test_env
176-
.metastore()
177-
.await
178-
.unwrap()
179-
.index_metadata(&index_id)
180-
.await
181-
.unwrap();
182-
assert!(
183-
!index_metadata
184-
.sources
185-
.get(INGEST_API_SOURCE_ID)
186-
.unwrap()
187-
.enabled
188-
);
189-
190-
// Enable
191-
let toggle_args = ToggleIngestApiArgs {
192-
cluster_endpoint: test_env.cluster_endpoint.clone(),
193-
enable: true,
194-
index_id: index_id.to_string(),
195-
};
196-
toggle_ingest_api_cli(toggle_args).await.unwrap();
197-
let index_metadata = test_env
198-
.metastore()
199-
.await
200-
.unwrap()
201-
.index_metadata(&index_id)
202-
.await
203-
.unwrap();
204-
assert!(
205-
index_metadata
206-
.sources
207-
.get(INGEST_API_SOURCE_ID)
208-
.unwrap()
209-
.enabled
210-
);
211-
}
212-
213159
#[tokio::test]
214160
async fn test_cmd_ingest_on_non_existing_index() {
215161
let index_id = append_random_suffix("index-does-not-exist");

‎quickwit/quickwit-metastore/src/error.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ impl ServiceError for MetastoreError {
110110
fn status_code(&self) -> ServiceErrorCode {
111111
match self {
112112
Self::ConnectionError { .. } => ServiceErrorCode::Internal,
113-
Self::Forbidden { .. } => ServiceErrorCode::Internal,
113+
Self::Forbidden { .. } => ServiceErrorCode::MethodNotAllowed,
114114
Self::IncompatibleCheckpointDelta(_) => ServiceErrorCode::BadRequest,
115115
Self::IndexAlreadyExists { .. } => ServiceErrorCode::BadRequest,
116116
Self::IndexDoesNotExist { .. } => ServiceErrorCode::NotFound,

‎quickwit/quickwit-serve/src/index_api/rest_handler.rs

+62-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@ use bytes::Bytes;
2323
use hyper::header::CONTENT_TYPE;
2424
use quickwit_common::simple_list::{from_simple_list, to_simple_list};
2525
use quickwit_common::FileEntry;
26-
use quickwit_config::{ConfigFormat, QuickwitConfig, SourceConfig};
26+
use quickwit_config::{
27+
ConfigFormat, QuickwitConfig, SourceConfig, CLI_INGEST_SOURCE_ID, INGEST_API_SOURCE_ID,
28+
};
2729
use quickwit_core::{IndexService, IndexServiceError};
2830
use quickwit_metastore::{
2931
IndexMetadata, ListSplitsQuery, Metastore, MetastoreError, Split, SplitState,
@@ -548,6 +550,15 @@ async fn toggle_source(
548550
metastore: Arc<dyn Metastore>,
549551
) -> Result<(), MetastoreError> {
550552
info!(index_id = %index_id, source_id = %source_id, enable = toggle_source.enable, "toggle-source");
553+
metastore.index_exists(&index_id).await?;
554+
if [CLI_INGEST_SOURCE_ID, INGEST_API_SOURCE_ID].contains(&source_id.as_str()) {
555+
return Err(MetastoreError::Forbidden {
556+
message: format!(
557+
"Source `{source_id}` is managed by Quickwit, you cannot enable or disable a \
558+
source managed by Quickwit."
559+
),
560+
});
561+
}
551562
metastore
552563
.toggle_source(&index_id, &source_id, toggle_source.enable)
553564
.await
@@ -583,6 +594,15 @@ async fn delete_source(
583594
metastore: Arc<dyn Metastore>,
584595
) -> Result<(), MetastoreError> {
585596
info!(index_id = %index_id, source_id = %source_id, "delete-source");
597+
metastore.index_exists(&index_id).await?;
598+
if [INGEST_API_SOURCE_ID, CLI_INGEST_SOURCE_ID].contains(&source_id.as_str()) {
599+
return Err(MetastoreError::Forbidden {
600+
message: format!(
601+
"Source `{source_id}` is managed by Quickwit, you cannot delete a source managed \
602+
by Quickwit."
603+
),
604+
});
605+
}
586606
metastore.delete_source(&index_id, &source_id).await
587607
}
588608

@@ -1066,6 +1086,23 @@ mod tests {
10661086
let index_metadata = metastore.index_metadata("hdfs-logs").await.unwrap();
10671087
assert!(!index_metadata.sources.contains_key("file-source"));
10681088

1089+
// Check cannot delete source managed by Quickwit.
1090+
let resp = warp::test::request()
1091+
.path(format!("/indexes/hdfs-logs/sources/{INGEST_API_SOURCE_ID}").as_str())
1092+
.method("DELETE")
1093+
.body(&source_config_body)
1094+
.reply(&index_management_handler)
1095+
.await;
1096+
assert_eq!(resp.status(), 405);
1097+
1098+
let resp = warp::test::request()
1099+
.path(format!("/indexes/hdfs-logs/sources/{CLI_INGEST_SOURCE_ID}").as_str())
1100+
.method("DELETE")
1101+
.body(&source_config_body)
1102+
.reply(&index_management_handler)
1103+
.await;
1104+
assert_eq!(resp.status(), 405);
1105+
10691106
// Check get a non exising source returns 404.
10701107
let resp = warp::test::request()
10711108
.path("/indexes/hdfs-logs/sources/file-source")
@@ -1244,6 +1281,9 @@ mod tests {
12441281
"file:///path/to/index/quickwit-demo-index",
12451282
))
12461283
});
1284+
metastore
1285+
.expect_index_exists()
1286+
.return_once(|index_id: &str| Ok(index_id == "quickwit-demo-index"));
12471287
metastore
12481288
.expect_delete_source()
12491289
.return_once(|index_id, source_id| {
@@ -1305,6 +1345,10 @@ mod tests {
13051345
#[tokio::test]
13061346
async fn test_source_toggle() -> anyhow::Result<()> {
13071347
let mut metastore = MockMetastore::new();
1348+
metastore
1349+
.expect_index_exists()
1350+
.returning(|index_id| Ok(index_id == "quickwit-demo-index"))
1351+
.times(3);
13081352
metastore.expect_toggle_source().return_once(
13091353
|index_id: &str, source_id: &str, enable: bool| {
13101354
if index_id == "quickwit-demo-index" && source_id == "source-to-toggle" && enable {
@@ -1322,6 +1366,7 @@ mod tests {
13221366
Arc::new(QuickwitConfig::for_test()),
13231367
)
13241368
.recover(recover_fn);
1369+
// Check server returns 405 if sources root path is used.
13251370
let resp = warp::test::request()
13261371
.path("/indexes/quickwit-demo-index/sources/source-to-toggle")
13271372
.method("PUT")
@@ -1344,6 +1389,22 @@ mod tests {
13441389
.reply(&index_management_handler)
13451390
.await;
13461391
assert_eq!(resp.status(), 400);
1392+
// Check cannot toggle source managed by Quickwit.
1393+
let resp = warp::test::request()
1394+
.path(format!("/indexes/hdfs-logs/sources/{INGEST_API_SOURCE_ID}/toggle").as_str())
1395+
.method("PUT")
1396+
.body(r#"{"enable": true}"#)
1397+
.reply(&index_management_handler)
1398+
.await;
1399+
assert_eq!(resp.status(), 405);
1400+
1401+
let resp = warp::test::request()
1402+
.path(format!("/indexes/hdfs-logs/sources/{CLI_INGEST_SOURCE_ID}/toggle").as_str())
1403+
.method("PUT")
1404+
.body(r#"{"enable": true}"#)
1405+
.reply(&index_management_handler)
1406+
.await;
1407+
assert_eq!(resp.status(), 405);
13471408
Ok(())
13481409
}
13491410
}

0 commit comments

Comments
 (0)