@@ -36,7 +36,7 @@ use http::StatusCode;
36
36
use itertools:: Itertools ;
37
37
use serde:: { Deserialize , Serialize } ;
38
38
use serde_json:: { json, Value } ;
39
- use std:: collections:: { HashMap , HashSet } ;
39
+ use std:: collections:: HashMap ;
40
40
use std:: pin:: Pin ;
41
41
use std:: sync:: Arc ;
42
42
use std:: time:: Instant ;
@@ -81,15 +81,14 @@ pub async fn get_records_and_fields(
81
81
query_request : & Query ,
82
82
req : & HttpRequest ,
83
83
) -> Result < ( Option < Vec < RecordBatch > > , Option < Vec < String > > ) , QueryError > {
84
- let tables = resolve_stream_names ( & query_request. query ) ?;
85
84
let session_state = QUERY_SESSION . state ( ) ;
86
-
87
85
let time_range =
88
86
TimeRange :: parse_human_time ( & query_request. start_time , & query_request. end_time ) ?;
87
+ let tables = resolve_stream_names ( & query_request. query ) ?;
88
+ //check or load streams in memory
89
+ create_streams_for_distributed ( tables. clone ( ) ) . await ?;
89
90
90
- let query: LogicalQuery =
91
- into_query ( query_request, & session_state, time_range, & tables) . await ?;
92
- update_schema_when_distributed ( & tables) . await ?;
91
+ let query: LogicalQuery = into_query ( query_request, & session_state, time_range) . await ?;
93
92
let creds = extract_session_key_from_req ( req) ?;
94
93
let permissions = Users . get_permissions ( & creds) ;
95
94
@@ -115,10 +114,10 @@ pub async fn query(req: HttpRequest, query_request: Query) -> Result<HttpRespons
115
114
let time_range =
116
115
TimeRange :: parse_human_time ( & query_request. start_time , & query_request. end_time ) ?;
117
116
let tables = resolve_stream_names ( & query_request. query ) ?;
117
+ //check or load streams in memory
118
+ create_streams_for_distributed ( tables. clone ( ) ) . await ?;
118
119
119
- let query: LogicalQuery =
120
- into_query ( & query_request, & session_state, time_range, & tables) . await ?;
121
- update_schema_when_distributed ( & tables) . await ?;
120
+ let query: LogicalQuery = into_query ( & query_request, & session_state, time_range) . await ?;
122
121
let creds = extract_session_key_from_req ( & req) ?;
123
122
let permissions = Users . get_permissions ( & creds) ;
124
123
@@ -408,35 +407,12 @@ pub async fn update_schema_when_distributed(tables: &Vec<String>) -> Result<(),
408
407
/// Create streams for querier if they do not exist
409
408
/// get list of streams from memory and storage
410
409
/// create streams for memory from storage if they do not exist
411
- pub async fn create_streams_for_querier ( ) -> Result < ( ) , QueryError > {
412
- let store = PARSEABLE . storage . get_object_store ( ) ;
413
- let querier_streams = PARSEABLE . streams . list ( ) ;
414
-
415
- let querier_streams_set: HashSet < _ > = querier_streams. into_iter ( ) . collect ( ) ;
416
- // fetch querier streams which have field list blank
417
- // now missing streams should be list of streams which are in storage but not in querier
418
- // and also have no fields in the schema
419
- // this is to ensure that we do not create streams for querier which already exist in querier
420
-
421
- let missing_streams: Vec < _ > = store
422
- . list_streams ( )
423
- . await ?
424
- . into_iter ( )
425
- . filter ( |stream_name| {
426
- !querier_streams_set. contains ( stream_name)
427
- || PARSEABLE
428
- . get_stream ( stream_name)
429
- . map ( |s| s. get_schema ( ) . fields ( ) . is_empty ( ) )
430
- . unwrap_or ( false )
431
- } )
432
- . collect ( ) ;
433
-
434
- if missing_streams. is_empty ( ) {
410
+ pub async fn create_streams_for_distributed ( streams : Vec < String > ) -> Result < ( ) , QueryError > {
411
+ if PARSEABLE . options . mode != Mode :: Query && PARSEABLE . options . mode != Mode :: Prism {
435
412
return Ok ( ( ) ) ;
436
413
}
437
-
438
414
let mut join_set = JoinSet :: new ( ) ;
439
- for stream_name in missing_streams {
415
+ for stream_name in streams {
440
416
join_set. spawn ( async move {
441
417
let result = PARSEABLE
442
418
. create_stream_and_schema_from_storage ( & stream_name)
@@ -494,7 +470,6 @@ pub async fn into_query(
494
470
query : & Query ,
495
471
session_state : & SessionState ,
496
472
time_range : TimeRange ,
497
- tables : & Vec < String > ,
498
473
) -> Result < LogicalQuery , QueryError > {
499
474
if query. query . is_empty ( ) {
500
475
return Err ( QueryError :: EmptyQuery ) ;
@@ -507,33 +482,7 @@ pub async fn into_query(
507
482
if query. end_time . is_empty ( ) {
508
483
return Err ( QueryError :: EmptyEndTime ) ;
509
484
}
510
- let raw_logical_plan = match session_state. create_logical_plan ( & query. query ) . await {
511
- Ok ( plan) => plan,
512
- Err ( _) => {
513
- let mut join_set = JoinSet :: new ( ) ;
514
- for stream_name in tables {
515
- let stream_name = stream_name. clone ( ) ;
516
- join_set. spawn ( async move {
517
- let result = PARSEABLE
518
- . create_stream_and_schema_from_storage ( & stream_name)
519
- . await ;
520
-
521
- if let Err ( e) = & result {
522
- warn ! ( "Failed to create stream '{}': {}" , stream_name, e) ;
523
- }
524
-
525
- ( stream_name, result)
526
- } ) ;
527
- }
528
-
529
- while let Some ( result) = join_set. join_next ( ) . await {
530
- if let Err ( join_error) = result {
531
- warn ! ( "Task join error: {}" , join_error) ;
532
- }
533
- }
534
- session_state. create_logical_plan ( & query. query ) . await ?
535
- }
536
- } ;
485
+ let raw_logical_plan = session_state. create_logical_plan ( & query. query ) . await ?;
537
486
538
487
Ok ( crate :: query:: Query {
539
488
raw_logical_plan,
0 commit comments