Skip to content

Commit 67350e8

Browse files
nikhilsinhaparseablenikhilsinhaparseable
authored
Merge branch 'main' into metastore
2 parents 22f905a + 7d5600b commit 67350e8

File tree

4 files changed

+52
-27
lines changed

4 files changed

+52
-27
lines changed

‎resources/ingest_demo_data.sh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@ create_target() {
377377
echo "Creating webhook target..." >&2
378378

379379
response=$(curl -s -H "Content-Type: application/json" -H "$AUTH_HEADER" -X POST "$P_URL/api/v1/targets" -d @- << EOF
380-
{"type":"webhook","endpoint":"https://webhook.site/8e1f26bd-2f5b-47a2-9d0b-3b3dabb30710","name":"Test Webhook","auth":{"username":"","password":""},"skipTlsCheck":false,"notificationConfig":{"interval":1,"times":1}}
380+
{"type":"webhook","endpoint":"https://webhook.site/8e1f26bd-2f5b-47a2-9d0b-3b3dabb30710","name":"Test Webhook","auth":{"username":"","password":""},"skipTlsCheck":false}
381381
EOF
382382
)
383383

@@ -415,7 +415,7 @@ create_alerts() {
415415
echo "Creating alerts with target ID: $target_id"
416416

417417
# Alert 1: Error Count (severity_number = 18)
418-
alert1_json="{\"severity\":\"high\",\"title\":\"error count\",\"alertType\":\"threshold\",\"query\": \"select count(severity_number) as count_severity_number from demodata where severity_number=18\",\"thresholdConfig\":{\"operator\":\">\",\"value\":1000},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5h\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"]}"
418+
alert1_json="{\"severity\":\"high\",\"title\":\"error count\",\"alertType\":\"threshold\",\"query\": \"select count(severity_number) as count_severity_number from demodata where severity_number=18\",\"thresholdConfig\":{\"operator\":\">\",\"value\":1000},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5 hours\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"],\"tags\":[],\"notificationConfig\":{\"interval\":1}}"
419419

420420
response1=$(curl_with_retry "$P_URL/api/v1/alerts" "POST" "$alert1_json" "application/json" 3)
421421
if [[ $? -eq 0 ]]; then
@@ -426,7 +426,7 @@ create_alerts() {
426426
fi
427427

428428
# Alert 2: 400 Errors
429-
alert2_json="{\"severity\":\"critical\",\"title\":\"400 Errors\",\"alertType\":\"threshold\",\"query\": \"select count(body) as count_body from demodata where body like '%400%'\",\"thresholdConfig\":{\"operator\":\">\",\"value\":10},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5h\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"]}"
429+
alert2_json="{\"severity\":\"critical\",\"title\":\"400 Errors\",\"alertType\":\"threshold\",\"query\": \"select count(body) as count_body from demodata where body like '%400%'\",\"thresholdConfig\":{\"operator\":\">\",\"value\":10},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5 hours\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"],\"tags\":[],\"notificationConfig\":{\"interval\":1}}"
430430

431431
response2=$(curl_with_retry "$P_URL/api/v1/alerts" "POST" "$alert2_json" "application/json" 3)
432432
if [[ $? -eq 0 ]]; then
@@ -437,7 +437,7 @@ create_alerts() {
437437
fi
438438

439439
# Alert 3: Trace ID null
440-
alert3_json="{\"severity\":\"high\",\"title\":\"Trace ID null\",\"alertType\":\"threshold\",\"query\": \"select count(trace_id) as count_trace_id from demodata where trace_id is null\",\"thresholdConfig\":{\"operator\":\">\",\"value\":0},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5h\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"]}"
440+
alert3_json="{\"severity\":\"high\",\"title\":\"Trace ID null\",\"alertType\":\"threshold\",\"query\": \"select count(trace_id) as count_trace_id from demodata where trace_id is null\",\"thresholdConfig\":{\"operator\":\">\",\"value\":0},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5 hours\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"],\"tags\":[],\"notificationConfig\":{\"interval\":1}}"
441441
response3=$(curl_with_retry "$P_URL/api/v1/alerts" "POST" "$alert3_json" "application/json" 3)
442442
if [[ $? -eq 0 ]]; then
443443
echo "Alert 3 (Trace ID null) created successfully"

‎src/handlers/http/query.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ async fn handle_count_query(
168168
stream: table_name.to_string(),
169169
start_time: query_request.start_time.clone(),
170170
end_time: query_request.end_time.clone(),
171-
num_bins: 1,
171+
num_bins: Some(1),
172172
conditions: None,
173173
};
174174
let count_records = counts_req.get_bin_density().await?;

‎src/prism/logstream/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ impl PrismDatasetRequest {
351351
stream: stream.to_owned(),
352352
start_time: "1h".to_owned(),
353353
end_time: "now".to_owned(),
354-
num_bins: 10,
354+
num_bins: Some(10),
355355
conditions: None,
356356
};
357357

‎src/query/mod.rs‎

Lines changed: 46 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -295,8 +295,10 @@ impl Query {
295295
/// Record of counts for a given time bin.
296296
#[derive(Debug, Serialize, Clone, Deserialize)]
297297
pub struct CountsRecord {
298+
#[serde(alias = "_bin_start_time_")]
298299
/// Start time of the bin
299300
pub start_time: String,
301+
#[serde(alias = "_bin_end_time_")]
300302
/// End time of the bin
301303
pub end_time: String,
302304
/// Number of logs in the bin
@@ -327,8 +329,8 @@ pub struct CountsRequest {
327329
pub start_time: String,
328330
/// Excluded end time for counts query
329331
pub end_time: String,
330-
/// Number of bins to divide the time range into
331-
pub num_bins: u64,
332+
/// optional number of bins to divide the time range into
333+
pub num_bins: Option<u64>,
332334
/// Conditions
333335
pub conditions: Option<CountConditions>,
334336
}
@@ -397,9 +399,28 @@ impl CountsRequest {
397399
.signed_duration_since(time_range.start)
398400
.num_minutes() as u64;
399401

402+
let num_bins = if let Some(num_bins) = self.num_bins {
403+
num_bins
404+
} else {
405+
// create number of bins based on total minutes
406+
if total_minutes <= 60 * 5 {
407+
// till 5 hours, 1 bin = 1 min
408+
total_minutes
409+
} else if total_minutes <= 60 * 24 {
410+
// till 1 day, 1 bin = 5 min
411+
total_minutes.div_ceil(5)
412+
} else if total_minutes <= 60 * 24 * 10 {
413+
// till 10 days, 1 bin = 1 hour
414+
total_minutes.div_ceil(60)
415+
} else {
416+
// > 10 days, 1 bin = 1 day
417+
total_minutes.div_ceil(1440)
418+
}
419+
};
420+
400421
// divide minutes by num bins to get minutes per bin
401-
let quotient = total_minutes / self.num_bins;
402-
let remainder = total_minutes % self.num_bins;
422+
let quotient = total_minutes / num_bins;
423+
let remainder = total_minutes % num_bins;
403424
let have_remainder = remainder > 0;
404425

405426
// now create multiple bounds [startTime, endTime)
@@ -409,9 +430,9 @@ impl CountsRequest {
409430
let mut start = time_range.start;
410431

411432
let loop_end = if have_remainder {
412-
self.num_bins
433+
num_bins
413434
} else {
414-
self.num_bins - 1
435+
num_bins - 1
415436
};
416437

417438
// Create bins for all but the last date
@@ -446,36 +467,40 @@ impl CountsRequest {
446467

447468
let dur = time_range.end.signed_duration_since(time_range.start);
448469

449-
let date_bin = if dur.num_minutes() <= 60 * 10 {
450-
// date_bin 1 minute
470+
let table_name = &self.stream;
471+
let start_time_col_name = "_bin_start_time_";
472+
let end_time_col_name = "_bin_end_time_";
473+
let date_bin = if dur.num_minutes() <= 60 * 5 {
474+
// less than 5 hour = 1 min bin
475+
format!(
476+
"CAST(DATE_BIN('1m', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as {start_time_col_name}, DATE_BIN('1m', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '1m' as {end_time_col_name}"
477+
)
478+
} else if dur.num_minutes() <= 60 * 24 {
479+
// 1 day = 5 min bin
451480
format!(
452-
"CAST(DATE_BIN('1 minute', \"{}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as start_time, DATE_BIN('1 minute', \"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '1 minute' as end_time",
453-
self.stream
481+
"CAST(DATE_BIN('5m', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as {start_time_col_name}, DATE_BIN('5m', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '5m' as {end_time_col_name}"
454482
)
455-
} else if dur.num_minutes() > 60 * 10 && dur.num_minutes() < 60 * 240 {
456-
// date_bin 1 hour
483+
} else if dur.num_minutes() < 60 * 24 * 10 {
484+
// 10 days = 1 hour bin
457485
format!(
458-
"CAST(DATE_BIN('1 hour', \"{}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as start_time, DATE_BIN('1 hour', \"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '1 hour' as end_time",
459-
self.stream
486+
"CAST(DATE_BIN('1h', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as {start_time_col_name}, DATE_BIN('1h', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '1h' as {end_time_col_name}"
460487
)
461488
} else {
462-
// date_bin 1 day
489+
// 1 day
463490
format!(
464-
"CAST(DATE_BIN('1 day', \"{}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as start_time, DATE_BIN('1 day', \"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '1 day' as end_time",
465-
self.stream
491+
"CAST(DATE_BIN('1d', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as {start_time_col_name}, DATE_BIN('1d', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '1d' as {end_time_col_name}"
466492
)
467493
};
468494

469495
let query = if let Some(conditions) = &count_conditions.conditions {
470496
let f = get_filter_string(conditions).map_err(QueryError::CustomError)?;
471497
format!(
472-
"SELECT {date_bin}, COUNT(*) as count FROM \"{}\" WHERE {} GROUP BY end_time,start_time ORDER BY end_time",
473-
self.stream, f
498+
"SELECT {date_bin}, COUNT(*) as count FROM \"{table_name}\" WHERE {} GROUP BY {end_time_col_name},{start_time_col_name} ORDER BY {end_time_col_name}",
499+
f
474500
)
475501
} else {
476502
format!(
477-
"SELECT {date_bin}, COUNT(*) as count FROM \"{}\" GROUP BY end_time,start_time ORDER BY end_time",
478-
self.stream
503+
"SELECT {date_bin}, COUNT(*) as count FROM \"{table_name}\" GROUP BY {end_time_col_name},{start_time_col_name} ORDER BY {end_time_col_name}",
479504
)
480505
};
481506
Ok(query)

0 commit comments

Comments
 (0)