Renamed perf structures; cleaned up perf traces; added initial storage performance tests

Author: LarryOsterman

sdk/core/azure_core_test/src/perf/config_tests.rs

@@ -20,11 +20,11 @@ fn create_failed_test(_runner: &PerfRunner) -> CreatePerfTestReturn {
 }
 
 // Helper function to create a basic test metadata for testing
-fn create_basic_test_metadata() -> TestMetadata {
-    TestMetadata {
+fn create_basic_test_metadata() -> PerfTestMetadata {
+    PerfTestMetadata {
         name: "basic_test",
         description: "A basic test for testing purposes",
-        options: vec![TestOption {
+        options: vec![PerfTestOption {
             name: "test-option",
             short_activator: 't',
             long_activator: "test-option",
@@ -38,12 +38,12 @@ fn create_basic_test_metadata() -> TestMetadata {
 }
 
 // Helper function to create test metadata with multiple options
-fn create_complex_test_metadata() -> TestMetadata {
-    TestMetadata {
+fn create_complex_test_metadata() -> PerfTestMetadata {
+    PerfTestMetadata {
         name: "complex_test",
         description: "A complex test with multiple options",
         options: vec![
-            TestOption {
+            PerfTestOption {
                 name: "mandatory-option",
                 short_activator: 'm',
                 long_activator: "mandatory",
@@ -52,7 +52,7 @@ fn create_complex_test_metadata() -> TestMetadata {
                 mandatory: true,
                 sensitive: false,
             },
-            TestOption {
+            PerfTestOption {
                 name: "sensitive-option",
                 short_activator: 's',
                 long_activator: "sensitive",
@@ -61,7 +61,7 @@ fn create_complex_test_metadata() -> TestMetadata {
                 mandatory: false,
                 sensitive: true,
             },
-            TestOption {
+            PerfTestOption {
                 name: "flag-option",
                 short_activator: 'f',
                 long_activator: "flag",
@@ -74,11 +74,11 @@ fn create_complex_test_metadata() -> TestMetadata {
 }
 
 // Helper function to create test metadata without short activators
-fn create_no_short_activator_test_metadata() -> TestMetadata {
-    TestMetadata {
+fn create_no_short_activator_test_metadata() -> PerfTestMetadata {
+    PerfTestMetadata {
         name: "no_short_test",
         description: "Test without short activators",
-        options: vec![TestOption {
+        options: vec![PerfTestOption {
             name: "long-only",
             short_activator: '\0',
             long_activator: "long-only",
@@ -494,7 +494,7 @@ fn test_perf_runner_options_debug() {
 
 #[test]
 fn test_test_option_debug_and_default() {
-    let option = TestOption::default();
+    let option = PerfTestOption::default();
 
     // Test default values
     assert_eq!(option.name, "");
@@ -592,11 +592,11 @@ fn complex_test_create(_runner: &PerfRunner) -> CreatePerfTestReturn {
 
 #[tokio::test]
 async fn test_perf_runner_with_test_functions() {
-    let tests = vec![TestMetadata {
+    let tests = vec![PerfTestMetadata {
         name: "complex_test",
         description: "A complex test with multiple options",
         options: vec![
-            TestOption {
+            PerfTestOption {
                 name: "mandatory-option",
                 short_activator: 'm',
                 long_activator: "mandatory",
@@ -605,7 +605,7 @@ async fn test_perf_runner_with_test_functions() {
                 mandatory: true,
                 sensitive: false,
             },
-            TestOption {
+            PerfTestOption {
                 name: "sensitive-option",
                 short_activator: 's',
                 long_activator: "sensitive",
@@ -614,7 +614,7 @@ async fn test_perf_runner_with_test_functions() {
                 mandatory: false,
                 sensitive: true,
             },
-            TestOption {
+            PerfTestOption {
                 name: "flag-option",
                 short_activator: 'f',
                 long_activator: "flag",

sdk/core/azure_core_test/src/perf/framework_tests.rs

@@ -91,10 +91,10 @@ async fn test_perf_runner_with_single_test() {
     let runner = PerfRunner::with_command_line(
         env!("CARGO_MANIFEST_DIR"),
         file!(),
-        vec![TestMetadata {
+        vec![PerfTestMetadata {
             name: "fibonacci1",
             description: "A basic test for testing purposes",
-            options: vec![TestOption {
+            options: vec![PerfTestOption {
                 name: "count",
                 mandatory: true,
                 short_activator: 'c',

sdk/core/azure_core_test/src/perf/mod.rs

@@ -30,21 +30,21 @@ pub type CreatePerfTestReturn =
 
 /// Metadata about a performance test.
 #[derive(Debug, Clone)]
-pub struct TestMetadata {
+pub struct PerfTestMetadata {
     /// The name of the test suite.
     pub name: &'static str,
     /// A brief description of the test suite.
     pub description: &'static str,
     /// The set of test options supported by this test.
-    pub options: Vec<TestOption>,
+    pub options: Vec<PerfTestOption>,
 
     /// A function used to create the performance test.
     pub create_test: fn(&PerfRunner) -> CreatePerfTestReturn,
 }
 
 /// #A `TestOptions` defines a set of options for the test which will be merged with the common test inputs to define the command line for the performance test.
 #[derive(Debug, Default, Clone)]
-pub struct TestOption {
+pub struct PerfTestOption {
     /// The name of the test option. This is used as the key in the `TestArguments` map.
     pub name: &'static str,
 
@@ -112,7 +112,7 @@ impl From<&ArgMatches> for PerfRunnerOptions {
 #[derive(Debug, Clone)]
 pub struct PerfRunner {
     options: PerfRunnerOptions,
-    tests: Vec<TestMetadata>,
+    tests: Vec<PerfTestMetadata>,
     arguments: ArgMatches,
     package_dir: &'static str,
     module_name: &'static str,
@@ -123,7 +123,7 @@ impl PerfRunner {
     pub fn new(
         package_dir: &'static str,
         module_name: &'static str,
-        tests: Vec<TestMetadata>,
+        tests: Vec<PerfTestMetadata>,
     ) -> azure_core::Result<Self> {
         let command = Self::get_command_from_metadata(&tests);
         let arguments = command.get_matches();
@@ -141,7 +141,7 @@ impl PerfRunner {
     pub fn with_command_line(
         package_dir: &'static str,
         module_name: &'static str,
-        tests: Vec<TestMetadata>,
+        tests: Vec<PerfTestMetadata>,
         args: Vec<&str>,
     ) -> azure_core::Result<Self> {
         let command = Self::get_command_from_metadata(&tests);
@@ -273,6 +273,7 @@ impl PerfRunner {
         _test_name: &str,
         duration: Duration,
     ) -> azure_core::Result<()> {
+        self.progress.store(0, Ordering::SeqCst);
         let mut tasks: JoinSet<Result<()>> = JoinSet::new();
         for _ in 0..self.options.parallel {
             let test_instance_clone = Arc::clone(&test_instance);
@@ -283,6 +284,7 @@ impl PerfRunner {
                 //                let context =
                 //                    TestContext::new(package_dir, module_name, " test_name_copy.as_str()")?;
 
+                tokio::task::yield_now().await;
                 loop {
                     test_instance_clone.run(/*&context*/).await?;
                     progress.fetch_add(1, Ordering::SeqCst);
@@ -295,12 +297,16 @@ impl PerfRunner {
                 _ = tokio::time::sleep(timeout) => {println!("Timeout reached, stopping test tasks: {:?}", start.elapsed());},
                 _ = tasks.join_all() =>  {println!("All test tasks completed: {:?}", start.elapsed());},
                 _ = async {
+                        let mut last_count = 0;
                         loop {
                             tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
-                            println!("{:<10?} elapsed: {:.5} op/sec, {:4} sec/operation.",
-                                start.elapsed(),
-                                self.progress.load(Ordering::SeqCst) as f64 / start.elapsed().as_secs_f64(),
-                                Duration::seconds_f64( start.elapsed().as_secs_f64() / self.progress.load(Ordering::SeqCst) as f64 ));
+                            let current_total = self.progress.load(Ordering::SeqCst);
+                            // println!("{:<10?} elapsed: {:.5} op/sec, {:4} sec/operation.",
+                            //     start.elapsed(),
+                            //     self.progress.load(Ordering::SeqCst) as f64 / start.elapsed().as_secs_f64(),
+                            //     Duration::seconds_f64( start.elapsed().as_secs_f64() / self.progress.load(Ordering::SeqCst) as f64 ));
+                            println!("Current {:3}, Total {:5} {:4}", current_total - last_count, current_total, Duration::seconds_f64( start.elapsed().as_secs_f64() / self.progress.load(Ordering::SeqCst) as f64 ));
+                            last_count = current_total;
                         }
                     }, if !self.options.disable_progress => {},
         );
@@ -323,7 +329,7 @@ impl PerfRunner {
     //   * Sync - run a synchronous version of the test
 
     /// Constructs a `clap::Command` from the provided test metadata.
-    fn get_command_from_metadata(tests: &[TestMetadata]) -> clap::Command {
+    fn get_command_from_metadata(tests: &[PerfTestMetadata]) -> clap::Command {
         let mut command = clap::Command::new("perf-tests")
             .about("Run performance tests for the Azure SDK for Rust")
             .arg(

sdk/keyvault/azure_security_keyvault_secrets/perf/get_secret.rs

@@ -1,11 +1,23 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+//! Keyvault Secrets performance tests.
+//!
+//! This test measures the performance of getting a secret from Azure Key Vault.
+//! It sets up a secret in the Key Vault during the setup phase and then repeatedly retrieves it
+//! during the run phase. The test can be configured with the vault URL via command line arguments
+//! to target different Key Vault instances.
+//!
+//! To run the test, use the following command line arguments:
+//!
+//! cargo test --package azure_security_keyvault_secrets --test performance_tests -- --duration 10 --parallel 20 get_secret -u https://<my_vault>.vault.azure.net/
+//!
+
 use std::sync::OnceLock;
 
 use azure_core::Result;
 use azure_core_test::{
-    perf::{CreatePerfTestReturn, PerfRunner, PerfTest, TestMetadata, TestOption},
+    perf::{CreatePerfTestReturn, PerfRunner, PerfTest, PerfTestMetadata, PerfTestOption},
     TestContext,
 };
 use azure_security_keyvault_secrets::{models::SetSecretParameters, SecretClient};
@@ -17,11 +29,11 @@ struct GetSecrets {
 }
 
 impl GetSecrets {
-    fn test_metadata() -> TestMetadata {
-        TestMetadata {
+    fn test_metadata() -> PerfTestMetadata {
+        PerfTestMetadata {
             name: "get_secret",
             description: "Get a secret from Key Vault",
-            options: vec![TestOption {
+            options: vec![PerfTestOption {
                 name: "vault_url",
                 display_message: "The URL of the Key Vault to use in the test",
                 mandatory: true,

sdk/storage/azure_storage_blob/Cargo.toml

@@ -37,3 +37,8 @@ azure_storage_blob_test.path = "../azure_storage_blob_test"
 futures.workspace = true
 tokio = { workspace = true, features = ["macros"] }
 tracing.workspace = true
+
+[[test]]
+name = "performance_tests"
+path = "perf/perf_tests.rs"
+harness = false

sdk/storage/azure_storage_blob/perf/list_blob_test.rs

@@ -0,0 +1,106 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+use azure_core::{Bytes, Result};
+use azure_core_test::{
+    perf::{CreatePerfTestReturn, PerfRunner, PerfTest, PerfTestMetadata, PerfTestOption},
+    TestContext,
+};
+use azure_identity::DeveloperToolsCredential;
+use azure_storage_blob::BlobContainerClient;
+use futures::TryStreamExt;
+
+pub struct ListBlobTest {
+    count: u32,
+    client: BlobContainerClient,
+}
+
+impl ListBlobTest {
+    fn create_list_blob_test(runner: &PerfRunner) -> CreatePerfTestReturn {
+        async fn create_test(runner: PerfRunner) -> Result<Box<dyn PerfTest>> {
+            let count: Option<&String> = runner.try_get_test_arg("count")?;
+
+            println!("ListBlobTest with count: {:?}", count);
+            let count = count.expect("count argument is mandatory").parse::<u32>()?;
+            println!("Parsed count: {}", count);
+
+            let endpoint: Option<&String> = runner.try_get_test_arg("endpoint")?;
+            let endpoint = endpoint.expect("endpoint argument is mandatory").clone();
+            println!("Using endpoint: {}", endpoint);
+
+            let container_name = format!("perf-container-{}", uuid::Uuid::new_v4());
+            let credential = DeveloperToolsCredential::new(None)?;
+            let client = BlobContainerClient::new(&endpoint, container_name, credential, None)?;
+
+            Ok(Box::new(ListBlobTest { count, client }) as Box<dyn PerfTest>)
+        }
+        // Here you would create and return an instance of your performance test.
+        // For example:
+        Box::pin(create_test(runner.clone()))
+    }
+
+    pub fn test_metadata() -> PerfTestMetadata {
+        PerfTestMetadata {
+            name: "list_blob",
+            description: "List blobs in a container",
+            options: vec![
+                PerfTestOption {
+                    name: "count",
+                    display_message: "The number of blobs to list",
+                    mandatory: true,
+                    short_activator: 'c',
+                    long_activator: "count",
+                    expected_args_len: 1,
+                    ..Default::default()
+                },
+                PerfTestOption {
+                    name: "endpoint",
+                    display_message: "The endpoint of the blob storage",
+                    mandatory: true,
+                    short_activator: 'e',
+                    long_activator: "endpoint",
+                    expected_args_len: 1,
+                    ..Default::default()
+                },
+            ],
+            create_test: Self::create_list_blob_test,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl PerfTest for ListBlobTest {
+    async fn setup(&self, _context: &TestContext) -> azure_core::Result<()> {
+        // Setup code before running the test
+
+        let result = self.client.create_container(None).await?;
+
+        for i in 0..self.count {
+            let blob_name = format!("blob-{}", i);
+            let blob_client = self.client.blob_client(blob_name);
+
+            let body = vec![0u8; 1024 * 1024]; // 1 MB blob
+            let body_bytes = Bytes::from(body);
+
+            let result = blob_client.upload(body_bytes.into(), true, 5, None).await?;
+        }
+
+        Ok(())
+    }
+
+    async fn run(&self) -> azure_core::Result<()> {
+        // The actual performance test code
+
+        let mut iterator = self.client.list_blobs(None)?;
+        while let Some(blob_segment) = iterator.try_next().await? {
+            let body = blob_segment.into_body()?;
+        }
+
+        Ok(())
+    }
+
+    async fn cleanup(&self, _context: &TestContext) -> azure_core::Result<()> {
+        // Cleanup code after running the test
+        Ok(())
+    }
+}

sdk/storage/azure_storage_blob/perf/perf_tests.rs

@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+/// list_blob performance test.
+mod list_blob_test;
+
+use azure_core_test::perf::PerfRunner;
+use list_blob_test::ListBlobTest;
+
+#[tokio::main]
+async fn main() -> azure_core::Result<()> {
+    let runner = PerfRunner::new(
+        env!("CARGO_MANIFEST_DIR"),
+        "foo",
+        vec![ListBlobTest::test_metadata()],
+    )?;
+
+    runner.run().await?;
+
+    Ok(())
+}