chore(optimizer): add tpcds unit test for optimizer

Cargo.toml

@@ -457,6 +457,7 @@ serde_stacker = { version = "0.1" }
 serde_test = "1.0"
 serde_urlencoded = "0.7.1"
 serde_with = { version = "3.8.1" }
+serde_yaml = { version = "0.9.34" }
 serfig = "0.1.0"
 sha1 = "0.10.5"
 sha2 = "0.10.8"

src/query/catalog/src/lib.rs

@@ -31,3 +31,6 @@ pub mod table_args;
 pub mod table_context;
 pub mod table_function;
 pub mod table_with_options;
+
+pub use statistics::BasicColumnStatistics;
+pub use table::TableStatistics;

src/query/service/Cargo.toml

@@ -190,6 +190,8 @@ mysql_async = { workspace = true }
 p256 = { workspace = true }
 pretty_assertions = { workspace = true }
 reqwest = { workspace = true }
+serde_json.workspace = true
+serde_yaml = { workspace = true }
 temp-env = { workspace = true }
 tempfile = { workspace = true }
 tower = { workspace = true }

src/query/service/tests/it/sql/planner/optimizer/data/README.md

@@ -0,0 +1,67 @@
+# TPC-DS Optimizer Test Data
+
+This directory contains test data for TPC-DS optimizer tests. The tests are structured as follows:
+
+## Directory Structure
+
+```
+data
+├── tables/    # SQL table definitions
+└── yaml/      # YAML test case definitions
+```
+
+## YAML Test Case Format
+
+Each test case is defined in a YAML file with the following structure:
+
+```yaml
+name: "Q3"                      # Test case name
+description: "Test description" # Optional description
+
+sql: |                          # SQL query to test
+  SELECT ...
+
+table_statistics:               # Table statistics
+  table_name:
+    num_rows: 1000
+    data_size: 102400
+    data_size_compressed: 51200
+    index_size: 20480
+    number_of_blocks: 10
+    number_of_segments: 2
+
+column_statistics:              # Column statistics
+  table_name.column_name:
+    min: 1990                   # Min value (can be number or string)
+    max: 2000                   # Max value (can be number or string)
+    ndv: 10                     # Number of distinct values
+    null_count: 0               # Number of null values
+
+raw_plan: |                     # Expected raw plan
+  ...
+
+optimized_plan: |               # Expected optimized plan
+  ...
+
+snow_plan: |                    # Optional expected snowflake plan
+  ...
+```
+
+## Table Definitions
+
+Table definitions are stored in SQL files in the `tables` directory. Each file contains a `CREATE TABLE` statement for a specific table used in the tests.
+
+## Adding New Tests
+
+To add a new test case:
+
+1. Create a new YAML file in the `yaml` directory with the test case definition.
+2. If the test uses new tables, add the table definitions to the `tables` directory.
+3. The test runner will automatically discover and run all test cases in the `yaml` directory.
+
+## Updating Existing Tests
+
+If the expected output of a test changes (e.g., due to optimizer improvements):
+
+1. Run the test to see the actual output.
+2. Update the `raw_plan`, `optimized_plan`, or `snow_plan` field in the YAML file to match the actual output.

src/query/service/tests/it/sql/planner/optimizer/data/tables/call_center.sql

@@ -0,0 +1,34 @@
+CREATE OR REPLACE TABLE call_center
+(
+    cc_call_center_sk         integer                               ,
+    cc_call_center_id         char(16)                              ,
+    cc_rec_start_date         date                              null,
+    cc_rec_end_date           date                              null,
+    cc_closed_date_sk         integer                           null,
+    cc_open_date_sk           integer                           null,
+    cc_name                   varchar(50)                       null,
+    cc_class                  varchar(50)                       null,
+    cc_employees              integer                           null,
+    cc_sq_ft                  integer                           null,
+    cc_hours                  char(20)                          null,
+    cc_manager                varchar(40)                       null,
+    cc_mkt_id                 integer                           null,
+    cc_mkt_class              char(50)                          null,
+    cc_mkt_desc               varchar(100)                      null,
+    cc_market_manager         varchar(40)                       null,
+    cc_division               integer                           null,
+    cc_division_name          varchar(50)                       null,
+    cc_company                integer                           null,
+    cc_company_name           char(50)                          null,
+    cc_street_number          char(10)                          null,
+    cc_street_name            varchar(60)                       null,
+    cc_street_type            char(15)                          null,
+    cc_suite_number           char(10)                          null,
+    cc_city                   varchar(60)                       null,
+    cc_county                 varchar(30)                       null,
+    cc_state                  char(2)                           null,
+    cc_zip                    char(10)                          null,
+    cc_country                varchar(20)                       null,
+    cc_gmt_offset             decimal(5,2)                      null,
+    cc_tax_percentage         decimal(5,2)                      null   
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/catalog_page.sql

@@ -0,0 +1,12 @@
+CREATE OR REPLACE TABLE catalog_page
+(
+    cp_catalog_page_sk        integer                               ,
+    cp_catalog_page_id        char(16)                              ,
+    cp_start_date_sk          integer                           null,
+    cp_end_date_sk            integer                           null,
+    cp_department             varchar(50)                       null,
+    cp_catalog_number         integer                           null,
+    cp_catalog_page_number    integer                           null,
+    cp_description            varchar(100)                      null,
+    cp_type                   varchar(100)                      null      
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/catalog_returns.sql

@@ -0,0 +1,30 @@
+CREATE OR REPLACE TABLE catalog_returns
+(
+    cr_returned_date_sk       integer                           null,
+    cr_returned_time_sk       integer                           null,
+    cr_item_sk                integer                               ,
+    cr_refunded_customer_sk   integer                           null,
+    cr_refunded_cdemo_sk      integer                           null,
+    cr_refunded_hdemo_sk      integer                           null,
+    cr_refunded_addr_sk       integer                           null,
+    cr_returning_customer_sk  integer                           null,
+    cr_returning_cdemo_sk     integer                           null,
+    cr_returning_hdemo_sk     integer                           null,
+    cr_returning_addr_sk      integer                           null,
+    cr_call_center_sk         integer                           null,
+    cr_catalog_page_sk        integer                           null,
+    cr_ship_mode_sk           integer                           null,
+    cr_warehouse_sk           integer                           null,
+    cr_reason_sk              integer                           null,
+    cr_order_number           integer                               ,
+    cr_return_quantity        integer                           null,
+    cr_return_amount          decimal(7,2)                      null,
+    cr_return_tax             decimal(7,2)                      null,
+    cr_return_amt_inc_tax     decimal(7,2)                      null,
+    cr_fee                    decimal(7,2)                      null,
+    cr_return_ship_cost       decimal(7,2)                      null,
+    cr_refunded_cash          decimal(7,2)                      null,
+    cr_reversed_charge        decimal(7,2)                      null,
+    cr_store_credit           decimal(7,2)                      null,
+    cr_net_loss               decimal(7,2)                      null       
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/customer.sql

@@ -0,0 +1,21 @@
+CREATE OR REPLACE TABLE customer
+(
+    c_customer_sk             integer                               ,
+    c_customer_id             char(16)                              ,
+    c_current_cdemo_sk        integer                           null,
+    c_current_hdemo_sk        integer                           null,
+    c_current_addr_sk         integer                           null,
+    c_first_shipto_date_sk    integer                           null,
+    c_first_sales_date_sk     integer                           null,
+    c_salutation              char(10)                          null,
+    c_first_name              char(20)                          null,
+    c_last_name               char(30)                          null,
+    c_preferred_cust_flag     char(1)                           null,
+    c_birth_day               integer                           null,
+    c_birth_month             integer                           null,
+    c_birth_year              integer                           null,
+    c_birth_country           varchar(20)                       null,
+    c_login                   char(13)                          null,
+    c_email_address           char(50)                          null,
+    c_last_review_date_sk     integer                           null   
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/customer_address.sql

@@ -0,0 +1,16 @@
+CREATE OR REPLACE TABLE customer_address
+(
+    ca_address_sk             integer                               ,
+    ca_address_id             char(16)                              ,
+    ca_street_number          char(10)                          null,
+    ca_street_name            varchar(60)                       null,
+    ca_street_type            char(15)                          null,
+    ca_suite_number           char(10)                          null,
+    ca_city                   varchar(60)                       null,
+    ca_county                 varchar(30)                       null,
+    ca_state                  char(2)                           null,
+    ca_zip                    char(10)                          null,
+    ca_country                varchar(20)                       null,
+    ca_gmt_offset             decimal(5,2)                      null,
+    ca_location_type          char(20)                          null     
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/customer_demographics.sql

@@ -0,0 +1,12 @@
+CREATE OR REPLACE TABLE customer_demographics
+(
+    cd_demo_sk                integer                               ,
+    cd_gender                 char(1)                           null,
+    cd_marital_status         char(1)                           null,
+    cd_education_status       char(20)                          null,
+    cd_purchase_estimate      integer                           null,
+    cd_credit_rating          char(10)                          null,
+    cd_dep_count              integer                           null,
+    cd_dep_employed_count     integer                           null,
+    cd_dep_college_count      integer                           null
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/date_dim.sql

@@ -0,0 +1,30 @@
+CREATE OR REPLACE TABLE date_dim (
+    d_date_sk INTEGER,
+    d_date_id VARCHAR,
+    d_date DATE,
+    d_month_seq INTEGER,
+    d_week_seq INTEGER,
+    d_quarter_seq INTEGER,
+    d_year INTEGER,
+    d_dow INTEGER,
+    d_moy INTEGER,
+    d_dom INTEGER,
+    d_qoy INTEGER,
+    d_fy_year INTEGER,
+    d_fy_quarter_seq INTEGER,
+    d_fy_week_seq INTEGER,
+    d_day_name VARCHAR,
+    d_quarter_name VARCHAR,
+    d_holiday CHAR(1),
+    d_weekend CHAR(1),
+    d_following_holiday CHAR(1),
+    d_first_dom INTEGER,
+    d_last_dom INTEGER,
+    d_same_day_ly INTEGER,
+    d_same_day_lq INTEGER,
+    d_current_day CHAR(1),
+    d_current_week CHAR(1),
+    d_current_month CHAR(1),
+    d_current_quarter CHAR(1),
+    d_current_year CHAR(1)
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/household_demographics.sql

@@ -0,0 +1,8 @@
+CREATE OR REPLACE TABLE household_demographics
+(
+    hd_demo_sk                integer                               ,
+    hd_income_band_sk         integer                           null,
+    hd_buy_potential          char(15)                          null,
+    hd_dep_count              integer                           null,
+    hd_vehicle_count          integer                           null    
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/income_band.sql

@@ -0,0 +1,6 @@
+CREATE OR REPLACE TABLE income_band
+(
+    ib_income_band_sk         integer                               ,
+    ib_lower_bound            integer                           null,
+    ib_upper_bound            integer                           null   
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/inventory.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE TABLE inventory
+(
+    inv_date_sk               integer                               ,
+    inv_item_sk               integer                               ,
+    inv_warehouse_sk          integer                               ,
+    inv_quantity_on_hand      integer                           null       
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/item.sql

@@ -0,0 +1,24 @@
+CREATE OR REPLACE TABLE item (
+    i_item_sk INTEGER,
+    i_item_id VARCHAR,
+    i_rec_start_date DATE,
+    i_rec_end_date DATE,
+    i_item_desc VARCHAR,
+    i_current_price DECIMAL(7,2),
+    i_wholesale_cost DECIMAL(7,2),
+    i_brand_id INTEGER,
+    i_brand VARCHAR,
+    i_class_id INTEGER,
+    i_class VARCHAR,
+    i_category_id INTEGER,
+    i_category VARCHAR,
+    i_manufact_id INTEGER,
+    i_manufact VARCHAR,
+    i_size VARCHAR,
+    i_formulation VARCHAR,
+    i_color VARCHAR,
+    i_units VARCHAR,
+    i_container VARCHAR,
+    i_manager_id INTEGER,
+    i_product_name VARCHAR
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/promotion.sql

@@ -0,0 +1,22 @@
+CREATE OR REPLACE TABLE promotion
+(
+    p_promo_sk                integer                               ,
+    p_promo_id                char(16)                              ,
+    p_start_date_sk           integer                           null,
+    p_end_date_sk             integer                           null,
+    p_item_sk                 integer                           null,
+    p_cost                    decimal(15,2)                     null,
+    p_response_target         integer                           null,
+    p_promo_name              char(50)                          null,
+    p_channel_dmail           char(1)                           null,
+    p_channel_email           char(1)                           null,
+    p_channel_catalog         char(1)                           null,
+    p_channel_tv              char(1)                           null,
+    p_channel_radio           char(1)                           null,
+    p_channel_press           char(1)                           null,
+    p_channel_event           char(1)                           null,
+    p_channel_demo            char(1)                           null,
+    p_channel_details         varchar(100)                      null,
+    p_purpose                 char(15)                          null,
+    p_discount_active         char(1)                           null     
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/reason.sql

@@ -0,0 +1,6 @@
+CREATE OR REPLACE TABLE reason
+(
+    r_reason_sk               integer                               ,
+    r_reason_id               char(16)                              ,
+    r_reason_desc             char(100)                         null    
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/ship_mode.sql

@@ -0,0 +1,9 @@
+CREATE OR REPLACE TABLE ship_mode
+(
+    sm_ship_mode_sk           integer                               ,
+    sm_ship_mode_id           char(16)                              ,
+    sm_type                   char(30)                          null,
+    sm_code                   char(10)                          null,
+    sm_carrier                char(20)                          null,
+    sm_contract               char(20)                          null 
+);

src/query/service/tests/it/sql/planner/optimizer/data/tables/store.sql

@@ -0,0 +1,32 @@
+CREATE OR REPLACE TABLE store
+(
+    s_store_sk                integer                               ,
+    s_store_id                char(16)                              ,
+    s_rec_start_date          date                              null,
+    s_rec_end_date            date                              null,
+    s_closed_date_sk          integer                           null,
+    s_store_name              varchar(50)                       null,
+    s_number_employees        integer                           null,
+    s_floor_space             integer                           null,
+    s_hours                   char(20)                          null,
+    s_manager                 varchar(40)                       null,
+    s_market_id               integer                           null,
+    s_geography_class         varchar(100)                      null,
+    s_market_desc             varchar(100)                      null,
+    s_market_manager          varchar(40)                       null,
+    s_division_id             integer                           null,
+    s_division_name           varchar(50)                       null,
+    s_company_id              integer                           null,
+    s_company_name            varchar(50)                       null,
+    s_street_number           varchar(10)                       null,
+    s_street_name             varchar(60)                       null,
+    s_street_type             char(15)                          null,
+    s_suite_number            char(10)                          null,
+    s_city                    varchar(60)                       null,
+    s_county                  varchar(30)                       null,
+    s_state                   char(2)                           null,
+    s_zip                     char(10)                          null,
+    s_country                 varchar(20)                       null,
+    s_gmt_offset              decimal(5,2)                      null,
+    s_tax_precentage          decimal(5,2)                      null
+);