diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1f07211..f375123 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,12 +26,14 @@ jobs: matrix: os: ["ubuntu-latest"] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + cratedb-version: ["5.5.3"] fail-fast: false services: cratedb: - image: crate/crate:nightly + image: crate/crate:${{ matrix.cratedb-version }} ports: + - 4200:4200 - 5432:5432 env: diff --git a/pyproject.toml b/pyproject.toml index b5ce852..2055b36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,8 @@ release = [ "twine<5", ] test = [ + "crate[sqlalchemy]", + "pandas<2.1", "pytest<8", "pytest-asyncio<1", "pytest-cov<5", @@ -122,7 +124,7 @@ show_missing = true [tool.ruff] line-length = 120 - +extend-ignore = ["PD901"] select = [ # Bandit "S", @@ -167,17 +169,17 @@ check = [ ] format = [ - { cmd = "black ." }, - { cmd = "isort ." }, + { cmd = "black sqlalchemy_postgresql_relaxed/ tests/" }, + { cmd = "isort sqlalchemy_postgresql_relaxed/ tests/" }, # Configure Ruff not to auto-fix (remove!) unused variables (F841) and `print` statements (T201). { cmd = "ruff --fix --ignore=ERA --ignore=F401 --ignore=F841 --ignore=T20 ." }, { cmd = "pyproject-fmt pyproject.toml" }, ] lint = [ - { cmd = "ruff check ." }, - { cmd = "black --check ." }, - { cmd = "isort --check ." }, + { cmd = "ruff check sqlalchemy_postgresql_relaxed/ tests/" }, + { cmd = "black --check sqlalchemy_postgresql_relaxed/ tests/" }, + { cmd = "isort --check sqlalchemy_postgresql_relaxed/ tests/" }, { cmd = "validate-pyproject pyproject.toml" }, { cmd = "proselint *.rst doc/*.rst" }, # { cmd = "mypy" }, diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..4d9d337 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,46 @@ +import pytest + + +def pytest_addoption(parser): + """ + Register custom options to support invocation by cr8. + Use cr8 to invoke a pytest test suite, see `run.sh`. + + Example:: + + pytest -vvv --http-host 127.0.0.1 --http-port 4200 --psql-host 127.0.0.1 --psql-port 5432 + + https://github.com/mfussenegger/cr8 + + TODO: Refactor to `cratedb-toolkit` or `pytest-cratedb` in the long run. + """ + parser.addoption("--http-url", action="store", default="localhost:4200") + parser.addoption("--http-host", action="store", default="localhost") + parser.addoption("--http-port", action="store", default="4200") + parser.addoption("--psql-host", action="store", default="localhost") + parser.addoption("--psql-port", action="store", default="5432") + + +@pytest.fixture +def cratedb_http_url(pytestconfig): + return pytestconfig.getoption("--http-url") + + +@pytest.fixture +def cratedb_http_host(pytestconfig): + return pytestconfig.getoption("--http-host") + + +@pytest.fixture +def cratedb_http_port(pytestconfig): + return pytestconfig.getoption("--http-port") + + +@pytest.fixture +def cratedb_psql_host(pytestconfig): + return pytestconfig.getoption("--psql-host") + + +@pytest.fixture +def cratedb_psql_port(pytestconfig): + return pytestconfig.getoption("--psql-port") diff --git a/tests/test_cratedb_pandas_read.py b/tests/test_cratedb_pandas_read.py new file mode 100644 index 0000000..80248fa --- /dev/null +++ b/tests/test_cratedb_pandas_read.py @@ -0,0 +1,68 @@ +import pandas as pd +import pytest +import sqlalchemy as sa +from pandas._testing import assert_frame_equal +from sqlalchemy.ext.asyncio import create_async_engine + +REFERENCE_FRAME = pd.DataFrame.from_records([{"mountain": "Mont Blanc", "height": 4808}]) +SQL_SELECT_STATEMENT = "SELECT mountain, height FROM sys.summits ORDER BY height DESC LIMIT 1;" + + +pytest.skip("Does not work on Python 3.7", allow_module_level=True) + + +def test_crate_read_sql(cratedb_http_host, cratedb_http_port): + engine = sa.create_engine( + url=f"crate://{cratedb_http_host}:{cratedb_http_port}", + echo=True, + ) + conn = engine.connect() + df = pd.read_sql(sql=sa.text(SQL_SELECT_STATEMENT), con=conn) + assert_frame_equal(df, REFERENCE_FRAME) + + +def test_psycopg_read_sql(cratedb_psql_host, cratedb_psql_port): + engine = sa.create_engine( + url=f"postgresql+psycopg_relaxed://crate@{cratedb_psql_host}:{cratedb_psql_port}", + isolation_level="AUTOCOMMIT", + use_native_hstore=False, + echo=True, + ) + conn = engine.connect() + df = pd.read_sql(sql=sa.text(SQL_SELECT_STATEMENT), con=conn) + assert_frame_equal(df, REFERENCE_FRAME) + + +@pytest.mark.asyncio +async def test_psycopg_async_read_sql(cratedb_psql_host, cratedb_psql_port): + engine = create_async_engine( + url=f"postgresql+psycopg_relaxed://crate@{cratedb_psql_host}:{cratedb_psql_port}", + isolation_level="AUTOCOMMIT", + use_native_hstore=False, + echo=True, + ) + + async with engine.begin() as conn: + df = await conn.run_sync(read_sql_sync, sa.text(SQL_SELECT_STATEMENT)) + assert_frame_equal(df, REFERENCE_FRAME) + + +@pytest.mark.asyncio +async def test_asyncpg_read_sql(cratedb_psql_host, cratedb_psql_port): + engine = create_async_engine( + url=f"postgresql+asyncpg_relaxed://crate@{cratedb_psql_host}:{cratedb_psql_port}", + isolation_level="AUTOCOMMIT", + echo=True, + ) + + async with engine.begin() as conn: + df = await conn.run_sync(read_sql_sync, sa.text(SQL_SELECT_STATEMENT)) + assert_frame_equal(df, REFERENCE_FRAME) + + +def read_sql_sync(conn, stmt): + """ + Making pd.read_sql connection the first argument to make it compatible + with conn.run_sync(), see https://stackoverflow.com/a/70861276. + """ + return pd.read_sql(stmt, conn) diff --git a/tests/test_cratedb_pandas_write.py b/tests/test_cratedb_pandas_write.py new file mode 100644 index 0000000..cb5dcca --- /dev/null +++ b/tests/test_cratedb_pandas_write.py @@ -0,0 +1,104 @@ +import pandas as pd +import pytest +import sqlalchemy as sa +from pandas._testing import assert_frame_equal +from sqlalchemy.ext.asyncio import create_async_engine + +INPUT_FRAME = pd.DataFrame.from_records([{"id": 1, "name": "foo", "value": 42.42}]) +SQL_SELECT_STATEMENT = "SELECT * FROM doc.foo;" +SQL_REFRESH_STATEMENT = "REFRESH TABLE doc.foo;" + + +pytest.skip("Does not work on Python 3.7", allow_module_level=True) + + +def test_crate_to_sql(cratedb_http_host, cratedb_http_port): + # Connect to database. + engine = sa.create_engine( + url=f"crate://{cratedb_http_host}:{cratedb_http_port}", + echo=True, + ) + con = engine.connect() + + # Insert data using pandas. + df = INPUT_FRAME + retval = df.to_sql(name="foo", con=con, if_exists="replace", index=False) + assert retval == -1 + + # Synchronize table content. + con.execute(sa.text(SQL_REFRESH_STATEMENT)) + + # Read back and verify data using pandas. + df = pd.read_sql(sql=sa.text(SQL_SELECT_STATEMENT), con=con) + assert_frame_equal(df, INPUT_FRAME) + + +@pytest.mark.skip(reason="Needs COLLATE and pg_table_is_visible") +def test_psycopg_to_sql(cratedb_psql_host, cratedb_psql_port): + # Connect to database. + engine = sa.create_engine( + url=f"postgresql+psycopg_relaxed://crate@{cratedb_psql_host}:{cratedb_psql_port}", + isolation_level="AUTOCOMMIT", + use_native_hstore=False, + echo=True, + ) + conn = engine.connect() + + # Insert data using pandas. + df = INPUT_FRAME + retval = df.to_sql(name="foo", con=conn, if_exists="replace", index=False) + assert retval == -1 + + # Synchronize table content. + conn.execute(sa.text(SQL_REFRESH_STATEMENT)) + + # Read back and verify data using pandas. + df = pd.read_sql(sql=sa.text(SQL_SELECT_STATEMENT), con=conn) + assert_frame_equal(df, INPUT_FRAME) + + +@pytest.mark.skip(reason="Needs COLLATE and pg_table_is_visible") +@pytest.mark.asyncio +async def test_psycopg_async_to_sql(cratedb_psql_host, cratedb_psql_port): + # Connect to database. + engine = create_async_engine( + url=f"postgresql+psycopg_relaxed://crate@{cratedb_psql_host}:{cratedb_psql_port}", + isolation_level="AUTOCOMMIT", + use_native_hstore=False, + echo=True, + ) + + # Insert data using pandas. + async with engine.begin() as conn: + df = INPUT_FRAME + retval = await conn.run_sync(to_sql_sync, df=df, name="foo", if_exists="replace", index=False) + assert retval == -1 + + # TODO: Read back dataframe and compare with original. + + +@pytest.mark.skip(reason="Needs COLLATE and pg_table_is_visible") +@pytest.mark.asyncio +async def test_asyncpg_to_sql(cratedb_psql_host, cratedb_psql_port): + # Connect to database. + engine = create_async_engine( + url=f"postgresql+asyncpg_relaxed://crate@{cratedb_psql_host}:{cratedb_psql_port}", + isolation_level="AUTOCOMMIT", + echo=True, + ) + + # Insert data using pandas. + async with engine.begin() as conn: + df = INPUT_FRAME + retval = await conn.run_sync(to_sql_sync, df=df, name="foo", if_exists="replace", index=False) + assert retval == -1 + + # TODO: Read back dataframe and compare with original. + + +def to_sql_sync(conn, df, name, **kwargs): + """ + Making df.to_sql connection the first argument to make it compatible + with conn.run_sync(), see https://stackoverflow.com/a/70861276. + """ + return df.to_sql(name=name, con=conn, **kwargs) diff --git a/tests/test_cratedb.py b/tests/test_cratedb_sqlalchemy.py similarity index 100% rename from tests/test_cratedb.py rename to tests/test_cratedb_sqlalchemy.py