From 7094e7e311ddfff3869fd0a8fbd0d26012774804 Mon Sep 17 00:00:00 2001 From: AJ <95496513+ajwelch4@users.noreply.github.com> Date: Tue, 8 Feb 2022 14:02:02 -0500 Subject: [PATCH] test: Support local integration tests for Teradata, Postgres and SQL Server (#364) * test: get Teradata user name from TERADATA_USER env var * test: add --no-cloud-sql flag to pytest options * test: instantiate CloudSQLResourceManager in a fixture when --no-cloud-sql is not passed * test: optionally get Postgres host from POSTGRES_HOST env var * test: optionally get SQL Server host from SQL_SERVER_HOST env var * test: optionally get SQL server user from SQL_SERVER_USER env var Co-authored-by: A.J. Welch --- tests/system/conftest.py | 4 ++ tests/system/data_sources/test_postgres.py | 48 ++++++++++-------- tests/system/data_sources/test_sql_server.py | 51 +++++++++++--------- tests/system/data_sources/test_teradata.py | 3 +- 4 files changed, 62 insertions(+), 44 deletions(-) diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 44dccf00e..a8bd52b88 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -23,6 +23,10 @@ ALPHABET = "abcdefghijklmnopqrstuvwxyz" +def pytest_addoption(parser): + parser.addoption("--no-cloud-sql", action="store_const", const=True) + + @pytest.fixture(scope="module") def bigquery_client(): project_id = os.environ["PROJECT_ID"] diff --git a/tests/system/data_sources/test_postgres.py b/tests/system/data_sources/test_postgres.py index 0a5829516..b51a9c928 100644 --- a/tests/system/data_sources/test_postgres.py +++ b/tests/system/data_sources/test_postgres.py @@ -19,38 +19,44 @@ import os +import pytest + # Local testing requires the Cloud SQL Proxy. # https://cloud.google.com/sql/docs/postgres/connect-admin-proxy - +# Cloud SQL proxy listens to localhost +POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost") POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD") PROJECT_ID = os.getenv("PROJECT_ID") -def test_postgres_count(): - """ Test count validation on Postgres instance """ - postgres_instance = CloudSQLResourceManager( - PROJECT_ID, - "POSTGRES_12", - "data-validator-postgres12", - POSTGRES_PASSWORD, - database_id="guestbook", - assign_public_ip=True, - authorized_networks=None, - cpu=1, - memory="4GB", - enable_bin_logs=False, - already_exists=True, - ) +@pytest.fixture +def cloud_sql(request): + if not request.config.getoption("--no-cloud-sql"): + postgres_instance = CloudSQLResourceManager( + PROJECT_ID, + "POSTGRES_12", + "data-validator-postgres12", + POSTGRES_PASSWORD, + database_id="guestbook", + assign_public_ip=True, + authorized_networks=None, + cpu=1, + memory="4GB", + enable_bin_logs=False, + already_exists=True, + ) - # If instance already exists, returns host IP and does not add new data - postgres_instance.setup() - postgres_instance.add_data("gs://pso-kokoro-resources/postgres_data.sql") + # If instance already exists, returns host IP and does not add new data + postgres_instance.setup() + postgres_instance.add_data("gs://pso-kokoro-resources/postgres_data.sql") - # Cloud SQL proxy listens to localhost + +def test_postgres_count(cloud_sql): + """ Test count validation on Postgres instance """ conn = { "source_type": "Postgres", - "host": "localhost", + "host": POSTGRES_HOST, "user": "postgres", "password": POSTGRES_PASSWORD, "port": 5432, diff --git a/tests/system/data_sources/test_sql_server.py b/tests/system/data_sources/test_sql_server.py index 3a2cc2ad2..da11768fe 100644 --- a/tests/system/data_sources/test_sql_server.py +++ b/tests/system/data_sources/test_sql_server.py @@ -19,39 +19,46 @@ import os +import pytest + # Local testing requires the Cloud SQL Proxy. # https://cloud.google.com/sql/docs/sqlserver/connect-admin-proxy - +# Cloud SQL Proxy listens on localhost +SQL_SERVER_HOST = os.getenv("SQL_SERVER_HOST", "localhost") +SQL_SERVER_USER = os.getenv("SQL_SERVER_USER", "sqlserver") SQL_SERVER_PASSWORD = os.getenv("SQL_SERVER_PASSWORD") PROJECT_ID = os.getenv("PROJECT_ID") -def test_sql_server_count(): - """ Test count validation on SQL Server instance """ - mssql_instance = CloudSQLResourceManager( - PROJECT_ID, - "SQLSERVER_2017_STANDARD", - "data-validator-mssql2017", - SQL_SERVER_PASSWORD, - database_id="guestbook", - assign_public_ip=True, - authorized_networks=None, - cpu=1, - memory="4GB", - enable_bin_logs=False, - already_exists=True, - ) +@pytest.fixture +def cloud_sql(request): + if not request.config.getoption("--no-cloud-sql"): + mssql_instance = CloudSQLResourceManager( + PROJECT_ID, + "SQLSERVER_2017_STANDARD", + "data-validator-mssql2017", + SQL_SERVER_PASSWORD, + database_id="guestbook", + assign_public_ip=True, + authorized_networks=None, + cpu=1, + memory="4GB", + enable_bin_logs=False, + already_exists=True, + ) - # If instance already exists, returns host IP and does not add new data - mssql_instance.setup() - mssql_instance.add_data("gs://pso-kokoro-resources/mssql_data.sql") + # If instance already exists, returns host IP and does not add new data + mssql_instance.setup() + mssql_instance.add_data("gs://pso-kokoro-resources/mssql_data.sql") - # Cloud SQL Proxy listens on localhost + +def test_sql_server_count(cloud_sql): + """ Test count validation on SQL Server instance """ conn = { "source_type": "MSSQL", - "host": "127.0.0.1", - "user": "sqlserver", + "host": SQL_SERVER_HOST, + "user": SQL_SERVER_USER, "password": SQL_SERVER_PASSWORD, "port": 1433, "database": "guestbook", diff --git a/tests/system/data_sources/test_teradata.py b/tests/system/data_sources/test_teradata.py index 2bcd8e707..7acd5ff44 100644 --- a/tests/system/data_sources/test_teradata.py +++ b/tests/system/data_sources/test_teradata.py @@ -16,6 +16,7 @@ from data_validation import data_validation, consts +TERADATA_USER = os.getenv("TERADATA_USER", "udf") TERADATA_PASSWORD = os.getenv("TERADATA_PASSWORD") TERADATA_HOST = os.getenv("TERADATA_HOST") PROJECT_ID = os.getenv("PROJECT_ID") @@ -23,7 +24,7 @@ conn = { "source_type": "Teradata", "host": TERADATA_HOST, - "user_name": "udf", + "user_name": TERADATA_USER, "password": TERADATA_PASSWORD, "port": 1025, }