test: Support local integration tests for Teradata, Postgres and SQL Server

tests/system/conftest.py

@@ -23,6 +23,10 @@
 ALPHABET = "abcdefghijklmnopqrstuvwxyz"
 
 
+def pytest_addoption(parser):
+    parser.addoption("--no-cloud-sql", action="store_const", const=True)
+
+
 @pytest.fixture(scope="module")
 def bigquery_client():
     project_id = os.environ["PROJECT_ID"]

tests/system/data_sources/test_postgres.py

@@ -19,38 +19,44 @@
 
 import os
 
+import pytest
+
 # Local testing requires the Cloud SQL Proxy.
 # https://cloud.google.com/sql/docs/postgres/connect-admin-proxy
 
-
+# Cloud SQL proxy listens to localhost
+POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
 POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD")
 PROJECT_ID = os.getenv("PROJECT_ID")
 
 
-def test_postgres_count():
-    """ Test count validation on Postgres instance """
-    postgres_instance = CloudSQLResourceManager(
-        PROJECT_ID,
-        "POSTGRES_12",
-        "data-validator-postgres12",
-        POSTGRES_PASSWORD,
-        database_id="guestbook",
-        assign_public_ip=True,
-        authorized_networks=None,
-        cpu=1,
-        memory="4GB",
-        enable_bin_logs=False,
-        already_exists=True,
-    )
+@pytest.fixture
+def cloud_sql(request):
+    if not request.config.getoption("--no-cloud-sql"):
+        postgres_instance = CloudSQLResourceManager(
+            PROJECT_ID,
+            "POSTGRES_12",
+            "data-validator-postgres12",
+            POSTGRES_PASSWORD,
+            database_id="guestbook",
+            assign_public_ip=True,
+            authorized_networks=None,
+            cpu=1,
+            memory="4GB",
+            enable_bin_logs=False,
+            already_exists=True,
+        )
 
-    # If instance already exists, returns host IP and does not add new data
-    postgres_instance.setup()
-    postgres_instance.add_data("gs://pso-kokoro-resources/postgres_data.sql")
+        # If instance already exists, returns host IP and does not add new data
+        postgres_instance.setup()
+        postgres_instance.add_data("gs://pso-kokoro-resources/postgres_data.sql")
 
-    # Cloud SQL proxy listens to localhost
+
+def test_postgres_count(cloud_sql):
+    """ Test count validation on Postgres instance """
     conn = {
         "source_type": "Postgres",
-        "host": "localhost",
+        "host": POSTGRES_HOST,
         "user": "postgres",
         "password": POSTGRES_PASSWORD,
         "port": 5432,

tests/system/data_sources/test_sql_server.py

@@ -19,39 +19,46 @@
 
 import os
 
+import pytest
+
 # Local testing requires the Cloud SQL Proxy.
 # https://cloud.google.com/sql/docs/sqlserver/connect-admin-proxy
 
-
+# Cloud SQL Proxy listens on localhost
+SQL_SERVER_HOST = os.getenv("SQL_SERVER_HOST", "localhost")
+SQL_SERVER_USER = os.getenv("SQL_SERVER_USER", "sqlserver")
 SQL_SERVER_PASSWORD = os.getenv("SQL_SERVER_PASSWORD")
 PROJECT_ID = os.getenv("PROJECT_ID")
 
 
-def test_sql_server_count():
-    """ Test count validation on SQL Server instance """
-    mssql_instance = CloudSQLResourceManager(
-        PROJECT_ID,
-        "SQLSERVER_2017_STANDARD",
-        "data-validator-mssql2017",
-        SQL_SERVER_PASSWORD,
-        database_id="guestbook",
-        assign_public_ip=True,
-        authorized_networks=None,
-        cpu=1,
-        memory="4GB",
-        enable_bin_logs=False,
-        already_exists=True,
-    )
+@pytest.fixture
+def cloud_sql(request):
+    if not request.config.getoption("--no-cloud-sql"):
+        mssql_instance = CloudSQLResourceManager(
+            PROJECT_ID,
+            "SQLSERVER_2017_STANDARD",
+            "data-validator-mssql2017",
+            SQL_SERVER_PASSWORD,
+            database_id="guestbook",
+            assign_public_ip=True,
+            authorized_networks=None,
+            cpu=1,
+            memory="4GB",
+            enable_bin_logs=False,
+            already_exists=True,
+        )
 
-    # If instance already exists, returns host IP and does not add new data
-    mssql_instance.setup()
-    mssql_instance.add_data("gs://pso-kokoro-resources/mssql_data.sql")
+        # If instance already exists, returns host IP and does not add new data
+        mssql_instance.setup()
+        mssql_instance.add_data("gs://pso-kokoro-resources/mssql_data.sql")
 
-    # Cloud SQL Proxy listens on localhost
+
+def test_sql_server_count(cloud_sql):
+    """ Test count validation on SQL Server instance """
     conn = {
         "source_type": "MSSQL",
-        "host": "127.0.0.1",
-        "user": "sqlserver",
+        "host": SQL_SERVER_HOST,
+        "user": SQL_SERVER_USER,
         "password": SQL_SERVER_PASSWORD,
         "port": 1433,
         "database": "guestbook",

tests/system/data_sources/test_teradata.py

@@ -16,14 +16,15 @@
 
 from data_validation import data_validation, consts
 
+TERADATA_USER = os.getenv("TERADATA_USER", "udf")
 TERADATA_PASSWORD = os.getenv("TERADATA_PASSWORD")
 TERADATA_HOST = os.getenv("TERADATA_HOST")
 PROJECT_ID = os.getenv("PROJECT_ID")
 
 conn = {
     "source_type": "Teradata",
     "host": TERADATA_HOST,
-    "user_name": "udf",
+    "user_name": TERADATA_USER,
     "password": TERADATA_PASSWORD,
     "port": 1025,
 }