test: Core data types integration tests (#766)

* test: Add dvt_core_types test table SQL for BigQuery, Hive, MySQL, Oracle, PostgreSQL

* test: Add dvt_core_types test table SQL for SQL Server and Teradata

* test: Add Oracle integration tests for dvt_core_types table

* test: Add (disabled) SQL Server integration tests for dvt_core_types table

* test: Add Teradata integration tests for dvt_core_types table

* test: Add BigQuery integration tests for dvt_core_types table

* test: Add Hive integration tests for dvt_core_types table

* test: Add Spanner integration tests for dvt_core_types table

* test: Add PostgreSQL integration tests for dvt_core_types table

* test: Reference github issues in disabled tests

* test: Fix Spanner integration tests

* test: Fix Spanner integration tests

* test: Disable timestamp column tests for Spanner

* test: Add MySQL integration tests for dvt_core_types table

* test: Experiment with Cloud SQL Proxy for MySQL integration tests

* test: Experiment with Cloud SQL Proxy for MySQL integration tests

* test: Move other MySQL integration test tables to pso_data_validator database

* test: Add PostgreSQL integration tests for dvt_core_types table

* test: Remove redundant TODO from mysql integration tests

ci/build.sh

@@ -25,11 +25,11 @@ python3.7 -m pip install --upgrade --quiet nox pip
 python3.7 -m nox --version
 
 # When NOX_SESSION is set, it only runs the specified session
-if [[ -n "${NOX_SESSION:-}" &&  ( "$NOX_SESSION" == "integration_postgres" || "$NOX_SESSION" == "integration_sql_server" ) ]]; then
+if [[ -n "${NOX_SESSION:-}" &&  ( "$NOX_SESSION" == "integration_postgres" || "$NOX_SESSION" == "integration_sql_server" || "$NOX_SESSION" == "integration_mysql" ) ]]; then
     ./cloud_sql_proxy -instances="$CLOUD_SQL_CONNECTION" & python3.7 -m nox --error-on-missing-interpreters -s "${NOX_SESSION:-}"
 elif [[ -n "${NOX_SESSION:-}" ]]; then
     python3.7 -m nox --error-on-missing-interpreters -s "${NOX_SESSION:-}"
-else 
+else
     echo "NOX_SESSION env var not set"
     exit 1
 fi

cloudbuild.yaml

@@ -38,8 +38,9 @@ steps:
   args: ['bash', './ci/build.sh']
   env:
   - 'NOX_SESSION=integration_mysql'
-  - 'MYSQL_HOST=35.184.142.16'
-  - 'MYSQL_PASSWORD=password'
+  - 'PROJECT_ID=pso-kokoro-resources'
+  - 'MYSQL_PASSWORD=dvt'
+  - 'CLOUD_SQL_CONNECTION=pso-kokoro-resources:us-central1:data-validator-mysql=tcp:3306'
   waitFor: ['-']
 - id: integration_postgres
   name: 'gcr.io/cloud-devrel-public-resources/python-multi'

noxfile.py

@@ -124,7 +124,7 @@ def integration_mysql(session):
     _setup_session_requirements(session, extra_packages=[])
 
     test_path = "tests/system/data_sources/test_mysql.py"
-    expected_env_vars = ["MYSQL_HOST", "MYSQL_PASSWORD"]
+    expected_env_vars = ["PROJECT_ID", "MYSQL_PASSWORD", "CLOUD_SQL_CONNECTION"]
     for env_var in expected_env_vars:
         if not os.environ.get(env_var, ""):
             raise Exception("Expected Env Var: %s" % env_var)

tests/resources/bigquery_test_tables.sql

@@ -8,3 +8,46 @@ SELECT
     CAST(2 AS STRING) text_type,
     CAST('2021-01-01 00:00:00' AS TIMESTAMP) timestamp_type
 
+
+-- Core data types test table, to be kept in sync with same table in other SQL engines
+CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_core_types`
+(   id              INT NOT NULL
+,   col_int8        TINYINT
+,   col_int16       SMALLINT
+,   col_int32       INT
+,   col_int64       INT64
+,   col_dec_20      NUMERIC(20)
+,   col_dec_38      BIGNUMERIC(38)
+,   col_dec_10_2    NUMERIC(10,2)
+,   col_float32     FLOAT64
+,   col_float64     FLOAT64
+,   col_varchar_30  STRING(30)
+,   col_char_2      STRING(2)
+,   col_string      STRING
+,   col_date        DATE
+,   col_datetime    DATETIME
+,   col_tstz        TIMESTAMP
+) OPTIONS (description='Core data types integration test table');
+
+INSERT INTO `pso_data_validator`.`dvt_core_types` VALUES
+(1,1,1,1,1
+ ,NUMERIC '12345678901234567890'
+ ,BIGNUMERIC '1234567890123456789012345'
+ ,NUMERIC '123.11',123456.1,12345678.1
+ ,'Hello DVT','A ','Hello DVT'
+ ,DATE '1970-01-01',DATETIME '1970-01-01 00:00:01'
+ ,TIMESTAMP '1970-01-01 00:00:01-01:00')
+,(2,2,2,2,2
+ ,NUMERIC '12345678901234567890'
+ ,BIGNUMERIC '1234567890123456789012345'
+ ,NUMERIC '123.22',123456.2,12345678.2
+ ,'Hello DVT','B ','Hello DVT'
+ ,DATE '1970-01-02',DATETIME '1970-01-02 00:00:02'
+ ,TIMESTAMP '1970-01-02 00:00:02-02:00')
+,(3,3,3,3,3
+ ,NUMERIC '12345678901234567890'
+ ,BIGNUMERIC '1234567890123456789012345'
+ ,NUMERIC '123.33',123456.3,12345678.3
+ ,'Hello DVT','C ','Hello DVT'
+ ,DATE '1970-01-03',DATETIME '1970-01-03 00:00:03'
+ ,TIMESTAMP '1970-01-03 00:00:03-03:00');

tests/resources/hive_test_tables.sql

@@ -0,0 +1,50 @@
+-- Copyright 2023 Google LLC
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE DATABASE IF NOT EXISTS pso_data_validator;
+DROP TABLE IF EXISTS pso_data_validator.dvt_core_types;
+CREATE TABLE pso_data_validator.dvt_core_types
+(   id              int NOT NULL
+,   col_int8        tinyint
+,   col_int16       smallint
+,   col_int32       int
+,   col_int64       bigint
+,   col_dec_20      decimal(20)
+,   col_dec_38      decimal(38)
+,   col_dec_10_2    decimal(10,2)
+,   col_float32     float
+,   col_float64     double
+,   col_varchar_30  varchar(30)
+,   col_char_2      char(2)
+,   col_string      string
+,   col_date        date
+,   col_datetime    timestamp
+,   col_tstz        timestamp
+)
+STORED AS ORC
+TBLPROPERTIES ('comment'='Core data types integration test table');
+
+INSERT INTO pso_data_validator.dvt_core_types VALUES
+(1,1,1,1,1
+ ,12345678901234567890,1234567890123456789012345,123.11,123456.1,12345678.1
+ ,'Hello DVT','A ','Hello DVT'
+ ,'1970-01-01','1970-01-01 00:00:01','1970-01-01 01:00:01')
+,(2,2,2,2,2
+ ,12345678901234567890,1234567890123456789012345,123.22,123456.2,12345678.2
+ ,'Hello DVT','B ','Hello DVT'
+ ,'1970-01-02','1970-01-02 00:00:02','1970-01-02 02:00:02')
+,(3,3,3,3,3
+ ,12345678901234567890,1234567890123456789012345,123.33,123456.3,12345678.3
+ ,'Hello DVT','C ','Hello DVT'
+ ,'1970-01-03','1970-01-03 00:00:03','1970-01-03 03:00:03');

tests/resources/mysql_test_tables.sql

@@ -0,0 +1,91 @@
+-- Copyright 2023 Google LLC
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE DATABASE pso_data_validator;
+
+DROP TABLE IF EXISTS `pso_data_validator`.`entries`;
+CREATE TABLE `pso_data_validator`.`entries` (
+  `guestName` varchar(255) DEFAULT NULL,
+  `content` varchar(255) DEFAULT NULL,
+  `entryID` int(11) NOT NULL AUTO_INCREMENT,
+  PRIMARY KEY (`entryID`)
+) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8;
+
+INSERT INTO `pso_data_validator`.`entries` VALUES
+('first guest','I got here!',1),('second guest','Me too!',2),
+('This Guy','More data Coming',3),('Joe','Me too!',4),
+('John','I got here!',5),('Alex','Me too!',6),('Zoe','zippp!',7);
+COMMIT;
+
+DROP TABLE `pso_data_validator`.`test_data_types_mysql_row`;
+CREATE TABLE `pso_data_validator`.`test_data_types_mysql_row` (
+  `serial_col` int(11) NOT NULL AUTO_INCREMENT,
+  `int_col` int(3) DEFAULT NULL,
+  `text_col` char(100) DEFAULT NULL,
+  `char_col` char(30) DEFAULT NULL,
+  `varchar_col` varchar(255) DEFAULT NULL,
+  `float_col` float(5,2) DEFAULT NULL,
+  `decimal_col` decimal(5,2) DEFAULT NULL,
+  `datetime_col` datetime DEFAULT NULL,
+  `date_col` date DEFAULT NULL,
+  PRIMARY KEY (`serial_col`)
+) ENGINE=InnoDB AUTO_INCREMENT=11 DEFAULT CHARSET=utf8;
+
+INSERT INTO `pso_data_validator`.`test_data_types_mysql_row` VALUES
+(1,10,'row1_text','1','row1_varchar',10.10,10.10,'2016-06-22 19:10:25','2020-01-01'),
+(2,20,'row2_text','2','row2_varchar',20.20,20.20,'2016-06-22 19:20:25','2020-02-02'),
+(3,30,'row3_text','3','row3_varchar',30.20,30.20,'2016-06-22 19:30:25','2020-03-03'),
+(4,40,'row4_text','4','row4_varchar',40.20,40.40,'2016-06-22 19:30:45','2020-04-04'),
+(5,50,'row5_text','5','row5_varchar',50.20,50.40,'2016-06-22 19:50:44','2020-05-05'),
+(6,60,'row6_text','6','row6_varchar',60.20,60.40,'2016-01-22 19:51:44','2020-06-06'),
+(7,70,'row7_text','7','row7_varchar',70.20,70.40,'2017-01-22 19:51:44','2020-06-07'),
+(8,80,'row8_text','8','row8_varchar',80.20,80.40,'2018-01-22 19:51:44','2020-06-08'),
+(9,90,'row9_text','9','row9_varchar',90.20,90.40,'2019-01-22 19:51:44','2020-06-09'),
+(10,100,'row10_text','1','row10_varchar',100.20,100.40,'2020-01-22 19:51:44','2021-06-09');
+COMMIT;
+
+DROP TABLE `pso_data_validator`.`dvt_core_types`;
+CREATE TABLE `pso_data_validator`.`dvt_core_types`
+(   id              int NOT NULL PRIMARY KEY
+,   col_int8        tinyint
+,   col_int16       smallint
+,   col_int32       int
+,   col_int64       bigint
+,   col_dec_20      decimal(20)
+,   col_dec_38      decimal(38)
+,   col_dec_10_2    decimal(10,2)
+,   col_float32     float
+,   col_float64     double
+,   col_varchar_30  varchar(30)
+,   col_char_2      char(2)
+,   col_string      varchar(21000)
+,   col_date        date
+,   col_datetime    datetime(3)
+,   col_tstz        timestamp(3)
+) COMMENT='Core data types integration test table';
+
+SET time_zone = '+00:00';
+INSERT INTO `pso_data_validator`.`dvt_core_types` VALUES
+(1,1,1,1,1
+ ,12345678901234567890,1234567890123456789012345,123.11,123456.1,12345678.1
+ ,'Hello DVT','A ','Hello DVT'
+ ,'1970-01-01','1970-01-01 00:00:01','1970-01-01 01:00:01'),
+(2,2,2,2,2
+ ,12345678901234567890,1234567890123456789012345,123.22,123456.2,12345678.2
+ ,'Hello DVT','B ','Hello DVT'
+ ,'1970-01-02','1970-01-02 00:00:02','1970-01-02 02:00:02'),
+(3,3,3,3,3
+ ,12345678901234567890,1234567890123456789012345,123.33,123456.3,12345678.3
+ ,'Hello DVT','C ','Hello DVT'
+ ,'1970-01-03','1970-01-03 00:00:03','1970-01-03 03:00:03');

tests/resources/oracle_test_tables.sql

@@ -0,0 +1,69 @@
+-- Copyright 2023 Google LLC
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE USER pso_data_validator IDENTIFIED BY pso_data_validator
+DEFAULT TABLESPACE users QUOTA UNLIMITED ON users
+ACCOUNT LOCK;
+
+-- Existing Oracle integration test table
+DROP TABLE pso_data_validator.items_price;
+CREATE TABLE pso_data_validator.items_price
+(ITEM_ID  NUMBER(10) NOT NULL
+,PRICE    NUMBER
+,CITY     VARCHAR2(50));
+INSERT INTO pso_data_validator.items_price VALUES (2,200,'xyz');
+INSERT INTO pso_data_validator.items_price VALUES (3,300,'xyz');
+COMMIT;
+
+-- Core data types test table, to be kept in sync with same table in other SQL engines
+DROP TABLE pso_data_validator.dvt_core_types;
+CREATE TABLE pso_data_validator.dvt_core_types
+(   id              NUMBER(8) NOT NULL PRIMARY KEY
+,   col_int8        NUMBER(2)
+,   col_int16       NUMBER(4)
+,   col_int32       NUMBER(9)
+,   col_int64       NUMBER(18)
+,   col_dec_20      NUMBER(20)
+,   col_dec_38      NUMBER(38)
+,   col_dec_10_2    NUMBER(10,2)
+,   col_float32     BINARY_FLOAT
+,   col_float64     BINARY_DOUBLE
+,   col_varchar_30  VARCHAR(30)
+,   col_char_2      CHAR(2)
+,   col_string      VARCHAR(4000)
+,   col_date        DATE
+,   col_datetime    TIMESTAMP(3)
+,   col_tstz        TIMESTAMP(3) WITH TIME ZONE
+);
+COMMENT ON TABLE pso_data_validator.dvt_core_types IS 'Core data types integration test table';
+
+INSERT INTO pso_data_validator.dvt_core_types VALUES
+(1,1,1,1,1
+,12345678901234567890,1234567890123456789012345,123.11,123456.1,12345678.1
+,'Hello DVT','A ','Hello DVT'
+,DATE'1970-01-01',TIMESTAMP'1970-01-01 00:00:01'
+,to_timestamp_tz('1970-01-01 00:00:01 -01:00','YYYY-MM-DD HH24:MI:SS TZH:TZM'));
+INSERT INTO pso_data_validator.dvt_core_types VALUES
+(2,2,2,2,2
+,12345678901234567890,1234567890123456789012345,123.22,123456.2,12345678.2
+,'Hello DVT','B','Hello DVT'
+,DATE'1970-01-02',TIMESTAMP'1970-01-02 00:00:02'
+,to_timestamp_tz('1970-01-02 00:00:02 -02:00','YYYY-MM-DD HH24:MI:SS TZH:TZM'));
+INSERT INTO pso_data_validator.dvt_core_types VALUES
+(3,3,3,3,3
+,12345678901234567890,1234567890123456789012345,123.33,123456.3,12345678.3
+,'Hello DVT','C ','Hello DVT'
+,DATE'1970-01-03',TIMESTAMP'1970-01-03 00:00:03'
+,to_timestamp_tz('1970-01-03 00:00:03 -03:00','YYYY-MM-DD HH24:MI:SS TZH:TZM'));
+COMMIT;

tests/resources/postgresql_test_tables.sql

@@ -0,0 +1,52 @@
+-- Copyright 2023 Google LLC
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE SCHEMA pso_data_validator;
+DROP TABLE pso_data_validator.dvt_core_types;
+CREATE TABLE pso_data_validator.dvt_core_types
+(   id              int NOT NULL PRIMARY KEY
+,   col_int8        smallint
+,   col_int16       smallint
+,   col_int32       int
+,   col_int64       bigint
+,   col_dec_20      decimal(20)
+,   col_dec_38      decimal(38)
+,   col_dec_10_2    decimal(10,2)
+,   col_float32     real
+,   col_float64     double precision
+,   col_varchar_30  varchar(30)
+,   col_char_2      char(2)
+,   col_string      text
+,   col_date        date
+,   col_datetime    timestamp(3)
+,   col_tstz        timestamp(3) with time zone
+);
+COMMENT ON TABLE pso_data_validator.dvt_core_types IS 'Core data types integration test table';
+
+INSERT INTO pso_data_validator.dvt_core_types VALUES
+(1,1,1,1,1
+ ,12345678901234567890,1234567890123456789012345,123.11,123456.1,12345678.1
+ ,'Hello DVT','A ','Hello DVT'
+ ,DATE'1970-01-01',TIMESTAMP'1970-01-01 00:00:01'
+ ,TIMESTAMP WITH TIME ZONE'1970-01-01 00:00:01 -01:00'),
+(2,2,2,2,2
+ ,12345678901234567890,1234567890123456789012345,123.22,123456.2,12345678.2
+ ,'Hello DVT','B','Hello DVT'
+ ,DATE'1970-01-02',TIMESTAMP'1970-01-02 00:00:02'
+ ,TIMESTAMP WITH TIME ZONE'1970-01-02 00:00:02 -02:00'),
+(3,3,3,3,3
+ ,12345678901234567890,1234567890123456789012345,123.33,123456.3,12345678.3
+ ,'Hello DVT','C ','Hello DVT'
+ ,DATE'1970-01-03',TIMESTAMP'1970-01-03 00:00:03'
+ ,TIMESTAMP WITH TIME ZONE'1970-01-03 00:00:03 -03:00');