diff --git a/README.md b/README.md index d6ff3fe72..8337beb00 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,25 @@ data-validation query The raw query to run against the supplied connection ``` +### Using Beta CLI Features + +There may be ocassions we want to release a new CLI feature under a Beta flag. +Any features under Beta may or may not make their way to production. However, if +there is a Beta feature you wish to use than it can be accessed using the +following. + +``` +data-validation beta --help +``` + +#### [Beta] Deploy Data Validation as a Local Service + +If you wish to use Data Validation as a Flask service, the following command +will help. This same logic is also expected to be used for Cloud Run, Cloud +Functions, and other deployment services. + +`data-validation beta deploy` + ## Query Configurations You can customize the configuration for any given validation by providing use diff --git a/data_validation/__main__.py b/data_validation/__main__.py index 255782462..ffe933457 100644 --- a/data_validation/__main__.py +++ b/data_validation/__main__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import logging import json @@ -391,6 +392,10 @@ def main(): print(run_raw_query_against_connection(args)) elif args.command == "validate": validate(args) + elif args.command == "deploy": + from data_validation import app + + app.app.run(debug=True, host="0.0.0.0", port=int(os.environ.get("PORT", 8080))) else: raise ValueError(f"Positional Argument '{args.command}' is not supported") diff --git a/data_validation/app.py b/data_validation/app.py new file mode 100644 index 000000000..3a3579a9b --- /dev/null +++ b/data_validation/app.py @@ -0,0 +1,75 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +from data_validation import data_validation +import flask +import pandas + +app = flask.Flask(__name__) + + +def _clean_dataframe(df): + rows = df.to_dict(orient="record") + for row in rows: + for key in row: + if type(row[key]) in [pandas.Timestamp]: + row[key] = str(row[key]) + + return json.dumps(rows) + + +def _get_request_content(request): + return request.json + + +def validate(config): + """Run Data Validation against the supplied config.""" + validator = data_validation.DataValidation(config) + df = validator.execute() + + return _clean_dataframe(df) + + +def main(request): + """ Handle incoming Data Validation requests. + + request (flask.Request): HTTP request object. + """ + try: + config = _get_request_content(request)["config"] + return validate(config) + except Exception as e: + return "Unknown Error: {}".format(e) + + +@app.route("/", methods=["POST"]) +def run(): + try: + config = _get_request_content(flask.request) + result = validate(config) + return str(result) + except Exception as e: + print(e) + return "Found Error: {}".format(e) + + +@app.route("/test", methods=["POST"]) +def other(): + return _get_request_content(flask.request) + + +if __name__ == "__main__": + app.run(debug=True, host="0.0.0.0", port=int(os.environ.get("PORT", 8080))) diff --git a/data_validation/cli_tools.py b/data_validation/cli_tools.py index 54fbf8b60..fd5ea6d7a 100644 --- a/data_validation/cli_tools.py +++ b/data_validation/cli_tools.py @@ -146,29 +146,37 @@ def configure_arg_parser(): parser.add_argument("--verbose", "-v", action="store_true", help="Verbose logging") - # beta feature only available in run/validate command - if "beta" in sys.argv: - parser.add_argument( - "beta", - nargs="?", - help="Beta flag to enable beta features for the tool.", - default="", - ) - subparsers = parser.add_subparsers(dest="command") - _configure_run_parser(subparsers) - _configure_validate_parser(subparsers) - else: - subparsers = parser.add_subparsers(dest="command") - _configure_validate_parser(subparsers) - _configure_run_config_parser(subparsers) - _configure_connection_parser(subparsers) - _configure_find_tables(subparsers) - _configure_raw_query(subparsers) - _configure_run_parser(subparsers) + subparsers = parser.add_subparsers(dest="command") + _configure_validate_parser(subparsers) + _configure_run_config_parser(subparsers) + _configure_connection_parser(subparsers) + _configure_find_tables(subparsers) + _configure_raw_query(subparsers) + _configure_run_parser(subparsers) + _configure_beta_parser(subparsers) return parser +def _configure_beta_parser(subparsers): + """Configure beta commands for the parser.""" + connection_parser = subparsers.add_parser( + "beta", help="Run a Beta command for new utilities and features." + ) + beta_subparsers = connection_parser.add_subparsers(dest="beta_cmd") + + _configure_run_parser(beta_subparsers) + _configure_validate_parser(beta_subparsers) + _configure_deploy(beta_subparsers) + + +def _configure_deploy(subparsers): + """Configure arguments for deploying as a service.""" + subparsers.add_parser( + "deploy", help="Deploy Data Validation as a Service (w/ Flask)" + ) + + def _configure_find_tables(subparsers): """Configure arguments for text search table matching.""" find_tables_parser = subparsers.add_parser( diff --git a/setup.py b/setup.py index 55f35f450..e910bdd0e 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ "setuptools>=34.0.0", "jellyfish==0.8.2", "tabulate==0.8.9", + "Flask==2.0.2", ] extras_require = {