feat: deploy flask app via CLI (#344)

* feat: add logic to deloy dvt as a service in the package itself

* linting

* adding beta flag to deploy

* cleaning the beta design to add it as a core visible feature and rtun properly

* cleaning the beta design to add it as a core visible feature and rtun properly

* adding README

README.md

@@ -176,6 +176,25 @@ data-validation query
           The raw query to run against the supplied connection
 ```
 
+### Using Beta CLI Features
+
+There may be ocassions we want to release a new CLI feature under a Beta flag.
+Any features under Beta may or may not make their way to production. However, if
+there is a Beta feature you wish to use than it can be accessed using the
+following.
+
+```
+data-validation beta --help
+```
+
+#### [Beta] Deploy Data Validation as a Local Service
+
+If you wish to use Data Validation as a Flask service, the following command
+will help. This same logic is also expected to be used for Cloud Run, Cloud
+Functions, and other deployment services.
+
+`data-validation beta deploy`
+
 ## Query Configurations
 
 You can customize the configuration for any given validation by providing use

data_validation/__main__.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 
 import logging
 import json
@@ -391,6 +392,10 @@ def main():
         print(run_raw_query_against_connection(args))
     elif args.command == "validate":
         validate(args)
+    elif args.command == "deploy":
+        from data_validation import app
+
+        app.app.run(debug=True, host="0.0.0.0", port=int(os.environ.get("PORT", 8080)))
     else:
         raise ValueError(f"Positional Argument '{args.command}' is not supported")
 

data_validation/app.py

@@ -0,0 +1,75 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+from data_validation import data_validation
+import flask
+import pandas
+
+app = flask.Flask(__name__)
+
+
+def _clean_dataframe(df):
+    rows = df.to_dict(orient="record")
+    for row in rows:
+        for key in row:
+            if type(row[key]) in [pandas.Timestamp]:
+                row[key] = str(row[key])
+
+    return json.dumps(rows)
+
+
+def _get_request_content(request):
+    return request.json
+
+
+def validate(config):
+    """Run Data Validation against the supplied config."""
+    validator = data_validation.DataValidation(config)
+    df = validator.execute()
+
+    return _clean_dataframe(df)
+
+
+def main(request):
+    """ Handle incoming Data Validation requests.
+
+        request (flask.Request): HTTP request object.
+    """
+    try:
+        config = _get_request_content(request)["config"]
+        return validate(config)
+    except Exception as e:
+        return "Unknown Error: {}".format(e)
+
+
+@app.route("/", methods=["POST"])
+def run():
+    try:
+        config = _get_request_content(flask.request)
+        result = validate(config)
+        return str(result)
+    except Exception as e:
+        print(e)
+        return "Found Error: {}".format(e)
+
+
+@app.route("/test", methods=["POST"])
+def other():
+    return _get_request_content(flask.request)
+
+
+if __name__ == "__main__":
+    app.run(debug=True, host="0.0.0.0", port=int(os.environ.get("PORT", 8080)))

data_validation/cli_tools.py

@@ -146,29 +146,37 @@ def configure_arg_parser():
 
     parser.add_argument("--verbose", "-v", action="store_true", help="Verbose logging")
 
-    # beta feature only available in run/validate command
-    if "beta" in sys.argv:
-        parser.add_argument(
-            "beta",
-            nargs="?",
-            help="Beta flag to enable beta features for the tool.",
-            default="",
-        )
-        subparsers = parser.add_subparsers(dest="command")
-        _configure_run_parser(subparsers)
-        _configure_validate_parser(subparsers)
-    else:
-        subparsers = parser.add_subparsers(dest="command")
-        _configure_validate_parser(subparsers)
-        _configure_run_config_parser(subparsers)
-        _configure_connection_parser(subparsers)
-        _configure_find_tables(subparsers)
-        _configure_raw_query(subparsers)
-        _configure_run_parser(subparsers)
+    subparsers = parser.add_subparsers(dest="command")
+    _configure_validate_parser(subparsers)
+    _configure_run_config_parser(subparsers)
+    _configure_connection_parser(subparsers)
+    _configure_find_tables(subparsers)
+    _configure_raw_query(subparsers)
+    _configure_run_parser(subparsers)
+    _configure_beta_parser(subparsers)
 
     return parser
 
 
+def _configure_beta_parser(subparsers):
+    """Configure beta commands for the parser."""
+    connection_parser = subparsers.add_parser(
+        "beta", help="Run a Beta command for new utilities and features."
+    )
+    beta_subparsers = connection_parser.add_subparsers(dest="beta_cmd")
+
+    _configure_run_parser(beta_subparsers)
+    _configure_validate_parser(beta_subparsers)
+    _configure_deploy(beta_subparsers)
+
+
+def _configure_deploy(subparsers):
+    """Configure arguments for deploying as a service."""
+    subparsers.add_parser(
+        "deploy", help="Deploy Data Validation as a Service (w/ Flask)"
+    )
+
+
 def _configure_find_tables(subparsers):
     """Configure arguments for text search table matching."""
     find_tables_parser = subparsers.add_parser(

setup.py

@@ -50,6 +50,7 @@
     "setuptools>=34.0.0",
     "jellyfish==0.8.2",
     "tabulate==0.8.9",
+    "Flask==2.0.2",
 ]
 
 extras_require = {