-
Notifications
You must be signed in to change notification settings - Fork 19
/
19-3 Spark configuration example
1 lines (1 loc) · 1.12 KB
/
19-3 Spark configuration example
1
{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"19-3 Spark configuration example","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyPxM0ifSfd0wMF7ZMrrNX0j"},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"code","metadata":{"id":"JMbCdY5jiEoI"},"source":["# Spark configuration example\n","# author: Gressling, T\n","# license: MIT License # code: github.com/gressling/examples\n","# activity: single example # index: 19-3"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"q3uP8P-jiN4r"},"source":["conf = (SparkConf()\n"," .setAppName('COVID-19-Dataset')\n"," .setMaster('spark://head_node:56887')\n"," .setAll([('spark.executor.memory', '80g'),\n"," ('spark.executor.cores', '12'),\n"," ('spark.cores.max', '12'),\n"," ('spark.driver.memory', '80g')]))\n","conf.set('spark.authenticate', True)\n","conf.set('spark.authenticate.secret', 'secret-key')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"3joh_svTiRJq"},"source":["sc = SparkContext(conf=conf)\n","sql_context = SQLContext(sc)"],"execution_count":null,"outputs":[]}]}