Updated some of the Exercise files

Author: matthew-mcateer

.ipynb_checkpoints/Pyspark Installation Testing Notebook-checkpoint.ipynb

@@ -0,0 +1,139 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Estimating the Value of PI with Pyspark"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'pyspark'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "\u001b[1;32m<ipython-input-2-c15ae3402d12>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mpyspark\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'pyspark'"
+     ]
+    }
+   ],
+   "source": [
+    "import pyspark"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'SparkContext' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[1;32m<ipython-input-1-357ae8606ddf>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# Reason why we have the getOrCreate code\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m# http://stackoverflow.com/questions/28999332/how-to-access-sparkcontext-in-pyspark-script\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0msc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSparkContext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetOrCreate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[1;31mNameError\u001b[0m: name 'SparkContext' is not defined"
+     ]
+    }
+   ],
+   "source": [
+    "# Reason why we have the getOrCreate code\n",
+    "# http://stackoverflow.com/questions/28999332/how-to-access-sparkcontext-in-pyspark-script\n",
+    "sc = SparkContext.getOrCreate()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "TOTAL = 1000000\n",
+    "dots = sc.parallelize([2.0 * np.random.random(2) - 1.0 for i in range(TOTAL)]).cache()\n",
+    "print(\"Number of random points:\", dots.count())\n",
+    "\n",
+    "stats = dots.stats()\n",
+    "print('Mean:', stats.mean())\n",
+    "print('stdev:', stats.stdev())\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "%matplotlib inline\n",
+    "from operator import itemgetter\n",
+    "from matplotlib import pyplot as plt\n",
+    "\n",
+    "plt.figure(figsize = (10, 5))\n",
+    "\n",
+    "# Plot 1\n",
+    "plt.subplot(1, 2, 1)\n",
+    "plt.xlim((-1.0, 1.0))\n",
+    "plt.ylim((-1.0, 1.0))\n",
+    "\n",
+    "sample = dots.sample(False, 0.01)\n",
+    "X = sample.map(itemgetter(0)).collect()\n",
+    "Y = sample.map(itemgetter(1)).collect()\n",
+    "plt.scatter(X, Y)\n",
+    "\n",
+    "# Plot 2\n",
+    "plt.subplot(1, 2, 2)\n",
+    "plt.xlim((-1.0, 1.0))\n",
+    "plt.ylim((-1.0, 1.0))\n",
+    "\n",
+    "inCircle = lambda v: np.linalg.norm(v) <= 1.0\n",
+    "dotsIn = sample.filter(inCircle).cache()\n",
+    "dotsOut = sample.filter(lambda v: not inCircle(v)).cache()\n",
+    "\n",
+    "# inside circle\n",
+    "Xin = dotsIn.map(itemgetter(0)).collect()\n",
+    "Yin = dotsIn.map(itemgetter(1)).collect()\n",
+    "plt.scatter(Xin, Yin, color = 'r')\n",
+    "\n",
+    "# outside circle\n",
+    "Xout = dotsOut.map(itemgetter(0)).collect()\n",
+    "Yout = dotsOut.map(itemgetter(1)).collect()\n",
+    "plt.scatter(Xout, Yout)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}