hands-on/filesystem_interaction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# File system interaction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-14T08:44:12.881704Z",
     "start_time": "2018-11-14T08:44:12.857600Z"
    }
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import fileinput\n",
    "import os\n",
    "import pathlib\n",
    "from pathlib import Path\n",
    "import re\n",
    "import shutil\n",
    "import stat\n",
    "import tempfile"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Contents of directories & properties"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Iterate over the contents of the current working directory, and for each file, print its name, size, and last access time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.498841Z",
     "start_time": "2018-11-13T12:13:11.483568Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shell_interaction.ipynb: 13615 bytes, last accessed: 2019-12-03 07:18:08\n",
      "system_information.ipynb: 8870 bytes, last accessed: 2019-12-03 06:59:53\n",
      "compressed_files.ipynb: 7857 bytes, last accessed: 2019-11-22 06:48:58\n",
      "julia.ipynb: 5311 bytes, last accessed: 2019-12-03 07:54:28\n",
      "filesystem_interaction.ipynb: 18180 bytes, last accessed: 2019-12-03 07:57:53\n"
     ]
    }
   ],
   "source": [
    "for item in Path.cwd().iterdir():\n",
    "    if item.is_file() and item.name.endswith('.ipynb'):\n",
    "        lstat = item.lstat()\n",
    "        access_time = datetime.strftime(datetime.fromtimestamp(lstat.st_atime),\n",
    "                                        '%Y-%m-%d %H:%M:%S')\n",
    "        print(f'{item.name}: {lstat.st_size} bytes, last accessed: {access_time}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## File permissions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Create a new directory, don't throw an error if it exists. Note that the mode must be specified in octal."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.512729Z",
     "start_time": "2018-11-13T12:13:11.508382Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "tmp_dir = Path.cwd() / 'tmp'\n",
    "tmp_dir.mkdir(mode=0o700, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "More specifically, testing whether the current user can access a file is also quite useful."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.542637Z",
     "start_time": "2018-11-13T12:13:11.533589Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if pathlib.os.access(tmp_dir, pathlib.os.W_OK):\n",
    "    print(f'current user can write in {tmp_dir.name}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Set the access mode a bit more liberal and check the permissions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.557631Z",
     "start_time": "2018-11-13T12:13:11.547135Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "tmp_dir.chmod(0o755)\n",
    "print(oct(tmp_dir.lstat().st_mode))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Detailed tests on access permissions can be written using the bitmasks defined in the `stat` module."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.570715Z",
     "start_time": "2018-11-13T12:13:11.559660Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "mode = tmp_dir.lstat().st_mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.587541Z",
     "start_time": "2018-11-13T12:13:11.578575Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if mode & stat.S_IREAD and mode & stat.S_IWRITE and mode & stat.S_IEXEC:\n",
    "    print(f'owner has full permissions on {tmp_dir.name}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.605838Z",
     "start_time": "2018-11-13T12:13:11.597820Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if not (mode & stat.S_IWGRP):\n",
    "    print(f'group can not write in {tmp_dir.name}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Creating files & globbing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Creating some file in the `tmp` directory can be done by creating a `Path` object for each file, and opening it directly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.626455Z",
     "start_time": "2018-11-13T12:13:11.616435Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "readme = tmp_dir / 'README.md'\n",
    "with readme.open('w') as readme_file:\n",
    "    print('# Contents', file=readme_file, end='\\n\\n')\n",
    "    nr_files = 5\n",
    "    for i in range(1, nr_files + 1):\n",
    "        file = tmp_dir / f'data_{i:03d}.txt'\n",
    "        with file.open('w') as data_file:\n",
    "            print(f'data {i}', file=data_file)\n",
    "        print(f'  * `{file.name}`: data file {i}', file=readme_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.779232Z",
     "start_time": "2018-11-13T12:13:11.634869Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "!cat tmp/README.md"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.814132Z",
     "start_time": "2018-11-13T12:13:11.785865Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "for file in tmp_dir.glob('*.txt'):\n",
    "    print(f'{file.name}: ', end='')\n",
    "    with file.open('r') as data_file:\n",
    "        data = data_file.readline().rstrip()\n",
    "        print(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Dissecting paths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Splitting a path into its various functional parts is of course OS dependent, pathlib lets you do this in a platform independent way."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.854012Z",
     "start_time": "2018-11-13T12:13:11.818782Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "readme.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.869910Z",
     "start_time": "2018-11-13T12:13:11.861556Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "readme.suffix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.890919Z",
     "start_time": "2018-11-13T12:13:11.878941Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "readme.parent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.907873Z",
     "start_time": "2018-11-13T12:13:11.896667Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "readme.drive"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.935865Z",
     "start_time": "2018-11-13T12:13:11.915625Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "tmp_dir.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.961728Z",
     "start_time": "2018-11-13T12:13:11.944904Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "tmp_dir.suffix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:11.984665Z",
     "start_time": "2018-11-13T12:13:11.969686Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "tmp_dir.parent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Copying, moving, deleting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Some operations can be done using `pathlib` such as moving or deleting files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:12.161281Z",
     "start_time": "2018-11-13T12:13:11.989087Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "!ls tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:12.172228Z",
     "start_time": "2018-11-13T12:13:12.163246Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "readme.unlink()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:12.324044Z",
     "start_time": "2018-11-13T12:13:12.178265Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "!ls tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Renaming files is straightforward."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:12.369906Z",
     "start_time": "2018-11-13T12:13:12.342133Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "for file in tmp_dir.glob('*.txt'):\n",
    "    new_file = file.parent / file.name.replace('.txt', '.dat')\n",
    "    file.rename(new_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:12.576310Z",
     "start_time": "2018-11-13T12:13:12.392829Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "!ls tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "`pathlib` has no facilities for copying files, however `shutil` does."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:12.627558Z",
     "start_time": "2018-11-13T12:13:12.596076Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "for file in tmp_dir.glob('*.dat'):\n",
    "    new_file = file.parent / file.name.replace('.dat', '.bak')\n",
    "    shutil.copy(file, new_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:12.824794Z",
     "start_time": "2018-11-13T12:13:12.644042Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "!ls tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Move the `.bak` files into a subdirectory `bak`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:12.840646Z",
     "start_time": "2018-11-13T12:13:12.826710Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "bak_dir = tmp_dir / 'bak'\n",
    "bak_dir.mkdir()\n",
    "for file in tmp_dir.glob('*.bak'):\n",
    "    file.rename(bak_dir / file.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Print the sizes of all the files in `tmp`, and also compute the total size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:12.862564Z",
     "start_time": "2018-11-13T12:13:12.850333Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "total_size = 0\n",
    "for dir_name, _, files in os.walk(tmp_dir, topdown=False):\n",
    "    dir_size = 0\n",
    "    print(f'{dir_name}:')\n",
    "    for file in map(lambda f: Path(dir_name) / f, files):\n",
    "        file_size = file.lstat().st_size\n",
    "        dir_size += file_size\n",
    "        print(f'\\t{file.name}: {file_size} bytes')\n",
    "    print(f'\\tTotal: {dir_size} bytes')\n",
    "    total_size += dir_size\n",
    "print(f'Total: {total_size} bytes')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "`Path`'s `rmdir` method can be used to remove an empty directory, however, if the directory is not empty, the `rmtree` function in `shutil` can be used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:12.883186Z",
     "start_time": "2018-11-13T12:13:12.870365Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "shutil.rmtree(tmp_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Temporary files and directories, `inputfile`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Especially when running application concurrently, it is important to ensure that names for temporary files are unique.  The module `tempfile` implements this. The temporary directory is created in `/tmp` and will be deleted with all its contents as soon as the end of the context is reached.  Files are created in that directory with random names, but will not be deleted. The default mode for temporary files is `w+b`, in this case we choose only `w` since the files are permanent, and we open them for reading later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T18:56:33.358640Z",
     "start_time": "2018-11-13T18:56:33.322179Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "nr_files = 3\n",
    "nr_lines = 4\n",
    "with tempfile.TemporaryDirectory(prefix='data_', dir='/tmp') as tmp_dir:\n",
    "    print(f'director {tmp_dir} exists: {Path(tmp_dir).exists()}')\n",
    "    # write the data files, use random names\n",
    "    for file_nr in range(nr_files):\n",
    "        with tempfile.NamedTemporaryFile(mode='w', prefix='data_', suffix='.txt',\n",
    "                                         dir=tmp_dir, delete=False) as tmp_file:\n",
    "            for i in range(nr_lines):\n",
    "                print(f'line {i + 1} of {file_nr + 1}', file=tmp_file)\n",
    "    # for each data file, print the name, and, indented, all the lines\n",
    "    for tmp_file in Path(tmp_dir).iterdir():\n",
    "        print(tmp_file.name)\n",
    "        with tmp_file.open('r') as file:\n",
    "            for line in file:\n",
    "                print(f'\\t{line.rstrip()}')\n",
    "    # treat all the data files as one input, and compute the sum of all the numbers in the file\n",
    "    total = 0\n",
    "    for line in fileinput.FileInput(Path(tmp_dir).glob('*.txt')):\n",
    "        total += sum(map(int, re.findall(r'\\d+', line)))\n",
    "    print(f'total = {total}')\n",
    "print(f'director {tmp_dir} exists: {Path(tmp_dir).exists()}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Miscellaneous"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the current working directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:12.907465Z",
     "start_time": "2018-11-13T12:13:12.890523Z"
    }
   },
   "outputs": [],
   "source": [
    "Path.cwd()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the current user's home directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-13T12:13:12.923462Z",
     "start_time": "2018-11-13T12:13:12.909259Z"
    }
   },
   "outputs": [],
   "source": [
    "Path.home()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the system's temporary directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-14T08:44:23.534609Z",
     "start_time": "2018-11-14T08:44:23.497824Z"
    }
   },
   "outputs": [],
   "source": [
    "tempfile.gettempdir()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}