Merge remote-tracking branch 'origin/development' into development

Author: gjbex

Diff for: source-code/README.md

@@ -28,6 +28,8 @@ to create it.  There is some material not covered in the presentation as well.
     configuration files.
 1. `data-formats`: illustrates how to deal with data formats such as CSV
     files, binary data and XML.
+1. `directory-entrypoint`: illustrates how to define an application entry point in
+   a directory.
 1. `enviroment-variables`: illustrates how to use environment variables to set paths
    in a script.
 1. `hydra`: Facebook Hydra application framework illustration.

Diff for: source-code/directory-entrypoint/README.md

@@ -0,0 +1,11 @@
+# Directory entry point
+
+Using `__main__.py` as an entry point, the Python interpreter can run a
+directory.
+
+## What is it?
+
+1. `messenger`: directory containing the application.
+1. `messenger/__main__.py`: Python application entry point.
+1. `messenger/messenger.py`: Python module defining some functions for the
+   application.

Diff for: source-code/directory-entrypoint/messenger/__main__.py

@@ -0,0 +1,18 @@
+import argparse
+import messenger
+import sys
+
+
+arg_parser = argparse.ArgumentParser(description='say something to the user')
+arg_parser.add_argument('--hello', action='store_true', help='say hello')
+arg_parser.add_argument('--bye', action='store_true', help='say bye')
+arg_parser.add_argument('name', help='name to talk to')
+options = arg_parser.parse_args()
+
+if options.hello:
+    messenger.say_hello(options.name)
+if options.bye:
+    messenger.say_bye(options.name)
+if not (options.hello or options.bye):
+    print(f"don't know what  to say to {options.name}", file=sys.stderr)
+    sys.exit(1)

Diff for: source-code/directory-entrypoint/messenger/messenger.py

@@ -0,0 +1,5 @@
+def say_hello(name):
+    print(f'hello {name}')
+
+def say_bye(name):
+    print(f'bye {name}')

Diff for: source-code/processes/monitor.py

@@ -50,6 +50,7 @@ def find_ancestor(pid=None, username=None):
     for parent in reversed(parents):
         if parent.username() == username:
             return parent
+    return process
 
 
 def get_cmdline(process):
@@ -128,21 +129,6 @@ def process_status(process, metrics):
     return ','.join(status)
 
 
-def get_filename_suffix(file_id):
-    filename = ''
-    if 'PBS_JOBID' in os.environ:
-        filename += f'_{os.environ["PBS_JOBID"]}'
-    if 'PBS_ARRAYID' in os.environ:
-        filename += f'_{os.environ["PBS_ARRAYJOBID"]}'
-    if file_id:
-        filename += f'_{file_id}'
-    return f'{filename}.txt'
-
-
-def get_filename(file_id):
-    return platform.node() + get_filename_suffix(file_id)
-
-
 def main():
     arg_parser = ArgumentParser(description='monitor processes')
     arg_parser.add_argument('--pid', type=int, help='parent process ID ot monitor')
@@ -152,33 +138,32 @@ def main():
     arg_parser.add_argument('--affinity', action='store_true',
                             help='monitor process affinity')
     arg_parser.add_argument('--files', action='store_true', help='monitor poen files')
-    output_group = arg_parser.add_mutually_exclusive_group()
-    output_group.add_argument('--output-file', help='name of file to store informatoin')
-    output_group.add_argument('--stdout', action='store_true',
-                              help='output to standard output')
-    output_group.add_argument('--file_id', help='identification string for output filename')
+    arg_parser.add_argument('--ancestor', action='store_true',
+                            help='search for ancestor owned by use and report on all its decendants')
+    arg_parser.add_argument('--output-file', help='name of file to store informatoin')
     options = arg_parser.parse_args()
-    ancestor = find_ancestor(options.pid, options.user)
+    if options.ancestor:
+        process = find_ancestor(options.pid, options.user)
+    else:
+        process = psutil.Process(options.pid)
     inactive = []
     if not options.affinity:
         inactive.append('affinity')
     if not options.files:
         inactive.append('read_files')
         inactive.append('write_files')
     metrics = define_actions(inactive)
-    if options.stdout:
-        file = sys.stdout
-    elif options.output_file:
+    if options.output_file:
         file = open(options.output_file, 'w')
     else:
-        file = open(get_filename(options.file_id), 'w')
+        file = sys.stdout
     try:
         with file:
             print(status_header(metrics), file=file)
             while True:
-                process_info = [process_status(ancestor, metrics)]
-                for process in ancestor.children(recursive=True):
-                    process_info.append(process_status(process, metrics))
+                process_info = [process_status(process, metrics)]
+                for child_process in process.children(recursive=True):
+                    process_info.append(process_status(child_process, metrics))
                 print('\n'.join(process_info), file=file)
                 time.sleep(options.delta)
     except KeyboardInterrupt:

Diff for: source-code/subprocess/README.md

@@ -8,5 +8,8 @@ The exit code can also be retrieved.
 
 ## What is it?
 
-`wc_metrics.py`: this script computes the number of characters, words, and
-lines in a file using the shell's `wc` commaand.
+1. `wc_metrics.py`: this script computes the number of characters, words, and
+  lines in a file using the shell's `wc` commaand.
+1. `async_handling.py`: this script illustrates how to monitor the standard error
+   generated by a subprocess.
+1. `process.sh`: Bash script to execute by `async_handling.py`.

Diff for: source-code/subprocess/async_handling.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+from datetime import datetime
+import subprocess
+import sys
+import time
+
+class Result:
+
+    def set(self, value):
+        self._value = value
+
+    @property
+    def value(self):
+        return self._value
+
+
+def execute(cmd, result):
+    process = subprocess.Popen(cmd,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE,
+                               text=True)
+    for stderr_line in iter(process.stderr.readline, ''):
+        yield stderr_line.strip()
+    stdout, stderr = process.communicate()
+    return_code = process.wait()
+    if return_code:
+        raise subprocess.CalledProcessError(return_code, cmd)
+    else:
+        result.set(stdout)
+
+result = Result()
+for stderr_line in execute(['bash', 'process.sh'], result):
+    print(f'{datetime.now()}: {stderr_line}')
+print(f'result:\n{result.value}')

Diff for: source-code/subprocess/process.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+for i in {1..4}
+do
+    (>&2 echo "### info: $i in progress...")
+    sleep 5
+    echo "item $i: $((i**2))"
+done