Merge pull request #13 from gjbex/development

Development

Author: gjbex

environment.yml

@@ -1,7 +1,8 @@
 name: python_for_systems_programming
 channels:
-  - defaults
+  - conda-forge
 dependencies:
+  - click
   - paramiko
   - matplotlib
   - jinja2
@@ -10,6 +11,10 @@ dependencies:
   - fabric
   - numpy
   - jupyterlab
-pip:
-  - schedule
-prefix: /home/gjb/miniconda3/envs/python_for_systems_programming
+  - uvicorn
+  - ca-certificates
+  - certifi
+  - openssl
+  - fastapi
+  - fire
+prefix: /home/gjb/mambaforge/envs/python_for_systems_programming

python_for_systems_programming.pptx

@@ -44,3 +44,4 @@ to create it.  There is some material not covered in the presentation as well.
 1. `subprocess`: illustrates executing a shell command from a Python script
     using the `subprocess` module.
 1. `xml-generator`: code to generate a random XML documents.
+1. `fastapi`: simple illustrations of using FastAPI for webservices.

python_for_systems_programming_linux64_conda_specs.txt

@@ -1,5 +1,2 @@
 def fac(n):
-    if n < 2:
-        return 1
-    else:
-        return n*fac(n-1)
+    return 1 if n < 2 else n*fac(n-1)

source-code/README.md

@@ -1,5 +1,2 @@
 def fib(n):
-    if n == 0 or n == 1:
-        return 1
-    else:
-        return fib(n-1) + fib(n-2)
+    return 1 if n in [0, 1] else fib(n-1) + fib(n-2)

source-code/code-evaluation/fac.py

@@ -23,3 +23,4 @@ $ ./generate_gaussian.py -h
    ./options_in_file.py --foo something @file_options.txt
    ```
 1. `file_options.txt`: file containing options for `options_in_file.py`.
+1. `Rerun`: experiment with file options.

source-code/code-evaluation/fib.py

@@ -0,0 +1,33 @@
+# Rerunnable
+
+Toy application to explore the possibilities created by
+using argparse options stored in a file.
+
+
+## What is it?
+1.  rerunnable.py`: script to explore the possibilities.
+
+
+## How to use?
+
+```bash
+$ ./rerunnable.py  --verbose  --number 5  --type bye  gjb
+$ ./rerunnable.py @rerunnable_cla.txt
+```
+
+This will rerun the application with all the settings specified for the
+previous run.
+
+To override options:
+```bash
+$ ./rerunnable.py  @rerunnable_cla.txt --number 3
+```
+
+
+## Conclusions
+
+This approach works well for command line options, e.g., `--number 5`.
+
+It is not flexibile for flags, e.g., `--verbose` since they can not be "unset".
+
+It is not flexible either for positional arguments since they can not be modified.

source-code/command-line-arguments/ArgParse/README.md

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+import argparse
+
+
+def dump_options(options, pos_args=None, exclude=None):
+    options_dict = vars(options)
+    with open('rerunnable_cla.txt', 'w') as file:
+        for key, value in options_dict.items():
+            if ((exclude is None or key not in exclude) and
+                (pos_args is None or key not in pos_args)):
+                if isinstance(value, bool):
+                    if value:
+                        print(f'--{key}', file=file)
+                else:
+                    print(f'--{key}\n{value}', file=file)
+        if pos_args is not None:
+            for key in pos_args:
+                print(options_dict[key], file=file)
+
+
+if __name__ == '__main__':
+    arg_parser = argparse.ArgumentParser(
+        fromfile_prefix_chars='@',
+        description='application that saves its command line options and can be rerun'
+    )
+    arg_parser.add_argument('--number', type=int, default=1,
+                            help='number of messages to write')
+    arg_parser.add_argument('--type', choices=('hello', 'bye'), default='hello',
+                            help='message type')
+    arg_parser.add_argument('--verbose', action='store_true',
+                            help='verbose output')
+    arg_parser.add_argument('name', help='person to message')
+    options = arg_parser.parse_args()
+    dump_options(options, pos_args=('name', ))
+    if options.verbose:
+        print(f'printing {options.number} messages')
+    for _ in range(options.number):
+        print(f'{options.type} {options.name}')

source-code/command-line-arguments/ArgParse/Rerun/README.md

@@ -3,14 +3,14 @@
 import argparse
 
 
-def amin():
+def main():
     arg_parser = argparse.ArgumentParser(fromfile_prefix_chars='@')
     arg_parser.add_argument('--foo', help='foo option')
-    arg_parser.add_argument('--bar', help='bar optoin')
+    arg_parser.add_argument('--bar', help='bar option')
     arg_parser.add_argument('--flag', action='store_true',
                             help='flag option')
     options = arg_parser.parse_args()
     print(options)
 
 if __name__ == '__main__':
-    amin()
+    main()

source-code/command-line-arguments/ArgParse/Rerun/rerunnable.py

@@ -19,5 +19,5 @@
                                         options.resoure_specs))
         print('resources: ' + ', '.join(specs))
     if options.account:
-        print('account: ' + options.account)
+        print(f'account: {options.account}')
     print('unparsed: ' + ', '.join(unparsed))

source-code/command-line-arguments/ArgParse/options_in_file.py

@@ -7,7 +7,7 @@
 
 
 def parse_job_script(file_name):
-    args = list()
+    args = []
     with open(file_name) as file:
         for line in file:
             if line.lstrip().startswith('#PBS '):

source-code/command-line-arguments/ArgParse/partial_parse.py

@@ -5,8 +5,10 @@ applications with a command line interface.
 
 ## What is it?
 
-1. `calculator.py`/`calculations.py`: Calculator application that
+1. `calculator.py`/`calculations.py`: calculator application that
    implements addition, substraction,  multiplication and division.
-1. `sayer.py`: Illustration of grouped commands.
+1. `sayer.py`: illustration of grouped commands.
 1. `cl_logger.py`/`log_management.py`/`log_operations.py`: logging
    from the command line example.
+1. `job.py`: simple class definition to explore interacting with
+   objects from the CLI.

source-code/command-line-arguments/ArgParse/two_stage_parse.py

@@ -1,14 +1,17 @@
-def add(x, y):
+def add(x: float, y: float) -> float:
     return x + y
 
-def sub(x, y):
+def sub(x: float, y: float) -> float:
     return x - y
 
-def mult(x, y):
+def mult(x: float, y: float) -> float:
     return x*y
 
-def div(x, y):
+def div(x: float, y: float) -> float:
     return x/y
 
-def mod(x, y):
+def idiv(x: int, y: int) -> int:
+    return x//y
+
+def mod(x: int, y: int) -> int:
     return x % y

source-code/command-line-arguments/Fire/README.md

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
-import fire
+
 import calculations
+import fire
 
 
 if __name__ == '__main__':

source-code/command-line-arguments/Fire/calculations.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+import fire
+
+
+class Job:
+
+    _nodes: int
+    _ntasks: int
+
+    def __init__(self, nodes=1, ntasks=1):
+        self.nodes = nodes
+        self.ntasks = ntasks
+
+    @property
+    def nodes(self):
+        return self._nodes
+
+    @nodes.setter
+    def nodes(self, nodes):
+        self._nodes = nodes
+        return self
+
+
+    @property
+    def ntasks(self):
+        return self._ntasks
+
+    @ntasks.setter
+    def ntasks(self, ntasks):
+        self._ntasks = ntasks
+        return self
+
+    def print(self):
+        return f'nodes={self.nodes} ntasks={self.ntasks}'
+
+if __name__ == '__main__':
+    fire.Fire(Job)

source-code/command-line-arguments/Fire/calculator.py

@@ -12,13 +12,12 @@ def __init__(self, name):
         self.name = name
 
     def to(self, name=None):
-        if name is None:
-            if self.name is None:
-                return 'No one to say hello to'
-            else:
-                return f'Hello to {self.name}'
-        else:
+        if name is not None:
             return f'Hello {name}'
+        if self.name is None:
+            return 'No one to say hello to'
+        else:
+            return f'Hello to {self.name}'
 
     def everyone(self):
         return 'hello to everyone'
@@ -34,15 +33,12 @@ def __init__(self, name):
 
     def to(self, name=None):
         if name is None:
-            if self.name is None:
-                return 'No one to say bye to'
-            else:
-                return f'Bye to {self.name}'
+            return 'No one to say bye to' if self.name is None else f'Bye to {self.name}'
         else:
             return f'Bye {name}'
 
     def no_one(self):
-        return f'Bye to no one'
+        return 'Bye to no one'
 
 
 class Sayer:

source-code/command-line-arguments/Fire/job.py

@@ -5,10 +5,7 @@
 
 
 def main():
-    if len(sys.argv) > 1:
-        cfg_file = sys.argv[1]
-    else:
-        cfg_file = 'defaults.conf'
+    cfg_file = sys.argv[1] if len(sys.argv) > 1 else 'defaults.conf'
     cfg_parser = SafeConfigParser()
     cfg_parser.read(cfg_file)
     print('Sections:')

source-code/command-line-arguments/Fire/sayer.py

@@ -15,7 +15,7 @@ def parse_config_line(meta_data, line):
         meta_data[symbol] = demangle_name(name)
 
 def parse_config(vcd_file):
-    meta_data = dict()
+    meta_data = {}
     for line in vcd_file:
         line = line.strip()
         if line == '$end':
@@ -37,16 +37,15 @@ def update_buffer(buffer, line, meta_data):
     buffer[key] = value
 
 def init_data(meta_data):
-    data = dict()
-    data['time'] = list()
+    data = {'time': []}
     for var in meta_data:
-        data[meta_data[var]] = list()
+        data[meta_data[var]] = []
     return data
 
 def parse_data(vcd_file, meta_data):
     data = init_data(meta_data)
     time_stamp = None
-    buffer = dict()
+    buffer = {}
     for line in vcd_file:
         line = line.strip()
         if line.startswith('#'):
@@ -68,9 +67,7 @@ def write_vcd_data_structure(out_file, data, sep=' '):
     columns = list(data.keys())
     out_file.write(sep.join(columns) + '\n')
     for time_step in range(len(data['time'])):
-        data_line = list()
-        for var in columns:
-            data_line.append(data[var][time_step])
+        data_line = [data[var][time_step] for var in columns]
         out_file.write(sep.join(str(data_item) for data_item in data_line))
         out_file.write('\n')
 

source-code/config-parser/config_reader.py

@@ -106,7 +106,7 @@ def _parse_data(self, agt_file):
         if not match:
             msg = "line {0:d}: invalid number of measurements '{1}'"
             raise AgtDataError(msg.format(self._current_line, nr_lines_str))
-        nr_lines = int(match.group(1))
+        nr_lines = int(match[1])
         self._current_line += 1
 # ignore header line
         agt_file.readline()

source-code/data-formats/Vcd/vcd_parser.py

@@ -47,11 +47,10 @@ def __iter__(self):
         return self
 
     def __next__(self):
-        if self._current < self.n:
-            self._current += 1
-            return self._distr(*self._params)
-        else:
+        if self._current >= self.n:
             raise StopIteration()
+        self._current += 1
+        return self._distr(*self._params)
 
 
 class DistributionCreator(object):
@@ -108,9 +107,9 @@ def __init__(self, file_name, table_name, col_defs):
         self._row = self._table.row
 
     def _create_table(self, table_name, col_defs):
-        description = {}
-        for col_def in col_defs:
-            description[col_def['name']] = self._typemap[col_def['type']]
+        description = {
+            col_def['name']: self._typemap[col_def['type']] for col_def in col_defs
+        }
         return self._file.create_table('/', table_name, description)
 
     def set_headers(self, headers):

source-code/data-formats/agt_parser.py

@@ -21,7 +21,7 @@ def main():
             print('{name} --- {weight}'.format(name=row['name'],
                                                weight=row['weight']))
             sum += float(row['weight'])
-        print('sum = {}'.format(sum))
+        print(f'sum = {sum}')
 
 if __name__ == '__main__':
     main()

source-code/data-formats/data_generator.py

@@ -13,10 +13,8 @@ def read_array(data_file, length):
     arg_parser.add_argument('file', help='binary file to read')
     options = arg_parser.parse_args()
     with open(options.file, 'rb') as data_file:
-        buffer = data_file.read(4);
-        while buffer:
+        while buffer := data_file.read(4):
             length = unpack('I', buffer)[0]
             values = read_array(data_file, length)
             value_str = ' '.join(f'{x:.2f}' for x in values)
             print(f'{length:d}: {value_str:s}')
-            buffer = data_file.read(4)

source-code/data-formats/read_csv.py

@@ -62,8 +62,7 @@ def startElement(self, name, attrs):
 
     def characters(self, contents):
         if self.in_item:
-            contents = contents.strip()
-            if contents:
+            if contents := contents.strip():
                 data = float(contents.strip())
                 logging.info(f"found '{data}'")
                 self._stack[-1].add_data(data)