Merge pull request #24 from fact-project/add_missing

Add missing stuff

Author: maxnoe

setup.py

@@ -3,7 +3,7 @@
 
 setup(
     name='smart_fact_crawler',
-    version='0.4.0',
+    version='0.4.1',
     description='acquieres data published on the smartfact web page',
     url='https://github.com/fact-project/smart_fact_crawler.git',
     author='Dominik Neise, Sebastian Mueller, Maximilian Nöthe',
@@ -21,7 +21,7 @@
     install_requires=[
         'requests',
     ],
-    tests_require=['pytest>=3.0'],
+    tests_require=['pytest>=3.0', 'freezegun'],
     setup_requires=['pytest-runner'],
     zip_safe=True,
 )

smart_fact_crawler/__init__.py

@@ -6,7 +6,6 @@
 from .tools import str2float as s2f
 from .tools import smartfact_time2datetime as sft2dt
 from .tools import smartfact2table
-from .tools import extract_run_id_from_system_status
 from .tools import get_entry
 
 import re
@@ -20,6 +19,17 @@
 Quantity = namedtuple('Quantity', ['value', 'unit'])
 
 
+run_re = re.compile(
+    '([0-9]{2}:[0-9]{2}:[0-9]{2}) '  # match the time part
+    '<#[a-z]+>'                      # html color
+    '([a-zA-Z\-]+) '                 # run type
+    '\[([a-zA-Z0-9 ]+)\] '           # source name
+    '\(Run (\d+)\)'                  # run number
+    '</#>'
+)
+Run = namedtuple('Run', ['start', 'type', 'source', 'id'])
+
+
 def to_namedtuple(name, dictionary):
     return namedtuple(name, dictionary.keys())(**dictionary)
 
@@ -305,14 +315,19 @@ def main_page(url=None, timeout=None, fallback=False):
     get = partial(get_entry, fallback=fallback)
 
     system_status = get(table, 1, 1)
+    power_val, power_unit = get(table, 6, 3, default='nan nan').split()
+    trigger_val, trigger_unit, _ = get(table, 5, 1, default='nan nan nan').split()
     return to_namedtuple('MainPage', {
         'timestamp_1': sft2dt(get(table, 0, 0)),
         'timestamp_2': sft2dt(get(table, 0, 1)),
         'system_status': system_status,
-        'run_id': extract_run_id_from_system_status(system_status),
         'relative_camera_temperature': Quantity(s2f(get(table, 3, 1)), 'deg_C'),
         'humidity': Quantity(s2f(get(table, 4, 1)), '%'),
         'wind_speed': Quantity(s2f(get(table, 4, 2)), 'km/h'),
+        'trigger_rate': Quantity(s2f(trigger_val), trigger_unit),
+        'median_current': Quantity(s2f(get(table, 6, 1)), 'uA'),
+        'max_current': Quantity(s2f(get(table, 6, 2)), 'uA'),
+        'power': Quantity(s2f(power_val), power_unit),
     })
 
 
@@ -335,14 +350,48 @@ def errorhist(url=None, timeout=None, fallback=False):
 
     table = smartfact2table(url, timeout=timeout)
     get = partial(get_entry, fallback=fallback)
-    timestamp = get(table, 0, 0)
 
     history = [
         h
         for h in get(table, 1, 1, default='').split("<->")[1].split("<br/>")
         if h
     ]
     return to_namedtuple('ErrorHistPage', {
-        'timestamp': sft2dt(timestamp) if timestamp else None,
+        'timestamp': sft2dt(get(table, 0, 0)),
         'history': history,
     })
+
+
+def build_run(tup):
+    start, run_type, source, run_id = tup
+    run_id = int(run_id)
+
+    now = datetime.utcnow()
+    start = datetime.strptime(start, '%H:%M:%S').replace(
+        year=now.year, month=now.month, day=now.day
+    )
+
+    if start > now:
+        start -= timedelta(hours=24)
+
+    return Run(start, run_type, source, run_id)
+
+
+def observations(url=None, timeout=None, fallback=False):
+    if url is None:
+        url = os.path.join(smartfacturl, 'observations.data')
+
+    table = smartfact2table(url, timeout=timeout)
+    get = partial(get_entry, fallback=fallback)
+
+    run_list = get(table, 1, 1)
+    if run_list is not None:
+        runs = list(map(build_run, run_re.findall(table[1][1])))
+        runs.sort(key=lambda r: r.id)
+    else:
+        runs = None
+
+    return to_namedtuple('ErrorHistPage', {
+        'timestamp': sft2dt(get(table, 0, 0)),
+        'runs': runs,
+    })

smart_fact_crawler/tests/test_observations.py

@@ -0,0 +1,57 @@
+from pkg_resources import resource_filename
+from freezegun import freeze_time
+from datetime import datetime
+
+test_file = resource_filename(
+    'smart_fact_crawler', 'resources/20160703_233149/observations.data'
+)
+
+def test_re():
+    from smart_fact_crawler import run_re
+
+    with open(test_file) as f:
+        m = run_re.findall(f.read())
+
+    assert len(m) ==  67
+
+    timestamp, run_type, source, run_id = m[0]
+
+    assert timestamp == '23:31:27'
+    assert run_type == 'data'
+    assert source == 'Mrk 501'
+    assert run_id == '67'
+
+
+@freeze_time('2016-07-03 23:32')
+def test_build_run():
+    from smart_fact_crawler import run_re, build_run
+
+    with open(test_file) as f:
+        m = run_re.findall(f.read())
+
+    run = build_run(m[0])
+
+    assert run.id == 67
+    assert run.start == datetime(2016, 7, 3, 23, 31, 27)
+
+
+@freeze_time('2016-07-04 00:32')
+def test_build_run_after_midnight():
+    from smart_fact_crawler import run_re, build_run
+
+    with open(test_file) as f:
+        m = run_re.findall(f.read())
+
+    run = build_run(m[0])
+
+    assert run.id == 67
+    assert run.start == datetime(2016, 7, 3, 23, 31, 27)
+
+
+@freeze_time('2016-07-03 23:32')
+def test_observations():
+    from smart_fact_crawler import observations
+
+    obs = observations(url='file:' + test_file)
+
+    assert obs.runs[-1].id == len(obs.runs)

smart_fact_crawler/tests/test_unit.py

@@ -2,7 +2,6 @@
 import requests
 import urllib
 from datetime import datetime
-import re
 
 
 def str2float(text):
@@ -48,27 +47,6 @@ def smartfact2table(url, timeout=None):
     return parse_table(text)
 
 
-def extract_run_id_from_system_status(system_status):
-    '''
-    system_status: string like 'data(47) [1169/279s]' or 'Idle [pedestal]'
-    returns: integer `run_id`
-        or None if no number between braces `()` found in system_status.
-
-    throws TypeError in case `system_status` is not string like
-    enough to be used inside re.match()
-    '''
-    if system_status is None:
-        return None
-
-    run_id_in_system_status_pattern = r'.*\((\d+)\).*'
-    match_run_id = re.match(run_id_in_system_status_pattern, system_status)
-
-    if match_run_id is None:
-        return None
-
-    return int(match_run_id.groups()[0])
-
-
 def get_entry(table, row, col, fallback=False, default=None):
     '''
     Get an element from a two dimensinoal list like the return value