Skip to content

Commit 83100c5

Browse files
maxnoemaxnoe
authored
Merge pull request #15 from fact-project/auxservices
Move reading auxfiles from aux2mongodb to PyFACT
2 parents 898576f + c395a2b commit 83100c5

File tree

7 files changed

+195
-2
lines changed

7 files changed

+195
-2
lines changed

‎.travis.yml

+1-2
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,12 @@ before_install:
1010
- chmod +x miniconda.sh
1111
- ./miniconda.sh -b -p /home/travis/miniconda
1212
- export PATH=/home/travis/miniconda/bin:$PATH
13-
- conda update --yes conda
1413
# The next couple lines fix a crash with multiprocessing on Travis and are not specific to using Miniconda
1514
- sudo rm -rf /dev/shm
1615
- sudo ln -s /run/shm /dev/shm
1716
# Install packages
1817
install:
19-
- conda install --yes python=$TRAVIS_PYTHON_VERSION atlas numpy scipy matplotlib nose pip pymongo flake8
18+
- conda install --yes python=$TRAVIS_PYTHON_VERSION atlas numpy scipy matplotlib nose pip pymongo flake8 pandas astropy
2019
- pip install .
2120
- python setup.py build_ext --inplace
2221

‎README.md

+14
Original file line numberDiff line numberDiff line change
@@ -63,3 +63,17 @@ from fact.plotting import get_pixel_coords
6363
6464
pixel_x, pixel_y = get_pixel_coords()
6565
```
66+
67+
## auxservices
68+
69+
Utilities to read in our aux fits files into pandas dataframes.
70+
71+
```{python}
72+
73+
from fact.auxservices import MagicWeather
74+
from datetime import date
75+
76+
weather = MagicWeather(auxdir='/fact/aux/')
77+
78+
df = weather.read_date(date(2016, 1, 1))
79+
```

‎fact/__init__.py

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from .time import fjd, iso2dt, run2dt, facttime, night, night_integer, datestr
22
from . import plotting
3+
from . import auxservices
34

45
__all__ = [
56
'fjd',

‎fact/auxservices/__init__.py

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
from .base import AuxService
2+
from .services import *

‎fact/auxservices/base.py

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
from astropy.table import Table
2+
from astropy.units import UnitsWarning
3+
import os
4+
import warnings
5+
6+
7+
class AuxService:
8+
9+
renames = {}
10+
ignored_columns = []
11+
transforms = {}
12+
basename = 'AUX_SERVICE'
13+
14+
def __init__(self, auxdir='/fact/aux'):
15+
self.auxdir = auxdir
16+
self.filename_template = os.path.join(
17+
self.auxdir, '{date:%Y}', '{date:%m}', '{date:%d}',
18+
'{date:%Y%m%d}.' + self.basename + '.fits'
19+
)
20+
21+
def read_file(self, filename):
22+
with warnings.catch_warnings():
23+
warnings.simplefilter('ignore', category=UnitsWarning)
24+
table = Table.read(filename)
25+
26+
for column in table.columns.keys():
27+
if column in self.ignored_columns:
28+
table.remove_column(column)
29+
30+
elif column in self.renames:
31+
table[column].name = self.renames[column]
32+
33+
for column in table.columns.keys():
34+
shape = table[column].shape
35+
if len(shape) > 1:
36+
for i in range(shape[1]):
37+
table[column + '_{}'.format(i)] = table[column][:, i]
38+
table.remove_column(column)
39+
40+
df = table.to_pandas()
41+
42+
for key, transform in self.transforms.items():
43+
df[key] = transform(df[key])
44+
45+
return df
46+
47+
def read_date(self, date):
48+
49+
filename = self.filename_template.format(date=date)
50+
return self.read_file(filename)

‎fact/auxservices/services.py

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
from .base import AuxService
2+
import pandas as pd
3+
4+
5+
def fact_mjd_to_datetime(fact_mjd):
6+
''' convert fact mjds (days since unix epoch) to pandas datetimes '''
7+
return pd.to_datetime(fact_mjd * 24 * 3600 * 1e9)
8+
9+
10+
__all__ = [
11+
'MagicWeather',
12+
'PfMini',
13+
'DriveTracking',
14+
'DrivePointing',
15+
'DriveSource',
16+
'FSCHumidity',
17+
'FSCTemperature'
18+
]
19+
20+
21+
class MagicWeather(AuxService):
22+
basename = 'MAGIC_WEATHER_DATA'
23+
renames = {
24+
'Time': 'timestamp',
25+
'T': 'temperature',
26+
'T_dew': 'dewpoint',
27+
'H': 'humidity',
28+
'P': 'pressure',
29+
'v': 'wind_speed',
30+
'v_max': 'wind_gust_speed',
31+
'd': 'wind_direction',
32+
}
33+
34+
ignored_columns = ['stat', 'QoS']
35+
transforms = {'timestamp': fact_mjd_to_datetime}
36+
37+
38+
class PfMini(AuxService):
39+
basename = 'PFMINI_CONTROL_DATA'
40+
renames = {
41+
'Time': 'timestamp',
42+
'Temperature': 'temperature',
43+
'Humidity': 'humidity',
44+
}
45+
46+
ignored_columns = ['QoS', ]
47+
transforms = {'timestamp': fact_mjd_to_datetime}
48+
49+
50+
class DriveTracking(AuxService):
51+
basename = 'DRIVE_CONTROL_TRACKING_POSITION'
52+
renames = {
53+
'Time': 'timestamp',
54+
'Ra': 'right_ascension',
55+
'Dec': 'declination',
56+
'Ha': 'hourangle',
57+
'SrcHa': 'hourangle_source',
58+
'SrcRa': 'right_ascension_source',
59+
'SrcDec': 'declination_source',
60+
'HaDec': 'hourangle_source',
61+
'Zd': 'zenith',
62+
'Az': 'azimuth',
63+
'dZd': 'zenith_deviation',
64+
'dAz': 'azimuth_deviation',
65+
'dev': 'absolute_control_deviation',
66+
'avgdev': 'average_control_deviation',
67+
}
68+
transforms = {'timestamp': fact_mjd_to_datetime}
69+
ignored_columns = ['QoS', ]
70+
71+
72+
class DrivePointing(AuxService):
73+
basename = 'DRIVE_CONTROL_POINTING_POSITION'
74+
renames = {
75+
'Time': 'timestamp',
76+
'Zd': 'zenith',
77+
'Az': 'azimuth',
78+
}
79+
transforms = {'timestamp': fact_mjd_to_datetime}
80+
ignored_columns = ['QoS', ]
81+
82+
83+
class DriveSource(AuxService):
84+
basename = 'DRIVE_CONTROL_SOURCE_POSITION'
85+
renames = {
86+
'Time': 'timestamp',
87+
'Ra_src': 'right_ascension_source',
88+
'Ra_cmd': 'right_ascension_command',
89+
'Dec_src': 'declination_source',
90+
'Dec_cmd': 'declination_command',
91+
'Offset': 'wobble_offset',
92+
'Angle': 'wobble_angle',
93+
'Name': 'source_name',
94+
'Period': 'orbit_period',
95+
}
96+
transforms = {'timestamp': fact_mjd_to_datetime}
97+
ignored_columns = ['QoS', ]
98+
99+
100+
class FSCHumidity(AuxService):
101+
basename = 'FSC_CONTROL_HUMIDITY'
102+
renames = {
103+
'Time': 'timestamp',
104+
't': 'fsc_uptime',
105+
'H': 'humidity',
106+
}
107+
transforms = {'timestamp': fact_mjd_to_datetime}
108+
ignored_columns = ['QoS', ]
109+
110+
111+
class FSCTemperature(AuxService):
112+
basename = 'FSC_CONTROL_TEMPERATURE'
113+
renames = {
114+
'Time': 'timestamp',
115+
't': 'fsc_uptime',
116+
'T_crate': 'crate_temperature',
117+
'T_sens': 'sensor_compartment_temperature',
118+
'T_ps': 'power_supply_temperature',
119+
'T_aux': 'auxiliary_power_supply_temperature',
120+
'T_back': 'ftm_backpanel_temperature',
121+
'T_eth': 'ethernet_temperature',
122+
}
123+
transforms = {'timestamp': fact_mjd_to_datetime}
124+
ignored_columns = ['QoS', ]

‎setup.py

+3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
'fact.plotting',
1515
'fact.slowdata',
1616
'fact.credentials',
17+
'fact.auxservices',
1718
],
1819
package_data={
1920
'': ['resources/*', 'credentials/credentials.encrypted']},
@@ -27,6 +28,8 @@
2728
'setuptools',
2829
'sqlalchemy',
2930
'pymysql',
31+
'pandas',
32+
'astropy',
3033
],
3134
zip_safe=False,
3235
)

0 commit comments

Comments
 (0)