Joshua Debugging Tips

$ LD_LIBRARY_PATH=/share/devcache/joshua/libs PYTHONPATH=/share/devcache/joshua/modules/fdb-joshua:/share/devcache/joshua/modules/site-packages python3 -i -m joshua.joshua --cluster-file /share/devcache/joshua/config/fdb.cluster
...
>>> list_active_ensembles(stopped=False)
>>> list_active_ensembles(stopped=True)

>>> joshua_model.db
<fdb.impl.Database object at 0x7f3eb2b9de48>
>>> joshua_model.dir_top
DirectorySubspace(path=('joshua',), prefix=b'\x154')
>>> joshua_model.dir_ensembles
DirectorySubspace(path=('joshua', 'ensembles'), prefix=b'\x15%')

>>> joshua_model.dir_all_ensembles['20201008-135157-XinDong-9848dd79e3aac22d']['properties']
Subspace(rawPrefix=b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00')
>>> joshua_model.dir_all_ensembles['20201008-135157-XinDong-9848dd79e3aac22d']['properties']['submitted']
Subspace(rawPrefix=b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00\x02submitted\x00')
>>> tr = joshua_model.db.create_transaction()
>>> tr.get(b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00\x02submitted\x00')
b'\x0220201008-135157\x00'

tr.reset()
kvs = tr.get_range(b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00', b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x01')
for k, v in kvs:
    print(k, v)

>>> tr.reset()
>>> kvs = tr.get_range(b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00', b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x01')
>>> for k, v in kvs:
...     print(k, v)
... 
b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00\x02compressed\x00' b"'"
b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00\x02data_size\x00' b'\x18\x01Bx\\'
b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00\x02fail_fast\x00' b'\x15\n'
b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00\x02max_runs\x00' b'\x17\x01\x86\xa0'
b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00\x02priority\x00' b'\x15d'
b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00\x02runtime\x00' b'\x022:57:10\x00'
b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00\x02sanity\x00' b'&'
b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00\x02stopped\x00' b'\x0220201008-164907\x00'
b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00\x02submitted\x00' b'\x0220201008-135157\x00'
b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00\x02timeout\x00' b'\x16\x15\x18'
b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\x02properties\x00\x02username\x00' b'\x02XinDong\x00'

>>> joshua_model.dir_ensemble_results_fail
DirectorySubspace(path=('joshua', 'ensembles', 'results', 'fail'), prefix=b'\x15\x08')
>>> joshua_model.dir_ensemble_results_fail[b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00']
Subspace(rawPrefix=b'\x15\x08\x01\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\xff\x00')

tr.reset()
kvs = tr.get_range(b'\x15\x08\x0220201007-155332-mengxurelease63-ff905403cf4fe1f3\x00', b'\x15\x08\x0220201007-155332-mengxurelease63-ff905403cf4fe1f3\x01', 5)
for k, v in kvs:
    print(k, v)

>>> joshua_model.dir_ensemble_results_pass[b'\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00']
Subspace(rawPrefix=b'\x15\x18\x01\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\xff\x00')

tr.reset()
kvs = tr.get_range(b'\x15\x18\x01\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\xff\x00', b'\x15\x18\x01\x157\x0220201008-135157-XinDong-9848dd79e3aac22d\x00\xff\x01', 5)
for k, v in kvs:
    print(k, v)

>>> time_start = int(time.time() - 60*60*24*7)
>>> time_end = timestamp_of(None)
>>> failures = joshua_model.get_agent_failures(time_start, time_end)
>>> failures[0]
(('2020-Oct-03 (Sat) 11:25:01 AM', 'fdb-awsshared9'), b'Traceback (most recent call last):\n  File "/opt/rh/rh-python36/root/usr/lib64/python3.6/site-packages/joshua/joshua_agent.py", line 515, in agent\n    retcode = run_ensemble(chosen_ensemble, save_on, work_dir=work_dir)\n  File "/opt/rh/rh-python36/root/usr/lib64/python3.6/site-packages/joshua/joshua_agent.py", line 316, in run_ensemble\n    output, _ = process.communicate(timeout=1)\n  File "/opt/rh/rh-python36/root/usr/lib64/python3.6/subprocess.py", line 863, in communicate\n    stdout, stderr = self._communicate(input, endtime, timeout)\n  File "/opt/rh/rh-python36/root/usr/lib64/python3.6/subprocess.py", line 1525, in _communicate\n    selector.register(self.stdout, selectors.EVENT_READ)\n  File "/opt/rh/rh-python36/root/usr/lib64/python3.6/selectors.py", line 351, in register\n    key = super().register(fileobj, events, data)\n  File "/opt/rh/rh-python36/root/usr/lib64/python3.6/selectors.py", line 237, in register\n    key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data)\n  File "/opt/rh/rh-python36/root/usr/lib64/python3.6/selectors.py", line 224, in _fileobj_lookup\n    return _fileobj_to_fd(fileobj)\n  File "/opt/rh/rh-python36/root/usr/lib64/python3.6/selectors.py", line 39, in _fileobj_to_fd\n    "{!r}".format(fileobj)) from None\nValueError: Invalid file object: <_io.BufferedReader name=22>\n')