Skip to content

Commit fd289cf

Browse files
drbenmorgandrbenmorgan
authored
Merge pull request #82 from fmauger/feature-urn-management-support
Feature urn management support
2 parents 859e649 + 512b948 commit fd289cf

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+2256
-659
lines changed

‎CMakeLists.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ endif()
3838
# CMake/project requirements and configuration
3939
#
4040
cmake_minimum_required(VERSION 3.3 FATAL_ERROR)
41-
project(Falaise VERSION "3.1.1")
41+
project(Falaise VERSION "3.1.2")
4242

4343
# - Load custom modules
4444
list(INSERT CMAKE_MODULE_PATH 0 ${PROJECT_SOURCE_DIR}/cmake)
@@ -94,7 +94,7 @@ option(FALAISE_WITH_DOCS "Build documentation for Falaise" ON)
9494
# - GSL
9595
# - ROOT
9696
# - Geant4
97-
set(FALAISE_BAYEUX_MIN_VERSION 3.0.0)
97+
set(FALAISE_BAYEUX_MIN_VERSION 3.1.1)
9898
message( STATUS "Searching Bayeux ${FALAISE_BAYEUX_MIN_VERSION} ...")
9999
find_package(Bayeux ${FALAISE_BAYEUX_MIN_VERSION} REQUIRED geant4)
100100
# message( STATUS "Bayeux_VERSION = '${Bayeux_VERSION}'")

‎documentation/CMakeLists.txt

+2
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ set(FALAISE_MARKDOWN_DOCS
8282
flreconstruct/UsingServices.md
8383
flreconstruct/DocumentingFLReconstructModules.md
8484
flvisualize/FLVisualize.md
85+
resource_file_management/FLResourceFileManagement.md
86+
tag_management/FLTagManagement.md
8587
)
8688

8789
set(FALAISE_ABSOLUTE_MARKDOWN_DOCS)

‎documentation/Mainpage.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,10 @@ introductions to using and extending the core applications:
2525
- [Working with Run/Event Data in plugin modules](@ref workingwitheventrecords).
2626
- [Using Services to access metadata](@ref usingservices).
2727
- [FLVisualize](@ref usingflvisualize) guide for viewing simulated and reconstructed data.
28+
- [Resource files](@ref manageresourcefiles): guide for the management of resource files published in Falaise
29+
- [FLTags](@ref managetags): guide for using registered tags as identifiers for entities
2830

2931
Contributing to Falaise Development
3032
===================================
3133
Contributions to the core of Falaise are welcome. Please visit our [GitHub repo](https://github.com/SuperNEMO-DBD/Falaise)
3234
for further information.
33-

‎documentation/flreconstruct/FLReconstruct.md

+1-2
Original file line numberDiff line numberDiff line change
@@ -790,7 +790,7 @@ dependency scheme associated to this specific FLReconstruct script is:
790790
+-- urn:snemo:demonstrator:setup:1.0:services
791791
| +-- urn:snemo:demonstrator:geometry:4.0 (used geometry model)
792792
+-- urn:snemo:demonstrator:setup:1.0:variants (used variant configuration)
793-
| +-- urn:snemo:demonstrator:setup:1.0:variants:profiles:basic-1.0.0 (variant profile is fixed)
793+
+-- urn:snemo:demonstrator:setup:1.0:variants:profiles:basic-1.0.0 (variant profile is fixed)
794794
+-- urn:snemo:demonstrator:reconstruction:1.0.0:pipeline (reconstruction pipeline)
795795
~~~~~
796796

@@ -855,4 +855,3 @@ reconstruction via tuning existing modules or adding new ones then
855855
custom pipeline scripts and modules can be used in `flreconstruct`. The
856856
[Writing FLReconstruct Pipeline Scripts](@ref writingflreconstructpipelinescripts) tutorial covers the syntax and
857857
structure of custom pipeline scripts.
858-

‎documentation/flsimulate/FLSimulate.md

+3-3
Original file line numberDiff line numberDiff line change
@@ -521,7 +521,7 @@ displayed on the table below:
521521
+-- urn:snemo:demonstrator:simulation:2.1:services (used service configuration)
522522
| +-- urn:snemo:demonstrator:geometry:4.0 (used geometry model)
523523
+-- urn:snemo:demonstrator:simulation:2.1:variants (used variant configuration)
524-
| +-- urn:snemo:demonstrator:simulation:2.1:variants:profiles:default (default variant profile)
524+
+-- urn:snemo:demonstrator:simulation:2.1:variants:profiles:default (default variant profile)
525525
+-- urn:snemo:demonstrator:simulation:vertexes:4.1 (used vertex generation setup)
526526
+-- urn:snemo:demonstrator:simulation:decays:1.2 (used decay generation setup)
527527
~~~~~
@@ -948,8 +948,8 @@ List of available simulation setups {#usingflsimulate_summaryofavailablesimulati
948948
* Associated to:
949949
+ Services system: `"urn:snemo:demonstrator:simulation:2.1:services"`
950950
+ Variant system: `"urn:snemo:demonstrator:simulation:2.1:variants"`
951-
- Blessed profiles: the default one below
952-
- Default profile: `"urn:snemo:demonstrator:simulation:2.1:variants:profiles:default"`
951+
+ Blessed profiles: `"urn:snemo:demonstrator:simulation:2.1:variants:profiles:basic-1.0"`
952+
+ Default profile: `"urn:snemo:demonstrator:simulation:2.1:variants:profiles:default"`
953953

954954

955955
Available MC hits output profiles {#usingflsimulate_hits_output_profiles}

‎documentation/resource_file_management/FLResourceFileManagement.md

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
Manage resource files in Falaise {#manageresourcefiles}
2+
=================================
3+
4+
\tableofcontents
5+
6+
Introduction to resource files {#manageresourcefiles_intro}
7+
==============================
8+
9+
Each version of Falaise is delivered with a set of resource files,
10+
stored from the `resources/` source directory.
11+
12+
A resource file is typically a *configuration file* used by the
13+
simulation or the reconstruction pipeline to setup some specific
14+
components of the data processing software chain. It may also consist
15+
in a *data* file that must be loaded at program startup to initiate
16+
some algorithms or functionalities.
17+
18+
Most of the configuration files published in Falaise use a text format
19+
compatible with the ``datatools::properties`` or
20+
``datatools::multi_properties`` classes provided by the Bayeux
21+
library.
22+
23+
Resource files directory {#manageresourcefiles_resourcefilesdirectory}
24+
========================
25+
26+
The resources/ source directory is located at the root of the
27+
Falaise's source tree. After installation, the resources files will
28+
be stored from the ̀̀{Falaise install path}/share/Falaise-{falaise
29+
version}/resources/`` path.
30+
31+
The resources/ source directory is organized by topics :
32+
configuration files, examples, tags (URN management). Each time a new
33+
resource file has to be published, it must be stored in the relevant
34+
subdirectory.
35+
36+
For example, the
37+
``config/snemo/demonstrator/geometry/4.0/manager.conf`` file is the
38+
top configuration file for the SuperNEMO demonstrator's virtual
39+
geometry model identified with the tag ``4.0``. Should one add an
40+
improved geometry model typically tagged ``4.1``, it would make sense
41+
to store its new main configuration file as:
42+
``config/snemo/demonstrator/geometry/4.1/manager.conf``.
43+
44+
Usually, building a new configuration for a given application
45+
(simulation, reconstruction...) or service (geometry...) in Falaise
46+
implies publishing several new resource files at the same time.
47+
Developers or experts of the new functionality have to populate the
48+
relevant files in specific possibly new subdirectories.
49+
50+
Resource file registration {#manageresourcefiles_resourcefileregistration}
51+
==========================
52+
53+
54+
Once stored in the `resources/` source directory, the new files must
55+
be registered in Falaise. Falaise provides a dedicated script to
56+
generate automatically the list of all resource files that are part of
57+
a given release: the `resources/_BuildResourceList.sh` script. This
58+
script must be run every time a resource file is added in or removed
59+
from the `resources/` source directory. It builds the updated
60+
official list of resource files associated to a given Falaise release.
61+
This list is stored in the `resources/_resources.cmake` file which in
62+
turn is read by CMake during the build & installation process.
63+
64+
Note that files of which the names start with an underscore (example:
65+
``_foo.txt``) are *not* registered in the official list of resource
66+
files. This convention enables to store in the ``resources/``
67+
directory some files that are not intended to be published but only
68+
used by experts during the development process.
69+
70+
Details are given in the `resources/_README_DEVELOPERS.rst` file.
71+
72+
Resource file path in Falaise {#manageresourcefiles_resourcefilepathinfalaise}
73+
=============================
74+
75+
Falaise has a special mechanism to automatically locate the
76+
installation path of any resource file. A special syntax is used to
77+
describe the path of a resource file relatively to the resource
78+
installation top directory. The syntax simply uses the ``@falaise:``
79+
prefix in front of the relative path of the resource file. The
80+
``datatools::fetch_path_with_env(std::string &)`` function, provided
81+
by Bayeux, is used to resolve the real installation path of the file.
82+
Essentially, this prefix acts as an environment variable storing the
83+
installation path of the resource file directory.
84+
85+
Example: the real path of the main configuration file of the SuperNEMO geometry model
86+
tagged 4.0 can be addressed with the following string:
87+
``"@falaise:config/geometry/snemo/demonstrator/3.0/manager.conf"``

‎documentation/tag_management/FLTagManagement.md

+159
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
Manage tags in Falaise {#managetags}
2+
======================
3+
4+
\tableofcontents
5+
6+
7+
Introduction to tags {#managetags_intro}
8+
====================
9+
10+
A tag is the unique identifier of an official configuration of the
11+
SuperNEMO experiment. Such a configuration may concern a specific
12+
geometry model, the version of a software component, a model of the
13+
experiment in the context of the Monte Carlo simulation, an assembly
14+
of data processing modules for the reconstruction pipeline, a special
15+
hardware setup for data collection runs... A tag can also be used to
16+
identify some specific set of resources provided by/stored in a unique
17+
resource file.
18+
19+
Tag are thus associated to some *logical entities*. The tag does not
20+
provide the address, if any, of the tagged entity it is associated to.
21+
However, a tag **may** be associated to some concrete resource (i.e. a
22+
file). The real location of such a resource must be provided by a
23+
*resolver* tool, when this makes sense. Such a location can be the path
24+
of a resource file in the Falaise installation directory. It can be an
25+
URL to be accessed through a network, using a specific protocol.
26+
27+
A tag is implemented through the URN concept (see
28+
https://fr.wikipedia.org/wiki/Uniform_Resource_Name). An URN is a
29+
simple character string which uses the **urn** scheme.
30+
31+
Example: the "urn:snemo:demonstrator:setup:1.0" tag is associated to a
32+
specific model of the SuperNEMO demonstrator experiment, including its
33+
geometry model and possibly the configuration of the electronics, the
34+
database, the control and monitoring system. The
35+
"urn:snemo:demonstrator:setup:1.0" tag is thus **unique**. Once the
36+
model of the experiment, built as the assembly of several core
37+
components, is represented in the Falaise library and/or companion
38+
tools, its unique tag is officially published. The relationship
39+
between a specific setup and its tag is bijective. This is a strict
40+
rule that must be applied during the full lifecycle of the experiment.
41+
42+
43+
44+
Tag categories {#managetags_tagcategories}
45+
==============
46+
47+
As mentioned above, a tag is associated to a logical entity, i.e. something that makes sense
48+
to remember, store, associate to some other entities.
49+
50+
A tag must be associated to a **category**. A category is represented by a simple character string which identify
51+
a concept or topic. Falaise supports several tag categories:
52+
53+
* ``experiment`` : this category is used to address a specific experiment.
54+
55+
Example: BiPo3, SuperNEMO demonstrator and ATLAS can be considered as experiments.
56+
It is thus possible to associate an unique tag/URN to each of these entities:
57+
58+
* Atlas: ``"urn:atlas"``
59+
* SuperNEMO Demonstrator: ``"urn:snemo:demonstrator"``
60+
* BiPo3: ``"urn:bipo3"
61+
62+
An experiment entity provides the general context or framework used to operate
63+
a specific data processing.
64+
65+
* ``geosetup`` : this category is associated to any *geometry model/setup*.
66+
67+
Example: the SuperNEMO Demonstrator experiment may have distinct
68+
geometry models which can be chosen from an offical list (this is a mock example):
69+
70+
* Simple and naive geometry : ``"urn:snemo:demonstrator:geometry:1.0"``
71+
* R&D model : ``"urn:snemo:demonstrator:geometry:2.0"``
72+
* Idealised geometry : ``"urn:snemo:demonstrator:geometry:4.0"``
73+
* Improved geometry model (but still approximated) : ``"urn:snemo:demonstrator:geometry:4.2"``
74+
* Realistic geometry of the detector : ``"urn:snemo:demonstrator:geometry:5.0"``
75+
* Realistic geometry of the detector including a realistic model of the detector's environment : ``"urn:snemo:demonstrator:geometry:5.2"``
76+
77+
Depending on the context and the goal to be reached, the user can choose any of these list of geometry setups
78+
to operate some data processing (simulation, reconstruction...).
79+
80+
* ``expsetup`` : this category is associated to a model of the full *experimental setup*.
81+
An experimental setup generally includes:
82+
83+
* a specific geometry model
84+
* a specific configuration of the electronics, trigger system and DAQ
85+
* a specific configuration of the control and monitoring system
86+
* a specific configuration of various companion services (database...)
87+
88+
As an assembly of more fondamental entities, a specific experimental setup is expected
89+
to depends on its components. An experimental setup is associated to its unique tag.
90+
This tag automatically implies the tags (if any) of the components.
91+
92+
Example: the configuration of the SuperNEMO Demonstrator experimental setup during
93+
the commissioning phase of the experiment is tagged : ``"urn:snemo:demonstrator:setup:0.2"``.
94+
It implies the core components with the following list of tags (this is a mock example):
95+
96+
* ``"urn:snemo:demonstrator:geometry:5.0"``
97+
* ``"urn:snemo:demonstrator:electronics:0.9-com"``
98+
* ``"urn:snemo:demonstrator:cms:0.1"``
99+
* ``"urn:snemo:demonstrator:db:0.3"``
100+
101+
Any change in the setup of at least one component implies a new experimental setup and thus
102+
a new tag of the ``expsetup`` category.
103+
104+
* ``simsetup`` : this category is associated to a *simulation setup*
105+
* ``recsetup`` : this category is associated to a *reconstruction setup*
106+
* ``services`` : this category is associated to the configuration of a
107+
*service management system*
108+
* ``configuration`` : the category associated to the configuration
109+
of some generic system or service which is not classified/specialized as the category listed above
110+
* ``variants`` : the category associated to the configuration of a
111+
*variant service*
112+
* ``varprofile`` : the category associated to a *variant profile*.
113+
114+
115+
116+
117+
Tag registration {#managetags_tagregistration}
118+
================
119+
120+
When a tag is publised as the offical and unique identifier of some
121+
entity used within Falaise, it must be registered in a special
122+
database. A dedicated *URN database service* is provided by the
123+
Falaise library to manage the registration of all tags used in the
124+
library.
125+
126+
Practically, tags are described in files stored in the
127+
``resources/urn/db/`` source directory. A new URN entry must be added
128+
in one of the existing files from this directory in order to update
129+
the list of published tags. Eventually, additional files can be added
130+
to address some new categories of tags. These files are named *URN DB
131+
files*. They use the ``datatools::multi_properties`` format.
132+
133+
As a minimum, a tag record provides the URN of the tag, its category
134+
and a short description of the entity identified by the tag. It can
135+
also contains other informations such as the dependees of the entity,
136+
addressed by topic (see below).
137+
138+
The ``resources/urn/db/snemo_setup_db.conf`` files defines the full
139+
list of tag registration *URN DB* files to be loaded when the Falaise
140+
library starts.
141+
142+
143+
Tag dependencies {#managetags_tagdependencies}
144+
================
145+
146+
TBD
147+
148+
149+
Tag to file resolver {#managetags_tagtofileresolver}
150+
====================
151+
152+
153+
The fltags application {#managetags_fltags}
154+
=======================
155+
156+
Print usage :
157+
~~~~~~
158+
$ fltags --help
159+
~~~~~~

‎modules/EventBrowser/programs/FLVisualizeArgs.cc

+5-4
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
// This project:
1414
#include <falaise/app/metadata_utils.h>
15+
#include <falaise/tags.h>
1516
#include <falaise/snemo/detector/detector_manager.h>
1617
#include <falaise/snemo/view/options_manager.h>
1718
#include "FLVisualizeCommandLine.h"
@@ -202,7 +203,7 @@ void FLVisualizeArgs::do_postprocess(FLVisualizeArgs& flVisParameters) {
202203
<< flVisParameters.experimentalSetupUrn << "'...");
203204
// Check URN registration from the system URN query service:
204205
{
205-
std::string conf_category = "expsetup";
206+
std::string conf_category = falaise::tags::experimental_setup_category();
206207
DT_THROW_IF(!dtkUrnQuery.check_urn_info(flVisParameters.experimentalSetupUrn, conf_category),
207208
std::logic_error,
208209
"Cannot query URN='" << flVisParameters.experimentalSetupUrn << "'!");
@@ -233,7 +234,7 @@ void FLVisualizeArgs::do_postprocess(FLVisualizeArgs& flVisParameters) {
233234
"Services configuration URN='" << flVisParameters.servicesConfigUrn << "'...");
234235
// Check URN registration from the system URN query service:
235236
{
236-
std::string conf_category = "service";
237+
std::string conf_category = falaise::tags::services_category();
237238
DT_THROW_IF(!dtkUrnQuery.check_urn_info(servicesConfigUrn, conf_category), std::logic_error,
238239
"Cannot query URN='" << servicesConfigUrn << "'!");
239240
}
@@ -266,7 +267,7 @@ void FLVisualizeArgs::do_postprocess(FLVisualizeArgs& flVisParameters) {
266267
geometryConfigUrn = servicesConfigUrnInfo.get_component("geometry");
267268
// Check URN registration from the system URN query service:
268269
{
269-
std::string conf_category = "configuration";
270+
std::string conf_category = falaise::tags::geometry_setup_category();
270271
DT_THROW_IF(!dtkUrnQuery.check_urn_info(geometryConfigUrn, conf_category), std::logic_error,
271272
"Cannot query URN='" << geometryConfigUrn << "'!");
272273
}
@@ -330,7 +331,7 @@ void FLVisualizeArgs::do_postprocess(FLVisualizeArgs& flVisParameters) {
330331
"Variant configuration URN='" << variantConfigUrn << "'...");
331332
// Check URN registration from the system URN query service:
332333
{
333-
std::string conf_category = "variant";
334+
std::string conf_category = falaise::tags::variant_service_category();
334335
DT_THROW_IF(!dtkUrnQuery.check_urn_info(flVisParameters.variantConfigUrn, conf_category),
335336
std::logic_error,
336337
"Cannot query URN='" << flVisParameters.variantConfigUrn << "'!");

0 commit comments

Comments
 (0)