Merge branch 'master' into bcp-008-01

Author: jonathan-r-thorpe

.github/workflows/build-test.yml

@@ -10,35 +10,28 @@ on: [pull_request, push]
 env:
   SECRET_GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS }}
   SECRET_RESULTS_SHEET_ID: ${{ secrets.RESULTS_SHEET_ID }}
-  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 jobs:
   build_and_test:
     name: '${{ matrix.os }}: build and test (install mdns: ${{ matrix.install_mdns }}, use conan: ${{ matrix.use_conan }}, force cpprest asio: ${{ matrix.force_cpprest_asio }}, dns-sd mode: ${{ matrix.dns_sd_mode}}, enable_authorization: ${{ matrix.enable_authorization }})'
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-11, windows-2019]
+        os: [ubuntu-20.04, windows-2019]
         install_mdns: [false, true]
         use_conan: [true]
         force_cpprest_asio: [false]
         dns_sd_mode: [multicast, unicast]
         enable_authorization: [false, true]
         exclude:
           # install_mdns is only meaningful on Linux
-          - os: macos-11
-            enable_authorization: false
           - os: windows-2019
             enable_authorization: false
           - os: ubuntu-20.04
             enable_authorization: false
-          - os: macos-11
-            install_mdns: true
           - os: windows-2019
             install_mdns: true
           # for now, unicast DNS-SD tests are only implemented on Linux
-          - os: macos-11
-            dns_sd_mode: unicast
           - os: windows-2019
             dns_sd_mode: unicast
           # for now, exclude unicast DNS-SD with mDNSResponder due to
@@ -112,81 +105,9 @@ jobs:
 
     @import build-and-test
 
-  build_and_test_ubuntu_14:
-    name: '${{ matrix.os }}: build and test (install mdns: ${{ matrix.install_mdns }}, use conan: ${{ matrix.use_conan }}, force cpprest asio: ${{ matrix.force_cpprest_asio }}, dns-sd mode: ${{ matrix.dns_sd_mode}}, enable_authorization: ${{ matrix.enable_authorization }})'
-    runs-on: ubuntu-20.04
-    container:
-      image: ubuntu:14.04
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-14.04]
-        install_mdns: [true]
-        use_conan: [false]
-        force_cpprest_asio: [false]
-        dns_sd_mode: [multicast]
-        enable_authorization: [true]
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: set environment variables
-      shell: bash
-      run: |
-        if [[ "${{ runner.os }}"  == "Linux" ]]; then
-          if [[ "${{ matrix.install_mdns }}" == "true" ]]; then
-            echo "BUILD_NAME=${{ matrix.os }}_mdns_${{ matrix.dns_sd_mode }}" >> $GITHUB_ENV
-          else
-            echo "BUILD_NAME=${{ matrix.os }}_avahi_${{ matrix.dns_sd_mode }}" >> $GITHUB_ENV
-          fi
-        elif [[ "${{ matrix.force_cpprest_asio }}" == "true" ]]; then
-          echo "BUILD_NAME=${{ matrix.os }}_asio" >> $GITHUB_ENV
-        else
-          echo "BUILD_NAME=${{ matrix.os }}" >> $GITHUB_ENV
-        fi
-        GITHUB_COMMIT=`echo "${{ github.sha }}" | cut -c1-7`
-        echo "GITHUB_COMMIT=$GITHUB_COMMIT" >> $GITHUB_ENV
-        # github.workspace points to the host path not the docker path, the home directory defaults to the workspace directory
-        echo "GITHUB_WORKSPACE=`pwd`" >> $GITHUB_ENV
-        cd ..
-        echo "RUNNER_WORKSPACE=`pwd`" >> $GITHUB_ENV
-
-    - name: install build tools
-      run: |
-        apt-get update -q
-        apt-get install -y software-properties-common
-        apt-get --allow-unauthenticated update -q
-        apt-get --allow-unauthenticated install -y curl g++ git make patch zlib1g-dev libssl-dev bsdmainutils dnsutils unzip
-        # ubuntu-14.04 ca-certificates are out of date
-        git config --global http.sslVerify false
-        # build and install openssl
-        curl -OsSk https://openssl.org/source/old/1.1.1/openssl-1.1.1v.tar.gz
-        tar xzf openssl-1.1.1v.tar.gz
-        cd openssl-1.1.1v
-        ./config --prefix=/usr/local/custom-openssl --libdir=lib --openssldir=/etc/ssl
-        make -j1 depend
-        make -j8
-        make install_sw
-        cd ..
-        # install ffi.h, which is required for python build
-        apt install libffi-dev
-        # build and install python
-        curl -sSk https://www.python.org/ftp/python/3.11.5/Python-3.11.5.tar.xz | tar -xJ
-        cd Python-3.11.5
-        ./configure -C --with-openssl=/usr/local/custom-openssl --with-openssl-rpath=auto
-        make -j8
-        make install
-        update-alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.11 3
-        ln -s /usr/local/bin/python3.11 /usr/bin/python
-        curl -sS https://bootstrap.pypa.io/pip/3.6/get-pip.py | python
-        curl -sS https://nodejs.org/dist/v12.16.2/node-v12.16.2-linux-x64.tar.xz | tar -xJ
-        echo "`pwd`/node-v12.16.2-linux-x64/bin" >> $GITHUB_PATH
-
-    @import build-and-test
-
   make_badges:
     if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-    needs: [build_and_test, build_and_test_ubuntu_14]
+    needs: [build_and_test]
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v3
@@ -199,14 +120,14 @@ jobs:
         echo "GITHUB_WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV
         echo "RUNNER_WORKSPACE=${{ runner.workspace }}" >> $GITHUB_ENV
 
-    - uses: actions/download-artifact@v2.0
+    - uses: actions/download-artifact@v4
       with:
         path: ${{ runner.workspace }}/artifacts
 
     - name: make badges
       run: |
-        # combine badges from all builds, exclude macos-11
-        ${{ github.workspace }}/Sandbox/make_badges.sh ${{ github.workspace }} ${{ runner.workspace }}/artifacts macos-11_auth macos-11_noauth
+        # combine badges from all builds, exclude macos-13
+        ${{ github.workspace }}/Sandbox/make_badges.sh ${{ github.workspace }} ${{ runner.workspace }}/artifacts macos-13_auth macos-13_noauth
 
         # force push to github onto an orphan 'badges' branch
         cd ${{ github.workspace }}

.github/workflows/src/build-test.yml

@@ -6,12 +6,12 @@
     export SHEET=https://docs.google.com/spreadsheets/d/${{ env.SECRET_RESULTS_SHEET_ID }}
     python nmos-testing/utilities/run-test-suites/gsheetsImport/resultsImporter.py --credentials ${{ env.GDRIVE_CREDENTIALS }} --sheet "$SHEET" --insert --json nmos-testing/results/${{ env.GITHUB_COMMIT }}-*.json || echo "upload failed"
 
-- uses: actions/upload-artifact@v1
+- uses: actions/upload-artifact@v4
   with:
     name: ${{ env.BUILD_NAME }}_badges
     path: ${{ runner.workspace }}/nmos-testing/badges
 
-- uses: actions/upload-artifact@v1
+- uses: actions/upload-artifact@v4
   with:
     name: ${{ env.BUILD_NAME }}_results
     path: ${{ runner.workspace }}/nmos-testing/results

.github/workflows/src/save-results.yml

@@ -47,11 +47,13 @@ set(NMOS_CPP_TEST_NMOS_TEST_SOURCES
     nmos/test/did_sdid_test.cpp
     nmos/test/event_type_test.cpp
     nmos/test/json_validator_test.cpp
+    nmos/test/jwt_generator_test.cpp
     nmos/test/jwt_validation_test.cpp
     nmos/test/paging_utils_test.cpp
     nmos/test/query_api_test.cpp
     nmos/test/sdp_test_utils.cpp
     nmos/test/sdp_utils_test.cpp
+    nmos/test/slog_test.cpp
     nmos/test/system_resources_test.cpp
     nmos/test/video_jxsv_test.cpp
     )

Development/cmake/NmosCppTest.cmake

@@ -51,12 +51,7 @@ namespace web
             return !(lhs == rhs);
         }
 
-        inline bool operator<(const web::json::object& lhs, const web::json::object& rhs)
-        {
-            using std::begin;
-            using std::end;
-            return std::lexicographical_compare(begin(lhs), end(lhs), begin(rhs), end(rhs));
-        }
+        inline bool operator<(const web::json::object& lhs, const web::json::object& rhs);
 
         inline bool operator>(const web::json::object& lhs, const web::json::object& rhs)
         {
@@ -87,12 +82,7 @@ namespace web
             return !(lhs == rhs);
         }
 
-        inline bool operator<(const web::json::array& lhs, const web::json::array& rhs)
-        {
-            using std::begin;
-            using std::end;
-            return std::lexicographical_compare(begin(lhs), end(lhs), begin(rhs), end(rhs));
-        }
+        inline bool operator<(const web::json::array& lhs, const web::json::array& rhs);
 
         inline bool operator>(const web::json::array& lhs, const web::json::array& rhs)
         {
@@ -142,6 +132,20 @@ namespace web
         {
             return !(lhs < rhs);
         }
+
+        inline bool operator<(const web::json::array& lhs, const web::json::array& rhs)
+        {
+            using std::begin;
+            using std::end;
+            return std::lexicographical_compare(begin(lhs), end(lhs), begin(rhs), end(rhs));
+        }
+
+        inline bool operator<(const web::json::object& lhs, const web::json::object& rhs)
+        {
+            using std::begin;
+            using std::end;
+            return std::lexicographical_compare(begin(lhs), end(lhs), begin(rhs), end(rhs));
+        }
     }
 }
 

Development/cpprest/json_ops.h

@@ -1,5 +1,12 @@
 #include "mdns/service_discovery_impl.h"
 
+#if defined(_WIN32)
+#include <ws2tcpip.h>
+#include <iphlpapi.h>
+#pragma comment(lib, "IPHLPAPI.lib")
+#else
+#include <net/if.h>
+#endif
 #include <boost/asio/ip/address.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/algorithm/string/erase.hpp>
@@ -25,6 +32,24 @@ namespace mdns_details
         return kDNSServiceInterfaceIndexLocalOnly == interfaceIndex ? kDNSServiceInterfaceIndexAny : interfaceIndex;
     }
 
+    static inline std::string get_interface_name(std::uint32_t interfaceIndex)
+    {
+        char if_name[IF_NAMESIZE];
+
+        if (kDNSServiceInterfaceIndexAny == interfaceIndex) return "any interface";
+        // hm, on Windows, if_indextoname returns a name that is neither the AdapterName nor the FriendlyName from GetAdaptersAddresses
+        if (0 != if_indextoname(interfaceIndex, if_name)) return if_name;
+        return "unknown interface";
+    }
+
+    struct if_name_manip
+    {
+        std::uint32_t i = kDNSServiceInterfaceIndexAny;
+        explicit if_name_manip(std::uint32_t i) : i(i) {}
+        friend std::ostream& operator<<(std::ostream& os, const if_name_manip& manip) { return os << manip.i << " (" << get_interface_name(manip.i) << ")"; }
+    };
+    static inline if_name_manip put_interface_id(std::uint32_t interface_id) { return if_name_manip{ interface_id }; }
+
     struct browse_context
     {
         // browse in-flight state
@@ -52,7 +77,7 @@ namespace mdns_details
             {
                 const browse_result result{ serviceName, regtype, replyDomain, make_interface_id(interfaceIndex) };
 
-                slog::log<slog::severities::more_info>(impl->gate, SLOG_FLF) << "After DNSServiceBrowse, DNSServiceBrowseReply got service: " << result.name << " for regtype: " << result.type << " domain: " << result.domain << " on interface: " << result.interface_id;
+                slog::log<slog::severities::more_info>(impl->gate, SLOG_FLF) << "After DNSServiceBrowse, DNSServiceBrowseReply got service: " << result.name << " for regtype: " << result.type << " domain: " << result.domain << " on interface: " << put_interface_id(result.interface_id);
 
                 impl->had_enough = impl->handler(result);
             }
@@ -81,8 +106,7 @@ namespace mdns_details
         const auto latest_timeout = now + latest_timeout_;
         const auto earliest_timeout = now + earliest_timeout_;
 
-        // could use if_indextoname to get a name for the interface (remembering that 0 means "do the right thing", i.e. usually any interface, and there are some other special values too; see dns_sd.h)
-        slog::log<slog::severities::more_info>(gate, SLOG_FLF) << "DNSServiceBrowse for regtype: " << type << " domain: " << domain << " on interface: " << interface_id;
+        slog::log<slog::severities::more_info>(gate, SLOG_FLF) << "DNSServiceBrowse for regtype: " << type << " domain: " << domain << " on interface: " << put_interface_id(interface_id);
 
         browse_context context{ handler, had_enough, more_coming, gate };
         DNSServiceErrorType errorCode = DNSServiceBrowse(&client, 0, interface_id, type.c_str(), !domain.empty() ? domain.c_str() : NULL, browse_reply, &context);
@@ -285,8 +309,7 @@ namespace mdns_details
         const auto latest_timeout = now + latest_timeout_;
         const auto earliest_timeout = now + earliest_timeout_;
 
-        // could use if_indextoname to get a name for the interface (remembering that 0 means "do the right thing", i.e. usually any interface, and there are some other special values too; see dns_sd.h)
-        slog::log<slog::severities::more_info>(gate, SLOG_FLF) << "DNSServiceResolve for name: " << name << " regtype: " << type << " domain: " << domain << " on interface: " << interface_id;
+        slog::log<slog::severities::more_info>(gate, SLOG_FLF) << "DNSServiceResolve for name: " << name << " regtype: " << type << " domain: " << domain << " on interface: " << put_interface_id(interface_id);
 
         resolve_context context{ handler, had_enough, more_coming, gate };
         DNSServiceErrorType errorCode = DNSServiceResolve(&client, 0, interface_id, name.c_str(), type.c_str(), domain.c_str(), (DNSServiceResolveReply)resolve_reply, &context);
@@ -352,12 +375,12 @@ namespace mdns_details
         if (protocol == kDNSServiceProtocol_IPv4 && interface_id >= kIPv6IfIndexBase)
         {
             // no point trying in this case!
-            slog::log<slog::severities::too_much_info>(gate, SLOG_FLF) << "DNSServiceGetAddrInfo not tried for hostname: " << host_name << " on interface: " << interface_id;
+            slog::log<slog::severities::too_much_info>(gate, SLOG_FLF) << "DNSServiceGetAddrInfo not tried for hostname: " << host_name << " on interface: " << put_interface_id(interface_id);
         }
         else
 #endif
         {
-            slog::log<slog::severities::more_info>(gate, SLOG_FLF) << "DNSServiceGetAddrInfo for hostname: " << host_name << " on interface: " << interface_id;
+            slog::log<slog::severities::more_info>(gate, SLOG_FLF) << "DNSServiceGetAddrInfo for hostname: " << host_name << " on interface: " << put_interface_id(interface_id);
 
             getaddrinfo_context context{ handler, had_enough, more_coming, gate };
             DNSServiceErrorType errorCode = DNSServiceGetAddrInfo(&client, 0, interface_id, protocol, host_name.c_str(), getaddrinfo_reply, &context);

Development/mdns/service_discovery_impl.cpp

@@ -1260,7 +1260,7 @@ void node_implementation_init(nmos::node_model& model, nmos::experimental::contr
                 {
                     const auto receiver_id = impl::make_id(seed_id, nmos::types::receiver, port, index);
 
-                    utility::stringstream_t role;
+                    utility::ostringstream_t role;
                     role << U("monitor-") << ++count;
                     const auto& receiver = nmos::find_resource(model.node_resources, receiver_id);
                     const auto receiver_monitor = nmos::make_receiver_monitor(++oid, true, nmos::root_block_oid, role.str(), nmos::fields::label(receiver->data), nmos::fields::description(receiver->data), value_of({ { nmos::details::make_nc_touchpoint_nmos({nmos::ncp_nmos_resource_types::receiver, receiver_id}) } }));

Development/nmos-cpp-node/node_implementation.cpp

@@ -162,8 +162,8 @@ namespace nmos
                                     {
                                         return false;
                                     }
-                                    auto now = std::chrono::system_clock::now();
-                                    auto exp = std::chrono::system_clock::from_time_t(expires_at);
+                                    const auto now = std::chrono::system_clock::now();
+                                    const auto exp = std::chrono::system_clock::from_time_t(expires_at);
                                     return (now > exp);
                                 };
 

Development/nmos/authorization_behaviour.cpp

@@ -22,7 +22,7 @@ namespace nmos
             else
             {
                 utility::ifstream_t ca_file(ca_certificate_file);
-                utility::stringstream_t cacerts;
+                utility::ostringstream_t cacerts;
                 cacerts << ca_file.rdbuf();
                 return cacerts.str();
             }
@@ -70,7 +70,7 @@ namespace nmos
                 const auto private_key_file = nmos::experimental::fields::private_key_file(server_certificate);
                 const auto certificate_chain_file = nmos::experimental::fields::certificate_chain_file(server_certificate);
 
-                utility::stringstream_t pkey;
+                utility::ostringstream_t pkey;
                 if (private_key_file.empty())
                 {
                     slog::log<slog::severities::warning>(gate, SLOG_FLF) << "Missing server private key file";
@@ -81,7 +81,7 @@ namespace nmos
                     pkey << pkey_file.rdbuf();
                 }
 
-                utility::stringstream_t cert_chain;
+                utility::ostringstream_t cert_chain;
                 if (certificate_chain_file.empty())
                 {
                     slog::log<slog::severities::warning>(gate, SLOG_FLF) << "Missing server certificate chain file";
@@ -114,7 +114,7 @@ namespace nmos
             else
             {
                 utility::ifstream_t dh_file(dh_param_file);
-                utility::stringstream_t dh_param;
+                utility::ostringstream_t dh_param;
                 dh_param << dh_file.rdbuf();
                 return dh_param.str();
             }
@@ -160,7 +160,7 @@ namespace nmos
                 const auto key_algorithm = nmos::experimental::fields::key_algorithm(server_certificate);
                 const auto private_key_file = nmos::experimental::fields::private_key_file(server_certificate);
 
-                utility::stringstream_t pkey;
+                utility::ostringstream_t pkey;
                 if (private_key_file.empty())
                 {
                     slog::log<slog::severities::warning>(gate, SLOG_FLF) << "Missing private key file";

Development/nmos/certificate_handlers.cpp

@@ -296,10 +296,10 @@ namespace nmos
     {
         slog::log<slog::severities::too_much_info>(gate, SLOG_FLF) << "Sending request";
         // see https://developer.mozilla.org/en-US/docs/Web/API/Resource_Timing_API#Resource_loading_timestamps
-        const auto start_time = std::chrono::system_clock::now();
+        const auto start_time = std::chrono::steady_clock::now();
         return client.request(request, token).then([start_time, &gate](web::http::http_response res)
         {
-            const auto response_start = std::chrono::system_clock::now();
+            const auto response_start = std::chrono::steady_clock::now();
             const auto request_dur = std::chrono::duration_cast<std::chrono::microseconds>(response_start - start_time).count() / 1000.0;
 
             // see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Server-Timing