Merge remote-tracking branch 'origin/candidate-9.10.x'

Author: GordonSmith

.github/workflows/build-gh_runner.yml

@@ -208,7 +208,7 @@ jobs:
           for plugin in "${plugins[@]}"; do
             rm -f ./build/CMakeCache.txt
             rm -rf ./build/CMakeFiles
-            cmake ${{ !contains(inputs.os, 'windows') && '-G Ninja' || '' }} -S ./${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B ./build -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }}
+            cmake ${{ !contains(inputs.os, 'windows') && '-G Ninja' || '' }} -S ./${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B ./build -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DTESTING=ON -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }}
             cmake --build ./build ${{ contains(inputs.os, 'windows') && '--config RelWithDebInfo' || ''}} --parallel ${{ inputs.upload-package == true && '--target package' || ''}}
           done
 

cmake_modules/commonSetup.cmake

@@ -746,6 +746,10 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
          add_definitions (-D_CONTAINERIZED)
       ENDIF(CONTAINERIZED)
 
+      IF (TESTING)
+         add_definitions (-D_TESTING)
+      ENDIF(TESTING)
+
       IF (USE_ICU)
         find_package(ICU COMPONENTS data i18n io tu uc)
         IF (ICU_FOUND)

common/workunit/workunit.cpp

@@ -5248,6 +5248,18 @@ IWorkUnit* CWorkUnitFactory::createNamedWorkUnit(const char *wuid, const char *a
     IWorkUnit* ret = &cw->lockRemote(false);   // Note - this may throw exception if user does not have rights.
     ret->setDebugValue("CREATED_BY", app, true);
     ret->setDebugValue("CREATED_FOR", scope, true);
+    if (secuser)
+    {
+        //Record employee ids and numbers in the workunit if they are available.
+        //Future: This should probably be stored in an un-modifiable attribute and set by setUser()
+        const char * id = secuser->getEmployeeID();
+        if (!isEmptyString(id))
+            ret->setDebugValue("CREATOR_EMPLOYEE_ID", id, true);
+
+        const char * number = secuser->getEmployeeNumber();
+        if (!isEmptyString(number))
+            ret->setDebugValue("CREATOR_EMPLOYEE_NUMBER", number, true);
+    }
     return ret;
 }
 

dali/base/daclient.cpp

@@ -29,6 +29,7 @@
 #include "dautils.hpp"
 
 #include "daclient.hpp"
+#include "sysinfologger.hpp"
 
 extern bool registerClientProcess(ICommunicator *comm, IGroup *& retcoven,unsigned timeout,DaliClientRole role);
 extern void stopClientProcess();
@@ -141,6 +142,9 @@ bool initClientProcess(IGroup *servergrp, DaliClientRole role, unsigned mpport,
     covengrp->Release();
     queryLogMsgManager()->setSession(myProcessSession());
 
+    if (getComponentConfigSP()->getPropBool("logging/@enableGlobalSysLog"))
+        useDaliForOperatorMessages(true);
+
     if (!isContainerized()) // The Environment is bare-metal only
     {
         // auto install environment monitor for server roles
@@ -179,6 +183,7 @@ void removeShutdownHook(IDaliClientShutdown &shutdown)
 
 void closedownClientProcess()
 {
+    useDaliForOperatorMessages(false);
     if (!daliClientIsActive)
         return;
     while (shutdownHooks.ordinality())

dali/base/dafdesc.cpp

@@ -3845,40 +3845,67 @@ void disableStoragePlanesDaliUpdates()
 
 bool getDefaultStoragePlane(StringBuffer &ret)
 {
-#ifdef _CONTAINERIZED
+    if (!isContainerized())
+        return false;
     if (getDefaultPlane(ret, "@dataPlane", "data"))
         return true;
 
     throwUnexpectedX("Default data plane not specified"); // The default should always have been configured by the helm charts
-#else
-    return false;
-#endif
 }
 
 bool getDefaultSpillPlane(StringBuffer &ret)
 {
-#ifdef _CONTAINERIZED
-    if (getDefaultPlane(ret, "@spillPlane", "spill"))
+    if (!isContainerized())
+        return false;
+    if (getComponentConfigSP()->getProp("@spillPlane", ret))
+        return true;
+    else if (getGlobalConfigSP()->getProp("storage/@spillPlane", ret))
+        return true;
+    else if (getDefaultPlane(ret, nullptr, "spill"))
         return true;
 
     throwUnexpectedX("Default spill plane not specified"); // The default should always have been configured by the helm charts
-#else
-    return false;
-#endif
 }
 
 bool getDefaultIndexBuildStoragePlane(StringBuffer &ret)
 {
-#ifdef _CONTAINERIZED
+    if (!isContainerized())
+        return false;
     if (getComponentConfigSP()->getProp("@indexBuildPlane", ret))
         return true;
     else if (getGlobalConfigSP()->getProp("storage/@indexBuildPlane", ret))
         return true;
     else
         return getDefaultStoragePlane(ret);
-#else
-    return false;
-#endif
+}
+
+bool getDefaultPersistPlane(StringBuffer &ret)
+{
+    if (!isContainerized())
+        return false;
+    if (getComponentConfigSP()->getProp("@persistPlane", ret))
+        return true;
+    else if (getGlobalConfigSP()->getProp("storage/@persistPlane", ret))
+        return true;
+    else
+        return getDefaultStoragePlane(ret);
+}
+
+bool getDefaultJobTempPlane(StringBuffer &ret)
+{
+    if (!isContainerized())
+        return false;
+    if (getComponentConfigSP()->getProp("@jobTempPlane", ret))
+        return true;
+    else if (getGlobalConfigSP()->getProp("storage/@jobTempPlane", ret))
+        return true;
+    else
+    {
+        // NB: In hthor jobtemps are written to the spill plane and hence ephemeral storage by default
+        // In Thor they are written to the default data storage plane by default.
+        // This is because HThor doesn't need them persisted beyond the lifetime of the process, but Thor does.
+        return getDefaultStoragePlane(ret);
+    }
 }
 
 //---------------------------------------------------------------------------------------------------------------------

dali/base/dafdesc.hpp

@@ -412,6 +412,8 @@ extern da_decl void disableStoragePlanesDaliUpdates();
 extern da_decl bool getDefaultStoragePlane(StringBuffer &ret);
 extern da_decl bool getDefaultSpillPlane(StringBuffer &ret);
 extern da_decl bool getDefaultIndexBuildStoragePlane(StringBuffer &ret);
+extern da_decl bool getDefaultPersistPlane(StringBuffer &ret);
+extern da_decl bool getDefaultJobTempPlane(StringBuffer &ret);
 extern da_decl IStoragePlane * getDataStoragePlane(const char * name, bool required);
 extern da_decl IStoragePlane * getRemoteStoragePlane(const char * name, bool required);
 extern da_decl IStoragePlane * createStoragePlane(IPropertyTree *meta);