diff --git a/.github/workflows/build-gh_runner.yml b/.github/workflows/build-gh_runner.yml index 092e3dcd140..d77e464f08b 100644 --- a/.github/workflows/build-gh_runner.yml +++ b/.github/workflows/build-gh_runner.yml @@ -208,7 +208,7 @@ jobs: for plugin in "${plugins[@]}"; do rm -f ./build/CMakeCache.txt rm -rf ./build/CMakeFiles - cmake ${{ !contains(inputs.os, 'windows') && '-G Ninja' || '' }} -S ./${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B ./build -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }} + cmake ${{ !contains(inputs.os, 'windows') && '-G Ninja' || '' }} -S ./${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B ./build -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DTESTING=ON -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }} cmake --build ./build ${{ contains(inputs.os, 'windows') && '--config RelWithDebInfo' || ''}} --parallel ${{ inputs.upload-package == true && '--target package' || ''}} done diff --git a/cmake_modules/commonSetup.cmake b/cmake_modules/commonSetup.cmake index 956aa3bb110..b64e82cb276 100644 --- a/cmake_modules/commonSetup.cmake +++ b/cmake_modules/commonSetup.cmake @@ -746,6 +746,10 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "") add_definitions (-D_CONTAINERIZED) ENDIF(CONTAINERIZED) + IF (TESTING) + add_definitions (-D_TESTING) + ENDIF(TESTING) + IF (USE_ICU) find_package(ICU COMPONENTS data i18n io tu uc) IF (ICU_FOUND) diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp index 96bbb7a12b6..85326e670e1 100644 --- a/common/workunit/workunit.cpp +++ b/common/workunit/workunit.cpp @@ -5248,6 +5248,18 @@ IWorkUnit* CWorkUnitFactory::createNamedWorkUnit(const char *wuid, const char *a IWorkUnit* ret = &cw->lockRemote(false); // Note - this may throw exception if user does not have rights. ret->setDebugValue("CREATED_BY", app, true); ret->setDebugValue("CREATED_FOR", scope, true); + if (secuser) + { + //Record employee ids and numbers in the workunit if they are available. + //Future: This should probably be stored in an un-modifiable attribute and set by setUser() + const char * id = secuser->getEmployeeID(); + if (!isEmptyString(id)) + ret->setDebugValue("CREATOR_EMPLOYEE_ID", id, true); + + const char * number = secuser->getEmployeeNumber(); + if (!isEmptyString(number)) + ret->setDebugValue("CREATOR_EMPLOYEE_NUMBER", number, true); + } return ret; } diff --git a/dali/base/daclient.cpp b/dali/base/daclient.cpp index a36d418a851..1273adde6f4 100644 --- a/dali/base/daclient.cpp +++ b/dali/base/daclient.cpp @@ -29,6 +29,7 @@ #include "dautils.hpp" #include "daclient.hpp" +#include "sysinfologger.hpp" extern bool registerClientProcess(ICommunicator *comm, IGroup *& retcoven,unsigned timeout,DaliClientRole role); extern void stopClientProcess(); @@ -141,6 +142,9 @@ bool initClientProcess(IGroup *servergrp, DaliClientRole role, unsigned mpport, covengrp->Release(); queryLogMsgManager()->setSession(myProcessSession()); + if (getComponentConfigSP()->getPropBool("logging/@enableGlobalSysLog")) + useDaliForOperatorMessages(true); + if (!isContainerized()) // The Environment is bare-metal only { // auto install environment monitor for server roles @@ -179,6 +183,7 @@ void removeShutdownHook(IDaliClientShutdown &shutdown) void closedownClientProcess() { + useDaliForOperatorMessages(false); if (!daliClientIsActive) return; while (shutdownHooks.ordinality()) diff --git a/dali/base/dafdesc.cpp b/dali/base/dafdesc.cpp index 25d19f44ff7..cd903c63187 100644 --- a/dali/base/dafdesc.cpp +++ b/dali/base/dafdesc.cpp @@ -3845,40 +3845,67 @@ void disableStoragePlanesDaliUpdates() bool getDefaultStoragePlane(StringBuffer &ret) { -#ifdef _CONTAINERIZED + if (!isContainerized()) + return false; if (getDefaultPlane(ret, "@dataPlane", "data")) return true; throwUnexpectedX("Default data plane not specified"); // The default should always have been configured by the helm charts -#else - return false; -#endif } bool getDefaultSpillPlane(StringBuffer &ret) { -#ifdef _CONTAINERIZED - if (getDefaultPlane(ret, "@spillPlane", "spill")) + if (!isContainerized()) + return false; + if (getComponentConfigSP()->getProp("@spillPlane", ret)) + return true; + else if (getGlobalConfigSP()->getProp("storage/@spillPlane", ret)) + return true; + else if (getDefaultPlane(ret, nullptr, "spill")) return true; throwUnexpectedX("Default spill plane not specified"); // The default should always have been configured by the helm charts -#else - return false; -#endif } bool getDefaultIndexBuildStoragePlane(StringBuffer &ret) { -#ifdef _CONTAINERIZED + if (!isContainerized()) + return false; if (getComponentConfigSP()->getProp("@indexBuildPlane", ret)) return true; else if (getGlobalConfigSP()->getProp("storage/@indexBuildPlane", ret)) return true; else return getDefaultStoragePlane(ret); -#else - return false; -#endif +} + +bool getDefaultPersistPlane(StringBuffer &ret) +{ + if (!isContainerized()) + return false; + if (getComponentConfigSP()->getProp("@persistPlane", ret)) + return true; + else if (getGlobalConfigSP()->getProp("storage/@persistPlane", ret)) + return true; + else + return getDefaultStoragePlane(ret); +} + +bool getDefaultJobTempPlane(StringBuffer &ret) +{ + if (!isContainerized()) + return false; + if (getComponentConfigSP()->getProp("@jobTempPlane", ret)) + return true; + else if (getGlobalConfigSP()->getProp("storage/@jobTempPlane", ret)) + return true; + else + { + // NB: In hthor jobtemps are written to the spill plane and hence ephemeral storage by default + // In Thor they are written to the default data storage plane by default. + // This is because HThor doesn't need them persisted beyond the lifetime of the process, but Thor does. + return getDefaultStoragePlane(ret); + } } //--------------------------------------------------------------------------------------------------------------------- diff --git a/dali/base/dafdesc.hpp b/dali/base/dafdesc.hpp index a9a4e1dee51..989686cfc8e 100644 --- a/dali/base/dafdesc.hpp +++ b/dali/base/dafdesc.hpp @@ -412,6 +412,8 @@ extern da_decl void disableStoragePlanesDaliUpdates(); extern da_decl bool getDefaultStoragePlane(StringBuffer &ret); extern da_decl bool getDefaultSpillPlane(StringBuffer &ret); extern da_decl bool getDefaultIndexBuildStoragePlane(StringBuffer &ret); +extern da_decl bool getDefaultPersistPlane(StringBuffer &ret); +extern da_decl bool getDefaultJobTempPlane(StringBuffer &ret); extern da_decl IStoragePlane * getDataStoragePlane(const char * name, bool required); extern da_decl IStoragePlane * getRemoteStoragePlane(const char * name, bool required); extern da_decl IStoragePlane * createStoragePlane(IPropertyTree *meta); diff --git a/dali/base/sysinfologger.cpp b/dali/base/sysinfologger.cpp index 35aca689ee1..64bca9f0146 100644 --- a/dali/base/sysinfologger.cpp +++ b/dali/base/sysinfologger.cpp @@ -16,7 +16,6 @@ ############################################################################## */ #include "sysinfologger.hpp" -#include "daclient.hpp" #include "jutil.hpp" #define SDS_LOCK_TIMEOUT (5*60*1000) // 5 minutes @@ -144,12 +143,12 @@ class CSysInfoLoggerMsg : implements ISysInfoLoggerMsg const char *msg = msgPtree->queryProp(nullptr); return msg ? msg : ""; } - void setHidden(bool _hidden) + virtual void setHidden(bool _hidden) override { ensureUpdateable(); msgPtree->setPropBool(ATTR_HIDDEN, _hidden); } - StringBuffer & getXpath(StringBuffer & xpath) + virtual StringBuffer & getXpath(StringBuffer & xpath) const override { unsigned year, month, day; extractDate(queryTimeStamp(), year, month, day); @@ -422,10 +421,10 @@ ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(unsigned __int64 msgId, c class CSysInfoLoggerMsgIterator : public CSimpleInterfaceOf { - Linked filter; // N.b. IRemoteConnection exists for the duration of the iterator so if this iterator exists for too long, it could cause // performance issues for other clients: consider caching some messages and releasing connection (and reopening as necessary). - Owned conn; + Linked conn; + Linked filter; bool updateable = false; Owned msgIter; CSysInfoLoggerMsg infoMsg; @@ -446,15 +445,8 @@ class CSysInfoLoggerMsgIterator : public CSimpleInterfaceOfisValid() : false; } }; +class NullSysInfoLoggerMsgIterator : public CSimpleInterfaceOf +{ +public: + virtual ISysInfoLoggerMsg & query() override + { + UNIMPLEMENTED; + } + virtual bool first() override + { + return false; + } + virtual bool next() override + { + return false; + } + virtual bool isValid() override + { + return false; + } +}; + +ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(IRemoteConnection * conn, IConstSysInfoLoggerMsgFilter * msgFilter, bool updateable) +{ + return new CSysInfoLoggerMsgIterator(conn, msgFilter, updateable); +} + +ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(IConstSysInfoLoggerMsgFilter * msgFilter, bool updateable) +{ + unsigned mode = updateable ? RTM_LOCK_WRITE : RTM_LOCK_READ; + Owned conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), mode, SDS_LOCK_TIMEOUT); + if (!conn) + return new NullSysInfoLoggerMsgIterator(); + return createSysInfoLoggerMsgIterator(conn, msgFilter, updateable); +} -ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source) +ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(IRemoteConnection * conn, bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source, bool updateable) { Owned filter = new CSysInfoLoggerMsgFilter(visibleOnly, hiddenOnly, year, month, day, source); - return new CSysInfoLoggerMsgIterator(filter, false); + return createSysInfoLoggerMsgIterator(conn, filter, updateable); } -ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(IConstSysInfoLoggerMsgFilter * msgFilter) +ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source, bool updateable) { - return new CSysInfoLoggerMsgIterator(msgFilter, false); + Owned filter = new CSysInfoLoggerMsgFilter(visibleOnly, hiddenOnly, year, month, day, source); + return createSysInfoLoggerMsgIterator(filter, updateable); } + // returns messageId unsigned __int64 logSysInfoError(const LogMsgCategory & cat, LogMsgCode code, const char *source, const char * msg, timestamp_type ts) { @@ -530,20 +558,20 @@ unsigned __int64 logSysInfoError(const LogMsgCategory & cat, LogMsgCode code, co return makeMessageId(ts, seqn); } -unsigned updateMessage(IConstSysInfoLoggerMsgFilter * msgFilter, std::function updateOp) +unsigned updateMessage(IConstSysInfoLoggerMsgFilter * msgFilter, std::function updateOp) { unsigned count = 0; - Owned iter = new CSysInfoLoggerMsgIterator(msgFilter, true); + Owned iter = createSysInfoLoggerMsgIterator(msgFilter, true); ForEach(*iter) { - CSysInfoLoggerMsg & sysInfoMsg = iter->queryInfoLoggerMsg(); + ISysInfoLoggerMsg & sysInfoMsg = iter->query(); updateOp(sysInfoMsg); ++count; } return count; } -unsigned updateMessage(unsigned __int64 msgId, const char *source, std::function updateOp) +unsigned updateMessage(unsigned __int64 msgId, const char *source, std::function updateOp) { Owned msgFilter = createSysInfoLoggerMsgFilter(msgId, source); return updateMessage(msgFilter, updateOp); @@ -551,22 +579,22 @@ unsigned updateMessage(unsigned __int64 msgId, const char *source, std::function unsigned hideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter) { - return updateMessage(msgFilter, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(true);}); + return updateMessage(msgFilter, [](ISysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(true);}); } bool hideLogSysInfoMsg(unsigned __int64 msgId, const char *source) { - return updateMessage(msgId, source, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(true);})==1; + return updateMessage(msgId, source, [](ISysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(true);})==1; } unsigned unhideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter) { - return updateMessage(msgFilter, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(false);}); + return updateMessage(msgFilter, [](ISysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(false);}); } bool unhideLogSysInfoMsg(unsigned __int64 msgId, const char *source) { - return updateMessage(msgId, source, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(false);})==1; + return updateMessage(msgId, source, [](ISysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(false);})==1; } unsigned deleteLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter) @@ -574,10 +602,10 @@ unsigned deleteLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter) std::vector deleteXpathList; Owned conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT); { - Owned iter = new CSysInfoLoggerMsgIterator(msgFilter, false, conn.getLink()); + Owned iter = createSysInfoLoggerMsgIterator(conn, msgFilter, false); ForEach(*iter) { - CSysInfoLoggerMsg & sysInfoMsg = iter->queryInfoLoggerMsg(); + IConstSysInfoLoggerMsg & sysInfoMsg = iter->query(); StringBuffer xpath; sysInfoMsg.getXpath(xpath); deleteXpathList.push_back(xpath.str()); @@ -702,3 +730,81 @@ unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigne return count; } + +class DaliMsgLoggerHandler : public CSimpleInterfaceOf +{ +public: + DaliMsgLoggerHandler(unsigned _messageFields=MSGFIELD_all) : messageFields(_messageFields) + { + } + virtual void handleMessage(const LogMsg & msg) override + { + LogMsgSysInfo sysInfo = msg.querySysInfo(); + time_t timeNum = sysInfo.queryTime(); + unsigned __int64 ts = sysInfo.queryTime() * 1000000 + sysInfo.queryUSecs(); + logSysInfoError(msg.queryCategory(), msg.queryCode(), queryComponentName(), msg.queryText(), ts); + } + virtual bool needsPrep() const override + { + return false; + } + virtual void prep() override + { + } + virtual unsigned queryMessageFields() const override + { + return messageFields; + } + virtual void setMessageFields(unsigned _fields = MSGFIELD_all) override + { + messageFields = _fields; + } + virtual void addToPTree(IPropertyTree * parent) const override + { + IPropertyTree * handlerTree = createPTree(ipt_caseInsensitive); + handlerTree->setProp("@type", "globalmessages"); + handlerTree->setPropInt("@fields", messageFields); + parent->addPropTree("handler", handlerTree); + } + virtual int flush() override + { + return 0; + } + virtual bool getLogName(StringBuffer &name) const override + { + return false; + } + virtual offset_t getLogPosition(StringBuffer &logFileName) const override + { + return 0; + } +private: + unsigned messageFields = MSGFIELD_all; +}; + +static Owned msgHandler; + +void useDaliForOperatorMessages(bool use) +{ + if (use) + { + if (!msgHandler) + { + msgHandler.setown(getDaliMsgLoggerHandler()); + Owned operatorFilter = getCategoryLogMsgFilter(MSGAUD_operator, + MSGCLS_disaster|MSGCLS_error|MSGCLS_warning, + WarnMsgThreshold); + queryLogMsgManager()->addMonitor(msgHandler, operatorFilter); + } + } + else if (msgHandler) + { + queryLogMsgManager()->removeMonitor(msgHandler); + msgHandler.clear(); + } +} + +ILogMsgHandler * getDaliMsgLoggerHandler() +{ + return new DaliMsgLoggerHandler(); +} diff --git a/dali/base/sysinfologger.hpp b/dali/base/sysinfologger.hpp index 9eb0d986e66..e62e66d7f08 100644 --- a/dali/base/sysinfologger.hpp +++ b/dali/base/sysinfologger.hpp @@ -20,6 +20,7 @@ #include "jlog.hpp" #include "jutil.hpp" +#include "daclient.hpp" #ifdef DALI_EXPORT #define SYSINFO_API DECL_EXPORT @@ -27,7 +28,7 @@ #define SYSINFO_API DECL_IMPORT #endif -interface ISysInfoLoggerMsg +interface IConstSysInfoLoggerMsg { virtual bool queryIsHidden() const = 0; virtual timestamp_type queryTimeStamp() const = 0; @@ -37,6 +38,12 @@ interface ISysInfoLoggerMsg virtual LogMsgClass queryClass() const = 0; virtual unsigned __int64 queryLogMsgId() const = 0; virtual const char * queryMsg() const = 0; + virtual StringBuffer & getXpath(StringBuffer & xpath) const = 0; +}; + +interface ISysInfoLoggerMsg : public IConstSysInfoLoggerMsg +{ + virtual void setHidden(bool _hidden) = 0; }; interface IConstSysInfoLoggerMsgFilter : public IInterface @@ -77,8 +84,10 @@ typedef IIteratorOf ISysInfoLoggerMsgIterator; SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(const char *source=nullptr); SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(unsigned __int64 msgId, const char *source=nullptr); -SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool _visibleOnly, bool _hiddenOnly, unsigned _year, unsigned _month, unsigned _day, const char *source=nullptr); -SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(ISysInfoLoggerMsgFilter * msgFilter); +SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(IRemoteConnection * conn, IConstSysInfoLoggerMsgFilter * msgFilter, bool updateable = false); +SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(IConstSysInfoLoggerMsgFilter * msgFilter, bool updateable = false); +SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(IRemoteConnection * conn, bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source, bool updateable = false); +SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source, bool updateable = false); SYSINFO_API unsigned __int64 logSysInfoError(const LogMsgCategory & cat, LogMsgCode code, const char *source, const char * msg, unsigned __int64 ts); SYSINFO_API unsigned hideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter); @@ -93,4 +102,6 @@ SYSINFO_API unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenO - nonSysInfoLogMsg flag is used to identify whether or not the message is managed by SysInfoLogger */ SYSINFO_API unsigned __int64 makeMessageId(unsigned __int64 ts, unsigned seqN, bool nonSysInfoLogMsg=false); SYSINFO_API unsigned __int64 makeMessageId(unsigned year, unsigned month, unsigned day, unsigned seqN, bool nonSysInfoLogMsg=false); +SYSINFO_API ILogMsgHandler * getDaliMsgLoggerHandler(); +SYSINFO_API void useDaliForOperatorMessages(bool use); #endif diff --git a/dockerfiles/incr.sh b/dockerfiles/incr.sh deleted file mode 100755 index b03584663c7..00000000000 --- a/dockerfiles/incr.sh +++ /dev/null @@ -1,228 +0,0 @@ -#!/bin/bash - -############################################################################## -# -# HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################## - -# Use this script to build local images for testing, while leveraging the work already done in the most recent tag -# on the branch that your local branch was based on -# -# Pass in a get commit (that you previously built using this script) if you want to override the default calculation -# of the base build - - -# NB: INPUT_* may be pre-set as environment variables. - -#The name of your "upstream" git remote -UPSTREAM=upstream - -while getopts “d:fhvlpt:n:u:b:r:a:” opt; do - case $opt in - l) TAGLATEST=1 ;; - n) CUSTOM_TAG_NAME=$OPTARG ;; - p) PUSH=1 ;; - t) INPUT_BUILD_THREADS=$OPTARG ;; - d) INPUT_DOCKER_REPO=$OPTARG ;; - u) INPUT_BUILD_USER=$OPTARG ;; - a) INPUT_GITHUB_TOKEN=$OPTARG ;; - b) INPUT_BUILD_TYPE=$OPTARG ;; - r) UPSTREAM=$OPTARG ;; - f) FORCE=1 ;; - v) BUILDKIT_PROGRESS=plain ;; - h) echo "Usage: incr.sh [options] [prev]" - echo " -d Specify the repo to publish images to" - echo " -f Force build from scratch" - echo " -b Build type (e.g. Debug / Release)" - echo " -h Display help" - echo " -l Tag the images as the latest" - echo " -n Tag the image with a custom name (e.g. -n HPCC-25285)" - echo " -p Push images to docker repo" - echo " -r Override git upstream repo name (default is 'upstream')" - echo " -t Override the number of build threads" - echo " -u Specify the build user" - echo " -a Personal access token for github packages" - echo " -v use verbose buildkit mode" - echo "The optional prev argument indicates a remote base image that will be pulled as the starting point." - echo "If not specified, will use the most recent local image, or attempt to deduce the best remote base" - echo "from the git history. prev should be in the form (e.g.) 9.0.0-rc1" - exit - ;; - esac -done -shift $(( $OPTIND-1 )) - -set -e - -# NB: Not used if FORCE build -PREV=$1 - -DOCKER_REPO=hpccsystems -[[ -n ${INPUT_DOCKER_REPO} ]] && DOCKER_REPO=${INPUT_DOCKER_REPO} -INCR_DOCKER_REPO=${DOCKER_REPO} -BUILD_TYPE=Debug -[[ -n ${INPUT_BUILD_TYPE} ]] && BUILD_TYPE=${INPUT_BUILD_TYPE} -BUILD_THREADS=$INPUT_BUILD_THREADS # If not set, picks up default based on nproc - -# BUILD_USER used when building from scatch with force option (-f) -BUILD_USER=hpccsystems -[[ -n ${INPUT_BUILD_USER} ]] && BUILD_USER=${INPUT_BUILD_USER} - -# GITHUB_TOKEN used when building from scatch with force option (-f) -GITHUB_TOKEN=none -[[ -n ${INPUT_GITHUB_TOKEN} ]] && GITHUB_TOKEN=${INPUT_GITHUB_TOKEN} - -HEAD=$(git rev-parse --short HEAD) -BUILD_LABEL="${HEAD}-${BUILD_TYPE}" - -if [[ -z "$FORCE" ]] ; then - # Look for an image that matches a commit, exclude images based on dirty working tree. - if [[ -z ${PREV} ]] ; then - docker images ${DOCKER_REPO}/platform-build --format {{.Tag}} | egrep -ve '-dirty.*$' > .candidate-tags || true - PREV=$(git log --format=format:%h-${BUILD_TYPE} $(git describe --abbrev=0 --tags)..HEAD | fgrep -f .candidate-tags | head -n 1) - rm -f .candidate-tags - PREV_COMMIT=$(echo "${PREV}" | sed -e "s/-${BUILD_TYPE}.*$//") - fi - -# Work out which candidate branch this one was probably forked from - - - # If not found above, look for latest tagged - if [[ -z ${PREV} ]] ; then - best=1000000 - for candidate in `git branch -r -l ${UPSTREAM}/candidate-9* | sed 's/\*//' | sed s/.*candidate-// | egrep -v [.]x$ | sort | uniq` ; do - delta=$(git rev-list --count --no-merges ${UPSTREAM}/candidate-$candidate...HEAD) - # echo $candidate : $delta - if [ $delta -lt $best ] ; then - best=$delta - PREV=`git tag -l | grep ${candidate} | tail -1 | sed s/.*_//` - fi - done - if [[ -z ${PREV} ]] ; then - echo "Could not locate version to base from - please specify manually" - exit - fi - if [[ ! "${BUILD_TYPE}" =~ "Release" ]] ; then - PREV=${PREV}-${BUILD_TYPE} - fi - echo Finding image based on most recent git tag: ${PREV} - INCR_DOCKER_REPO=hpccsystems - docker pull ${INCR_DOCKER_REPO}/platform-build:${PREV} - if [ $? -ne 0 ]; then - echo "Could not locate docker image based on PREV tag: ${PREV} for docker user: ${DOCKER_REPO}" - exit - fi - else - if [[ ! "${BUILD_TYPE}" =~ "Release" ]] ; then - if [[ ! "${PREV}" =~ "-${BUILD_TYPE}" ]] ; then - PREV=${PREV}-${BUILD_TYPE} - fi - fi - fi - - if [[ -z ${PREV_COMMIT} ]] ; then - PREV_COMMIT=community_$(echo "${PREV}" | sed -e "s/-${BUILD_TYPE}.*$//") - # check if manually specified base (PREV) was gold release. Gold if no trailing - - # if gold, add "-1" to match git tagging. - pat=".+-.+$" - if [[ ! ${PREV_COMMIT} =~ ${pat} ]] ; then - PREV_COMMIT=${PREV_COMMIT}-1 - fi - fi - - # create empty patch file - echo -n > platform-build-incremental/hpcc.gitpatch - porcelain=$(git status -uno --porcelain) - if [[ -n "${porcelain}" || "${HEAD}" != "${PREV_COMMIT}" ]] ; then - echo "git diff --binary ${PREV_COMMIT} ':!./' > platform-build-incremental/hpcc.gitpatch" - git diff --binary ${PREV_COMMIT} ':!./' > platform-build-incremental/hpcc.gitpatch - # PATCH_MD5 is an ARG of the docker file, which ensures that if different from cached version, image will rebuild from that stage - PATCH_MD5=$(md5sum platform-build-incremental/hpcc.gitpatch | awk '{print $1}') - if [[ -n "${porcelain}" ]] ; then - DIRTY="-dirty-${PATCH_MD5}" - fi - # If working tree is dirty, annotate build tag, so doesn't conflict with base commit - BUILD_LABEL="${BUILD_LABEL}${DIRTY}" - elif [[ "$BUILD_LABEL" == "$PREV" ]] ; then - echo Checking if docker image ${DOCKER_REPO}/platform-core:${BUILD_LABEL} already exists - EXISTING_IMAGE_LABEL=$(git log --format=format:%h-${BUILD_TYPE} $(git describe --abbrev=0 --tags)..HEAD | grep `docker images ${DOCKER_REPO}/platform-build --format {{.Tag}} | head -n 2 | tail -n 1`) - if [[ -n ${EXISTING_IMAGE_LABEL} ]] ; then - PREV=${EXISTING_IMAGE_LABEL} - PREV_COMMIT=$(echo "${PREV}" | sed -e "s/-${BUILD_TYPE}//") - fi - fi -fi - -set -e - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -pushd $DIR 2>&1 > /dev/null - - -build_image() { - local name=$1 - local dockerfolder=$2 - local tags=("${BUILD_LABEL}") - # if 2nd arg. present, it names the docker folder, otherwise same as $name - [[ -z ${dockerfolder} ]] && dockerfolder=$name - local label=$BUILD_LABEL - - if ! docker pull ${DOCKER_REPO}/${name}:${label} ; then - docker image build -t ${DOCKER_REPO}/${name}:${label} \ - --build-arg DOCKER_REPO=${DOCKER_REPO} \ - --build-arg BUILD_LABEL=${BUILD_LABEL} \ - --build-arg BUILD_TYPE=${BUILD_TYPE} ${@:3} \ - ${dockerfolder}/ - fi - - if [ "$TAGLATEST" = "1" ]; then - local tag="latest" - docker tag ${DOCKER_REPO}/${name}:${label} ${DOCKER_REPO}/${name}:${tag} - tags+=("$tag") - fi - - if [ -n "$CUSTOM_TAG_NAME" ]; then - local tag="$CUSTOM_TAG_NAME" - docker tag ${DOCKER_REPO}/${name}:${label} ${DOCKER_REPO}/${name}:${tag} - tags+=("$tag") - fi - - if [ "$PUSH" = "1" ] ; then - for tag in "${tags[@]}"; - do - docker push ${DOCKER_REPO}/${name}:${tag} - echo "${tag} pushed" - done - fi -} - -if [[ -n "$FORCE" ]] ; then - echo Building local forced build images [ BUILD_LABEL=${BUILD_LABEL} ] - build_image platform-build platform-build --build-arg BUILD_USER=${BUILD_USER} --build-arg GITHUB_ACTOR=${BUILD_USER} --build-arg GITHUB_TOKEN=${GITHUB_TOKEN} --build-arg BUILD_TAG=${HEAD} --build-arg BUILD_THREADS=${BUILD_THREADS} -else - echo Building local incremental images [ BUILD_LABEL=${BUILD_LABEL} ] based on ${PREV} - build_image platform-build platform-build-incremental --build-arg DOCKER_REPO=${INCR_DOCKER_REPO} --build-arg PREV_LABEL=${PREV} --build-arg PATCH_MD5=${PATCH_MD5} --build-arg BUILD_THREADS=${BUILD_THREADS} -# rm platform-build-incremental/hpcc.gitpatch -fi - -if [[ "${BUILD_TYPE}" =~ "Release" ]] ; then - echo BUILDING: build_image platform-core - build_image platform-core -else - echo BUILDING: build_image platform-core platform-core-debug - build_image platform-core platform-core-debug -fi - -echo Built ${DOCKER_REPO}/*:${BUILD_LABEL} diff --git a/docs/BuildTools/fo.xsl b/docs/BuildTools/fo.xsl index d47047f7723..e3b89560c6f 100644 --- a/docs/BuildTools/fo.xsl +++ b/docs/BuildTools/fo.xsl @@ -99,7 +99,7 @@ - + @@ -107,7 +107,8 @@ - + + sans-serif @@ -116,13 +117,44 @@ - + + + + + 7pt + + + + + + + 6pt + + + + + + + 5pt + + + + + + + 4pt + + + + + sans-serif + serif @@ -130,12 +162,14 @@ + 10pt + 10pt @@ -143,7 +177,8 @@ - + + false @@ -153,14 +188,14 @@ - - + 0.83em + @@ -265,7 +300,7 @@ - + @@ -353,7 +388,7 @@ - + @@ -365,6 +400,7 @@ + @@ -372,6 +408,7 @@ + diff --git a/docs/BuildTools/fo.xsl.in b/docs/BuildTools/fo.xsl.in index 8778ec168d0..c3d9d355111 100644 --- a/docs/BuildTools/fo.xsl.in +++ b/docs/BuildTools/fo.xsl.in @@ -1,4 +1,4 @@ - + @@ -99,7 +99,7 @@ - + @@ -107,17 +107,17 @@ - + + - sans-serif + sans-serif 10pt - - + @@ -140,13 +140,21 @@ - + + + 4pt + + + + + sans-serif + serif @@ -154,12 +162,14 @@ + 10pt + 10pt @@ -168,23 +178,24 @@ - + false left - + - + 0.83em + @@ -289,7 +300,7 @@ - + @@ -377,7 +388,7 @@ - + @@ -389,6 +400,7 @@ + @@ -396,6 +408,7 @@ + diff --git a/docs/EN_US/ECLLanguageReference/ECLR-includer.xml b/docs/EN_US/ECLLanguageReference/ECLR-includer.xml index 165906a37ee..8a444946335 100644 --- a/docs/EN_US/ECLLanguageReference/ECLR-includer.xml +++ b/docs/EN_US/ECLLanguageReference/ECLR-includer.xml @@ -7,7 +7,7 @@ - + @@ -38,25 +38,25 @@ the user, as available; otherwise, the CC BY-ND 3.0 Unported (https://creativecommons.org/licenses/by-nd/3.0/). - + + xmlns:xi="http://www.w3.org/2001/XInclude"/> - © HPCC - Systems®. All rights reserved. - Except where otherwise noted, ECL Language Reference content licensed - under Creative Commons public license. + © HPCC + Systems®. All rights reserved. Except where + otherwise noted, ECL Language Reference content licensed under Creative + Commons public license. HPCC Systems® - + @@ -66,11 +66,11 @@ + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> @@ -78,43 +78,43 @@ + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> @@ -122,31 +122,31 @@ + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> @@ -162,67 +162,67 @@ + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> @@ -231,27 +231,27 @@ + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> @@ -259,11 +259,11 @@ + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> @@ -271,19 +271,19 @@ + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> @@ -291,55 +291,55 @@ + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> @@ -347,35 +347,35 @@ + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> @@ -384,595 +384,599 @@ + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + + + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> @@ -980,59 +984,59 @@ + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> @@ -1040,105 +1044,107 @@ + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + + xmlns:xi="http://www.w3.org/2001/XInclude"/> + + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> @@ -1146,16 +1152,16 @@ + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> + xmlns:xi="http://www.w3.org/2001/XInclude"/> - + diff --git a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-GETSECRET.xml b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-GETSECRET.xml new file mode 100644 index 00000000000..9b867c67599 --- /dev/null +++ b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-GETSECRET.xml @@ -0,0 +1,86 @@ + + + + GETSECRET + + GETSECRET + GETSECRET + + GETSECRET function + ( secretname, + valuename ) + + + + + + + + + + secretname + + A string constant containing the name of the + secret. + + + + valuename + + A string constant containing the name of the key within the + secretname for which you want the + value. + + + + Return: + + GETSECRET returns a STRING value. + + + + + + The GETSECRET function retrieves a + Kubernetes or Vault secret. + + ECL code can only access secrets under the eclUser category. Other categories are intended for + system use or use only by internal ECL functions. + + A secret is a collection of key value pairs. The first argument is the + name of the secret and that secret can contain multiple key value pairs. The + second argument is the actual key within that secret for which you want to + retrieve the value. + + Example: + + // This example assumes a secret named k8s-example has been created on your K8s deployment +// and it contains a key named crypt.key + +IMPORT STD; + +STRING pubKey := '-----BEGIN PUBLIC KEY-----' + '\n' + + 'MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCWnKkGM0l3Y6pKhxMq87hAGBL6' + '\n' + + 'FfEo2HC6XCSQuaAMLkdf7Yjn3FpvFIEO6A1ZYJy70cT8+HOFta+sSUyMn2fDc5cv' + '\n' + + 'VdX8v7XCycYXEBeZ4KsTCHHPCUoO/nxNbxhNz09T8dx/JsIH50LHipR6FTLTSCXR' + '\n' + + 'N9KVLaPXs5DdQx6PjQIDAQAB' + '\n' + + '-----END PUBLIC KEY-----' + '\n'; + +//-------------- +//K8S Example +//-------------- + +DATA k8sData := x'5C62E1843162330ED7BDAB7F37E50F892A669B54B8A466ED421F14954AA'+ + '0505BA9EADAC4DA1D1FB1FD53EBDCF729D1049F893B3EE53ECCE48813A5'+ + '46CF58EBBB26EF5B9247002F7A8D1F90C3C372544501A126CEFC4B385BF'+ + '540931FC0224D4736E4E1E4CF0C67D035063900887C240C8C4F365F0186'+ + 'D0515E98B23C75E482A'; + +VARSTRING k8sKey := (VARSTRING) GETSECRET('k8s-example', 'crypt.key'); +k8sEncModule := Std.Crypto.PKEncryptionFromBuffer('RSA', pubKey, k8sKey); +OUTPUT( (STRING)k8sEncModule.Decrypt(k8sData), NAMED('k8s_message')); + + + diff --git a/ecl/hqlcpp/hqlres.cpp b/ecl/hqlcpp/hqlres.cpp index 5c61222ca20..c2410dd2513 100644 --- a/ecl/hqlcpp/hqlres.cpp +++ b/ecl/hqlcpp/hqlres.cpp @@ -613,22 +613,28 @@ bool ResourceManager::flush(StringBuffer &filename, const char *basename, bool f unsigned id = s.id; VStringBuffer binfile("%s_%s_%u.bin", filename.str(), type, id); VStringBuffer label("%s_%u_txt_start", type, id); + + //Note on targets where the @ character is the start of a comment (eg ARM) then another character is used instead. For example the ARM port uses the % character. +#if defined(__arm__) || defined(__aarch64__) + const char typePrefix = '%'; +#else + const char typePrefix = '@'; +#endif + if (compilerType == ClangCppCompiler) { #ifdef __APPLE__ if (id <= 1200) // There is a limit of 255 sections before linker complains - and some are used elsewhere -#endif fprintf(f, " .section __TEXT,%s_%u\n", type, id); +#else + fprintf(f, " .section .text.%s_%u,\"a\",%cprogbits\n", type, id, typePrefix); +#endif fprintf(f, " .global _%s\n", label.str()); // For some reason apple needs a leading underbar and linux does not fprintf(f, "_%s:\n", label.str()); } else { -#if defined(__linux__) && defined(__GNUC__) && defined(__arm__) - fprintf(f, " .section .note.GNU-stack,\"\",%%progbits\n"); // Prevent the stack from being marked as executable -#else - fprintf(f, " .section .note.GNU-stack,\"\",@progbits\n"); // Prevent the stack from being marked as executable -#endif + fprintf(f, " .section .note.GNU-stack,\"\",%cprogbits\n", typePrefix); // Prevent the stack from being marked as executable fprintf(f, " .section %s_%u,\"a\"\n", type, id); fprintf(f, " .global %s\n", label.str()); fprintf(f, " .type %s,STT_OBJECT\n", label.str()); diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp index fe6f5725fae..bb099c58e22 100644 --- a/ecl/hthor/hthor.cpp +++ b/ecl/hthor/hthor.cpp @@ -358,6 +358,8 @@ ClusterWriteHandler *createClusterWriteHandler(IAgentContext &agent, IHThorIndex StringBuffer defaultCluster; if (isIndex) getDefaultIndexBuildStoragePlane(defaultCluster); + else if (dwHelper && (TDWpersist & dwHelper->getFlags())) + getDefaultPersistPlane(defaultCluster); else getDefaultStoragePlane(defaultCluster); Owned clusterHandler; @@ -1280,7 +1282,7 @@ void CHThorIndexWriteActivity::execute() reccount++; } builder->finish(metadata, &fileCrc, maxRecordSizeSeen); - duplicateKeyCount = builder->getStatistic(StNumDuplicateKeyCount); + duplicateKeyCount = builder->getStatistic(StNumDuplicateKeys); cummulativeDuplicateKeyCount += duplicateKeyCount; numLeafNodes = builder->getStatistic(StNumLeafCacheAdds); numBranchNodes = builder->getStatistic(StNumNodeCacheAdds); diff --git a/esp/bindings/http/platform/httpprot.hpp b/esp/bindings/http/platform/httpprot.hpp index 84a48d7ad5c..f9d7a9efa8e 100644 --- a/esp/bindings/http/platform/httpprot.hpp +++ b/esp/bindings/http/platform/httpprot.hpp @@ -43,9 +43,9 @@ class CPooledHttpThread : public CInterface, implements IPooledThread, implement Owned m_socket; CEspApplicationPort* m_apport = nullptr; StringAttr m_context; - int m_MaxRequestEntityLength; - bool m_is_ssl; - ISecureSocketContext* m_ssctx; + int m_MaxRequestEntityLength = DEFAULT_MAX_REQUEST_ENTITY_LENGTH; + bool m_is_ssl = false; + ISecureSocketContext* m_ssctx = nullptr; IPersistentHandler* m_persistentHandler = nullptr; bool m_shouldClose = false; IHttpServerService* m_httpserver = nullptr; @@ -86,7 +86,7 @@ class CHttpThread: public CEspProtocolThread CEspApplicationPort* m_apport; bool m_viewConfig; StringAttr m_context; - int m_MaxRequestEntityLength; + int m_MaxRequestEntityLength = DEFAULT_MAX_REQUEST_ENTITY_LENGTH; bool m_is_ssl; ISecureSocketContext* m_ssctx; IPersistentHandler* m_persistentHandler = nullptr; diff --git a/esp/scm/ws_workunits.ecm b/esp/scm/ws_workunits.ecm index 3a9b731ed57..138a9d98a90 100644 --- a/esp/scm/ws_workunits.ecm +++ b/esp/scm/ws_workunits.ecm @@ -25,7 +25,7 @@ EspInclude(ws_workunits_queryset_req_resp); ESPservice [ auth_feature("DEFERRED"), //This declares that the method logic handles feature level authorization - version("2.00"), default_client_version("2.00"), cache_group("ESPWsWUs"), + version("2.01"), default_client_version("2.01"), cache_group("ESPWsWUs"), noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits { ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/workunits.xslt")] WUQuery(WUQueryRequest, WUQueryResponse); @@ -109,7 +109,7 @@ ESPservice [ ESPmethod [cache_seconds(60), min_ver("1.57")] WUListArchiveFiles(WUListArchiveFilesRequest, WUListArchiveFilesResponse); ESPmethod [cache_seconds(60), min_ver("1.57")] WUGetArchiveFile(WUGetArchiveFileRequest, WUGetArchiveFileResponse); ESPmethod [cache_seconds(60), min_ver("1.61")] WUGetNumFileToCopy(WUGetNumFileToCopyRequest, WUGetNumFileToCopyResponse); - ESPmethod [min_ver("1.71")] WUDetails(WUDetailsRequest, WUDetailsResponse); + ESPmethod [min_ver("2.01")] WUDetails(WUDetailsRequest, WUDetailsResponse); ESPmethod [cache_seconds(600), min_ver("1.71")] WUDetailsMeta(WUDetailsMetaRequest, WUDetailsMetaResponse); ESPmethod [clear_cache_group, min_ver("1.72")] WUEclDefinitionAction(WUEclDefinitionActionRequest, WUEclDefinitionActionResponse); ESPmethod [cache_seconds(60), min_ver("1.74")] WUGetThorJobQueue(WUGetThorJobQueueRequest, WUGetThorJobQueueResponse); diff --git a/esp/scm/ws_workunits_struct.ecm b/esp/scm/ws_workunits_struct.ecm index 21883a9e4de..eb0d4c1abf1 100644 --- a/esp/scm/ws_workunits_struct.ecm +++ b/esp/scm/ws_workunits_struct.ecm @@ -836,6 +836,7 @@ ESPstruct [nil_remove] WUResponseNote nonNegativeInteger ErrorCode; string Severity; nonNegativeInteger Cost; + [min_ver("2.01")] uint64 Id; // unique identifier for note }; ESPstruct [nil_remove] WUResponseScope diff --git a/esp/services/ws_workunits/ws_wudetails.cpp b/esp/services/ws_workunits/ws_wudetails.cpp index bd295fbab37..b38c0aa66ad 100644 --- a/esp/services/ws_workunits/ws_wudetails.cpp +++ b/esp/services/ws_workunits/ws_wudetails.cpp @@ -22,6 +22,9 @@ #include "workunit.hpp" #include "jset.hpp" #include "jstatcodes.h" +#include "sysinfologger.hpp" + +#define COMPONENT_NOTE_MASK 0x80000000 typedef std::pair AttribValuePair; bool operator==(const AttribValuePair & p1, const AttribValuePair & p2) @@ -487,8 +490,16 @@ void WUDetails::processRequest(IEspWUDetailsRequest &req, IEspWUDetailsResponse { Owned componentConfig = getComponentConfig(); IArrayOf espWuResponseNotes; - + // Global messages + // Note: there are 2 types of 'notes': + // 1) static component warnings (usually from Helm templates) + // 2) dynamic global messages generated from executing components + // Static component messages will have COMPONENT_NOTE_MASK. Owned iter = componentConfig->getElements("warnings"); + unsigned __int64 entry_num=1; + CDateTime dt; + dt.setNow(); + timestamp_type ts = dt.getTimeStamp(); ForEach(*iter) { IPropertyTree & cur = iter->query(); @@ -502,6 +513,27 @@ void WUDetails::processRequest(IEspWUDetailsRequest &req, IEspWUDetailsResponse espWuResponseNote->setErrorCode_null(); espWuResponseNote->setSeverity(cur.queryProp("@severity")); espWuResponseNote->setCost(0); + VStringBuffer idstr("%" I64F "u", makeMessageId(ts, entry_num, true)); + espWuResponseNote->setId(idstr.str()); + espWuResponseNotes.append(*espWuResponseNote.getClear()); + entry_num++; + } + Owned msgFilter = createSysInfoLoggerMsgFilter(); + msgFilter->setVisibleOnly(); + Owned msgIter = createSysInfoLoggerMsgIterator(msgFilter); + ForEach(*msgIter) + { + ISysInfoLoggerMsg & sysInfoMsg = msgIter->query(); + Owned espWuResponseNote = createWUResponseNote("",""); + StringBuffer tmpbuf; + encodeXML(sysInfoMsg.queryMsg(), tmpbuf, ENCODE_NEWLINES, strlen(sysInfoMsg.queryMsg()), true); + espWuResponseNote->setSource(sysInfoMsg.querySource()); + espWuResponseNote->setMessage(tmpbuf.str()); + espWuResponseNote->setErrorCode(sysInfoMsg.queryLogMsgCode()); + espWuResponseNote->setSeverity(LogMsgClassToVarString(sysInfoMsg.queryClass())); + espWuResponseNote->setCost(0); + VStringBuffer idstr("%" I64F "u", sysInfoMsg.queryLogMsgId()); + espWuResponseNote->setId(idstr.str()); espWuResponseNotes.append(*espWuResponseNote.getClear()); } Owned respScope = createWUResponseScope("",""); diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index 62805eafa92..2e77d711efa 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -16,7 +16,7 @@ "@fluentui/react-icons-mdl2": "1.3.82", "@fluentui/react-migration-v8-v9": "9.6.48", "@hpcc-js/chart": "2.86.0", - "@hpcc-js/codemirror": "2.64.0", + "@hpcc-js/codemirror": "2.65.0", "@hpcc-js/common": "2.73.0", "@hpcc-js/comms": "2.99.0", "@hpcc-js/dataflow": "8.1.7", @@ -2121,10 +2121,9 @@ } }, "node_modules/@hpcc-js/codemirror": { - "version": "2.64.0", - "resolved": "https://registry.npmjs.org/@hpcc-js/codemirror/-/codemirror-2.64.0.tgz", - "integrity": "sha512-3BTYDnPiL75W2tSZONEU3XaPvZ7GT8gSHpS8sO3CM3njm0XXM6qsLsQsai0Mij0zRVcK0Zgj2jbqdwGMWLOGQQ==", - "license": "Apache-2.0", + "version": "2.65.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/codemirror/-/codemirror-2.65.0.tgz", + "integrity": "sha512-hEzD61aV1ZI/9AtR61y4fqn/nz/7CXZ5eH0omvTB34yg6J7b94oeUbbLfzLOKenxNt5OIeLnDnbnF8JHIhOZtg==", "dependencies": { "@hpcc-js/common": "^2.73.0" } diff --git a/esp/src/package.json b/esp/src/package.json index ca1ec13d739..ea8f02aea3a 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -45,7 +45,7 @@ "@fluentui/react-icons-mdl2": "1.3.82", "@fluentui/react-migration-v8-v9": "9.6.48", "@hpcc-js/chart": "2.86.0", - "@hpcc-js/codemirror": "2.64.0", + "@hpcc-js/codemirror": "2.65.0", "@hpcc-js/common": "2.73.0", "@hpcc-js/comms": "2.99.0", "@hpcc-js/dataflow": "8.1.7", diff --git a/esp/src/src-react/components/Helpers.tsx b/esp/src/src-react/components/Helpers.tsx index 972a7453a4c..cc9e39fd61d 100644 --- a/esp/src/src-react/components/Helpers.tsx +++ b/esp/src/src-react/components/Helpers.tsx @@ -1,23 +1,17 @@ import * as React from "react"; -import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link } from "@fluentui/react"; +import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluentui/react"; +import { TreeItemValue } from "@fluentui/react-components"; import * as ESPRequest from "src/ESPRequest"; import nlsHPCC from "src/nlsHPCC"; import { HelperRow, useWorkunitHelpers } from "../hooks/workunit"; import { HolyGrail } from "../layouts/HolyGrail"; -import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; +import { DockPanel, DockPanelItem, ResetableDockPanel } from "../layouts/DockPanel"; +import { pushUrl } from "../util/history"; import { ShortVerticalDivider } from "./Common"; -import { SearchParams } from "../util/hashUrl"; -import { hashHistory } from "../util/history"; +import { FlatItem, HelpersTree } from "./HelpersTree"; +import { FetchEditor } from "./SourceEditor"; -function canShowContent(type: string) { - switch (type) { - case "dll": - return false; - } - return true; -} - -function getURL(item: HelperRow, option) { +function getURL(item: HelperRow, option?) { let params = ""; const uriEncodedParams: { [key: string]: any } = { @@ -82,165 +76,198 @@ function getURL(item: HelperRow, option) { return ESPRequest.getBaseURL() + params + (option ? `&Option=${encodeURIComponent(option)}` : "&Option=1"); } -function getTarget(id, row: HelperRow) { - if (canShowContent(row.Type)) { - let sourceMode = "text"; - switch (row.Type) { - case "ECL": - sourceMode = "ecl"; - break; - case "Workunit XML": - case "Archive Query": - case "xml": - sourceMode = "xml"; - break; - } - return { - sourceMode, - url: getURL(row, id) - }; - } - return null; -} - -const defaultUIState = { - hasSelection: false, - canShowContent: false -}; - interface HelpersProps { wuid: string; + mode?: "ecl" | "xml" | "text" | "yaml"; + url?: string; + parentUrl?: string; } export const Helpers: React.FunctionComponent = ({ - wuid + wuid, + mode, + url, + parentUrl = `/workunits/${wuid}/helpers` }) => { - const [uiState, setUIState] = React.useState({ ...defaultUIState }); + const [fullscreen, setFullscreen] = React.useState(false); + const [dockpanel, setDockpanel] = React.useState(); const [helpers, refreshData] = useWorkunitHelpers(wuid); - const [data, setData] = React.useState([]); - const { - selection, setSelection, - setTotal, - refreshTable } = useFluentStoreState({}); - - // Grid --- - const columns = React.useMemo((): FluentColumns => { - return { - sel: { - width: 27, - selectorType: "checkbox" - }, - Type: { - label: nlsHPCC.Type, - width: 160, - formatter: (Type, row) => { - const target = getTarget(row.id, row); - if (target) { - const searchParams = new SearchParams(hashHistory.location.search); - searchParams.param("mode", encodeURIComponent(target.sourceMode)); - searchParams.param("src", encodeURIComponent(target.url)); - const linkText = Type.replace("Slave", "Worker") + (row?.Description ? " (" + row.Description + ")" : ""); - return {linkText}; + const [noDataMsg, setNoDataMsg] = React.useState(""); + const [checkedRows, setCheckedRows] = React.useState([]); + const [checkedItems, setCheckedItems] = React.useState([]); + const [selection, setSelection] = React.useState(); + const [treeItems, setTreeItems] = React.useState([]); + const [openItems, setOpenItems] = React.useState>([]); + + React.useEffect(() => { + helpers.forEach(helper => { + helper.Orig = { + url: getURL(helper), + ...helper.Orig + }; + }); + }, [helpers]); + + React.useEffect(() => { + setSelection(helpers.filter(item => url === getURL(item))[0]); + }, [helpers, url]); + + React.useEffect(() => { + if (helpers.length && selection !== undefined) { + setNoDataMsg(selection?.Type === "dll" ? nlsHPCC.CannotDisplayBinaryData : ""); + } + }, [helpers, selection]); + + React.useEffect(() => { + const rows = []; + checkedItems.forEach(value => { + const filtered = helpers.filter(row => row.id === value || row?.Name?.indexOf(value) > -1)[0]; + if (treeItems.filter(item => item.value === value)[0].url && filtered) { + rows.push(filtered); + } + }); + setCheckedRows(rows); + }, [checkedItems, helpers, treeItems]); + + React.useEffect(() => { + const flatTreeItems: FlatItem[] = []; + helpers.forEach(helper => { + let parentValue; + const helperPath = helper.Path?.replace("/var/lib/HPCCSystems/", ""); + const fileName = helper.Name?.split("/").pop(); + if (helperPath) { + const pathDirs = helperPath.split("/"); + let parentFolder; + let folderName; + for (let i = 0; i < pathDirs.length; i++) { + folderName = parentFolder ? parentFolder + "/" + pathDirs[i] : pathDirs[i]; + if (flatTreeItems.filter(item => item.value === folderName).length === 0) { + flatTreeItems.push({ + value: folderName, + content: pathDirs[i], + parentValue: parentFolder, + url: "" + }); + } + if (!parentFolder) { + parentFolder = pathDirs[i]; + } else { + parentFolder += "/" + pathDirs[i]; } - return Type; } - }, - Description: { - label: nlsHPCC.Description - }, - FileSize: { - label: nlsHPCC.FileSize, - width: 90, - justify: "right" + flatTreeItems.push({ + value: helperPath + "/" + fileName, + content: fileName, + fileSize: helper.FileSize, + parentValue: parentFolder, + url: helper.Orig.url + }); + } else { + flatTreeItems.push({ + value: helper.id, + parentValue: parentValue ?? undefined, + content: helper.Description ?? helper.Name ?? helper.Type, + fileSize: helper.FileSize, + url: helper.Orig.url + }); } - }; - }, []); + }); + setTreeItems(flatTreeItems.sort((a, b) => { + if (a.parentValue === undefined && b.parentValue === undefined) return 0; + if (a.parentValue === undefined) return -1; + if (b.parentValue === undefined) return 1; + return a.parentValue?.toString().localeCompare(b.parentValue?.toString(), undefined, { ignorePunctuation: false }); + })); + setOpenItems(flatTreeItems.map(item => item.value)); + }, [helpers]); + + const treeItemLeafNodes = React.useMemo(() => { + return treeItems.filter(item => item.url !== ""); + }, [treeItems]); // Command Bar --- const buttons = React.useMemo((): ICommandBarItemProps[] => [ { key: "refresh", text: nlsHPCC.Refresh, iconProps: { iconName: "Refresh" }, - onClick: () => refreshData() + onClick: () => { + refreshData(); + pushUrl(`${parentUrl}`); + } }, { key: "divider_1", itemType: ContextualMenuItemType.Divider, onRender: () => }, { - key: "open", text: nlsHPCC.Open, disabled: !uiState.canShowContent, iconProps: { iconName: "WindowEdit" }, + key: "selectAll", text: checkedItems.length === treeItemLeafNodes.length ? nlsHPCC.DeselectAll : nlsHPCC.SelectAll, iconProps: { iconName: checkedItems.length === treeItemLeafNodes.length ? "Checkbox" : "CheckboxComposite" }, onClick: () => { - if (selection.length === 1) { - const target = getTarget(selection[0].id, selection[0]); - if (target) { - window.location.href = `#/text?mode=${target.sourceMode}&src=${encodeURIComponent(target.url)}`; - } + if (checkedItems.length < treeItemLeafNodes.length) { + setCheckedItems(treeItemLeafNodes.map(i => i.value)); } else { - for (let i = 0; i < selection.length; ++i) { - const target = getTarget(selection[i].id, selection[i]); - if (target) { - window.open(`#/text?mode=${target.sourceMode}&src=${encodeURIComponent(target.url)}`, "_blank"); - } - } + setCheckedItems([]); } } }, - { key: "divider_2", itemType: ContextualMenuItemType.Divider, onRender: () => }, { - key: "file", text: nlsHPCC.File, disabled: !uiState.hasSelection, iconProps: { iconName: "Download" }, + key: "file", text: nlsHPCC.File, disabled: checkedRows.filter(item => item.url !== "").length === 0, iconProps: { iconName: "Download" }, onClick: () => { - selection.forEach(item => { + checkedRows.forEach(item => { window.open(getURL(item, 1)); }); } }, { - key: "zip", text: nlsHPCC.Zip, disabled: !uiState.hasSelection, iconProps: { iconName: "Download" }, + key: "zip", text: nlsHPCC.Zip, disabled: checkedRows.filter(item => item.url !== "").length === 0, iconProps: { iconName: "Download" }, onClick: () => { - selection.forEach(item => { + checkedRows.forEach(item => { window.open(getURL(item, 2)); }); } }, { - key: "gzip", text: nlsHPCC.GZip, disabled: !uiState.hasSelection, iconProps: { iconName: "Download" }, + key: "gzip", text: nlsHPCC.GZip, disabled: checkedRows.filter(item => item.url !== "").length === 0, iconProps: { iconName: "Download" }, onClick: () => { - selection.forEach(item => { + checkedRows.forEach(item => { window.open(getURL(item, 3)); }); } } + ], [checkedItems, checkedRows, parentUrl, refreshData, treeItemLeafNodes]); - ], [refreshData, selection, uiState.canShowContent, uiState.hasSelection]); + const rightButtons = React.useMemo((): ICommandBarItemProps[] => [ + { + key: "fullscreen", title: nlsHPCC.MaximizeRestore, iconProps: { iconName: fullscreen ? "ChromeRestore" : "FullScreen" }, + onClick: () => setFullscreen(!fullscreen) + } + ], [fullscreen]); - const copyButtons = useCopyButtons(columns, selection, "helpers"); + const setSelectedItem = React.useCallback((selId: string) => { + pushUrl(`${parentUrl}/${selId}`); + }, [parentUrl]); - // Selection --- React.useEffect(() => { - const state = { ...defaultUIState }; - - selection.forEach(row => { - state.hasSelection = true; - if (canShowContent(row.Type)) { - state.canShowContent = true; + if (dockpanel) { + // Should only happen once on startup --- + const layout: any = dockpanel.layout(); + if (Array.isArray(layout?.main?.sizes) && layout.main.sizes.length === 2) { + layout.main.sizes = [0.3, 0.7]; + dockpanel.layout(layout).lazyRender(); } - }); - setUIState(state); - }, [selection]); - - React.useEffect(() => { - setData(helpers); - }, [helpers]); + } + }, [dockpanel]); return } + header={} main={ - + + + { // Only render after archive is loaded (to ensure it "defaults to open") --- + helpers?.length && + + } + + + + + } />; }; diff --git a/esp/src/src-react/components/HelpersTree.tsx b/esp/src/src-react/components/HelpersTree.tsx new file mode 100644 index 00000000000..ec06b243886 --- /dev/null +++ b/esp/src/src-react/components/HelpersTree.tsx @@ -0,0 +1,76 @@ +import * as React from "react"; +import { FlatTree, useHeadlessFlatTree_unstable, HeadlessFlatTreeItemProps, TreeItem, TreeItemLayout, TreeItemValue, TreeOpenChangeEvent, TreeOpenChangeData } from "@fluentui/react-components"; +import { FolderOpen20Regular, Folder20Regular, FolderOpen20Filled, Folder20Filled, Document20Regular, Document20Filled } from "@fluentui/react-icons"; +import nlsHPCC from "src/nlsHPCC"; +import { convertedSize } from "src/Utility"; + +export type FlatItem = HeadlessFlatTreeItemProps & { fileSize?: number, content: string, url?: string }; + +interface HelpersTreeProps { + checkedItems: string[]; + selectedUrl?: string; + treeItems?: FlatItem[]; + openItems?: Iterable; + setOpenItems?: (openItems: any) => void; + setCheckedItems: (prevChecked: any) => void; + setSelectedItem: (eclId: string) => void; +} + +export const HelpersTree: React.FunctionComponent = ({ + checkedItems, + selectedUrl, + treeItems = [], + openItems = [], + setOpenItems, + setCheckedItems, + setSelectedItem +}) => { + + const handleOpenChange = (evt: TreeOpenChangeEvent, data: TreeOpenChangeData) => { + setOpenItems(data.openItems); + }; + + const flatTree = useHeadlessFlatTree_unstable(treeItems, { openItems, onOpenChange: handleOpenChange, selectionMode: "multiselect" }); + + const onClick = React.useCallback((evt) => { + if (evt.target.nodeName === "INPUT" || evt.currentTarget.dataset.url === "") return; + const contentUrl = `${evt.currentTarget.dataset.name}?src=${encodeURIComponent(evt.currentTarget?.dataset?.url)}`; + setSelectedItem(contentUrl); + }, [setSelectedItem]); + + const checkChange = React.useCallback((evt, item) => { + const isChecked = evt.target.checked; + setCheckedItems((prevChecked) => { + const selectedItem = treeItems.filter(i => item.value === i.value)[0]; + if (isChecked) { + return [...prevChecked, selectedItem.value]; + } else { + return prevChecked.filter(value => value !== item.value); + } + }); + }, [treeItems, setCheckedItems]); + + const { ...treeProps } = flatTree.getTreeProps(); + return
+ + { + Array.from(flatTree.items(), (flatTreeItem, k) => { + const { fileSize, content, url, ...treeItemProps } = flatTreeItem.getTreeItemProps(); + return + : : + selectedUrl === url ? : ) : + selectedUrl === url ? : + } + > + {content}{fileSize ? (` (${convertedSize(fileSize)})`) : ("")} + + ; + }) + } + +
; +}; diff --git a/esp/src/src-react/components/QueryTests.tsx b/esp/src/src-react/components/QueryTests.tsx index 0f8abea68c1..8b0058347a8 100644 --- a/esp/src/src-react/components/QueryTests.tsx +++ b/esp/src/src-react/components/QueryTests.tsx @@ -1,12 +1,10 @@ import * as React from "react"; import { Pivot, PivotItem } from "@fluentui/react"; import { join, scopedLogger } from "@hpcc-js/util"; -import { SizeMe } from "react-sizeme"; import nlsHPCC from "src/nlsHPCC"; import * as ESPQuery from "src/ESPQuery"; import * as WsTopology from "src/WsTopology"; import { useWorkunitResults } from "../hooks/workunit"; -import { pivotItemStyle } from "../layouts/pivot"; import { IFrame } from "./IFrame"; import { pushUrl } from "../util/history"; import { XMLSourceEditor } from "./SourceEditor"; @@ -19,6 +17,8 @@ const buildFrameUrl = (url, suffix) => { return urlParts[0] + "&src=" + encodeURIComponent(join(src, suffix)); }; +const frameStyle = { position: "absolute", zIndex: 0, padding: 4, overflow: "none", inset: "44px 0 4px 0" } as React.CSSProperties; + interface QueryTestsProps { querySet: string; queryId: string; @@ -99,43 +99,41 @@ export const QueryTests: React.FunctionComponent = ({ setResultNames(names); }, [wuResults]); - return {({ size }) => - { - pushUrl(`/queries/${querySet}/${queryId}/testPages/${evt.props.itemKey}`); - }} - > - -