[pull] master from netdata:master

src/collectors/python.d.plugin/python.d.plugin.in

@@ -51,7 +51,6 @@ ENV_NETDATA_PLUGINS_DIR = 'NETDATA_PLUGINS_DIR'
 ENV_NETDATA_USER_PLUGINS_DIRS = 'NETDATA_USER_PLUGINS_DIRS'
 ENV_NETDATA_LIB_DIR = 'NETDATA_LIB_DIR'
 ENV_NETDATA_UPDATE_EVERY = 'NETDATA_UPDATE_EVERY'
-ENV_NETDATA_LOCK_DIR = 'NETDATA_LOCK_DIR'
 
 
 def add_pythond_packages():
@@ -66,7 +65,6 @@ add_pythond_packages()
 from bases.collection import safe_print
 from bases.loggers import PythonDLogger
 from bases.loaders import load_config
-from third_party import filelock
 
 try:
     from collections import OrderedDict
@@ -91,10 +89,7 @@ def dirs():
         ENV_NETDATA_PLUGINS_DIR,
         os.path.dirname(__file__),
     )
-    locks = os.getenv(
-        ENV_NETDATA_LOCK_DIR,
-        os.path.join('@varlibdir_POST@', 'lock')
-    )
+
     modules_user_config = os.path.join(plugin_user_config, 'python.d')
     modules_stock_config = os.path.join(plugin_stock_config, 'python.d')
     modules = os.path.abspath(pluginsd + '/../python.d')
@@ -112,7 +107,6 @@ def dirs():
             'modules',
             'user_modules',
             'var_lib',
-            'locks',
         ]
     )
     return Dirs(
@@ -123,7 +117,6 @@ def dirs():
         modules,
         user_modules,
         var_lib,
-        locks,
     )
 
 
@@ -501,54 +494,15 @@ class PluginConfig(dict):
         return self['default_run']
 
 
-class FileLockRegistry:
-    def __init__(self, path):
-        self.path = path
-        self.locks = dict()
-
-    @staticmethod
-    def rename(name):
-        # go version name is 'docker'
-        if name.startswith("dockerd"):
-            name = "docker" + name[7:]
-        return name
-
-    def register(self, name):
-        name = self.rename(name)
-        if name in self.locks:
-            return
-        file = os.path.join(self.path, '{0}.collector.lock'.format(name))
-        lock = filelock.FileLock(file)
-        lock.acquire(timeout=0)
-        self.locks[name] = lock
-
-    def unregister(self, name):
-        name = self.rename(name)
-        if name not in self.locks:
-            return
-        lock = self.locks[name]
-        lock.release()
-        del self.locks[name]
-
-
-class DummyRegistry:
-    def register(self, name):
-        pass
-
-    def unregister(self, name):
-        pass
-
-
 class Plugin:
     config_name = 'python.d.conf'
     jobs_status_dump_name = 'pythond-jobs-statuses.json'
 
-    def __init__(self, modules_to_run, min_update_every, registry):
+    def __init__(self, modules_to_run, min_update_every):
         self.modules_to_run = modules_to_run
         self.min_update_every = min_update_every
         self.config = PluginConfig(PLUGIN_BASE_CONF)
         self.log = PythonDLogger()
-        self.registry = registry
         self.started_jobs = collections.defaultdict(dict)
         self.jobs = list()
         self.saver = None
@@ -706,30 +660,12 @@ class Plugin:
                 continue
             self.log.info('{0}[{1}] : check success'.format(job.module_name, job.real_name))
 
-            try:
-                self.registry.register(job.full_name())
-            except filelock.Timeout as error:
-                self.log.info('{0}[{1}] : already registered by another process, skipping the job ({2})'.format(
-                    job.module_name, job.real_name, error))
-                job.status = JOB_STATUS_DROPPED
-                continue
-            except Exception as error:
-                self.log.warning('{0}[{1}] : registration failed: {2}, skipping the job'.format(
-                    job.module_name, job.real_name, error))
-                job.status = JOB_STATUS_DROPPED
-                continue
-
             try:
                 job.create()
             except Exception as error:
                 self.log.warning("{0}[{1}] : unhandled exception on create : {2}, skipping the job".format(
                     job.module_name, job.real_name, repr(error)))
                 job.status = JOB_STATUS_DROPPED
-                try:
-                    self.registry.unregister(job.full_name())
-                except Exception as error:
-                    self.log.warning('{0}[{1}] : deregistration failed: {2}'.format(
-                        job.module_name, job.real_name, error))
                 continue
 
             self.started_jobs[job.module_name] = job.actual_name
@@ -803,7 +739,6 @@ def parse_command_line():
 
     debug = False
     trace = False
-    nolock = False
     update_every = 1
     modules_to_run = list()
 
@@ -820,9 +755,6 @@ def parse_command_line():
     if 'trace' in opts:
         trace = True
         opts.remove('trace')
-    if 'nolock' in opts:
-        nolock = True
-        opts.remove('nolock')
     if opts:
         modules_to_run = list(opts)
 
@@ -832,14 +764,12 @@ def parse_command_line():
             'update_every',
             'debug',
             'trace',
-            'nolock',
             'modules_to_run',
         ])
     return cmd(
         update_every,
         debug,
         trace,
-        nolock,
         modules_to_run,
     )
 
@@ -907,11 +837,6 @@ def main():
 
     log.info('using python v{0}'.format(PY_VERSION[0]))
 
-    if DIRS.locks and not cmd.nolock:
-        registry = FileLockRegistry(DIRS.locks)
-    else:
-        registry = DummyRegistry()
-
     unique_avail_module_names = set([m.name for m in AVAILABLE_MODULES])
     unknown = set(cmd.modules_to_run) - unique_avail_module_names
     if unknown:
@@ -924,8 +849,7 @@ def main():
     p = Plugin(
         get_modules_to_run(cmd),
         cmd.update_every,
-        registry,
-    )
+)
 
     # cheap attempt to reduce chance of python.d job running before go.d
     # TODO: better implementation needed

src/daemon/buildinfo.c

@@ -1532,7 +1532,6 @@ static void populate_directories(void) {
     build_info_set_value(BIB_DIR_PLUGINS, netdata_configured_primary_plugins_dir);
     build_info_set_value(BIB_DIR_WEB, netdata_configured_web_dir);
     build_info_set_value(BIB_DIR_LOG, netdata_configured_log_dir);
-    build_info_set_value(BIB_DIR_LOCK, netdata_configured_lock_dir);
     build_info_set_value(BIB_DIR_HOME, netdata_configured_home_dir);
 }
 

src/daemon/common.h

@@ -84,7 +84,6 @@ extern const char *netdata_configured_primary_plugins_dir;
 extern const char *netdata_configured_web_dir;
 extern const char *netdata_configured_cache_dir;
 extern const char *netdata_configured_varlib_dir;
-extern const char *netdata_configured_lock_dir;
 extern const char *netdata_configured_cloud_dir;
 extern const char *netdata_configured_home_dir;
 extern const char *netdata_configured_host_prefix;

src/daemon/config/netdata-conf-directories.c

@@ -24,7 +24,6 @@ void netdata_conf_section_directories(void) {
     netdata_configured_cache_dir        = inicfg_get(&netdata_config, CONFIG_SECTION_DIRECTORIES, "cache",        netdata_configured_cache_dir);
     netdata_configured_varlib_dir       = inicfg_get(&netdata_config, CONFIG_SECTION_DIRECTORIES, "lib",          netdata_configured_varlib_dir);
 
-    netdata_configured_lock_dir = get_varlib_subdir_from_config(netdata_configured_varlib_dir, "lock");
     netdata_configured_cloud_dir = get_varlib_subdir_from_config(netdata_configured_varlib_dir, "cloud.d");
 
     pluginsd_initialize_plugin_directories();

src/daemon/daemon.c

@@ -77,15 +77,12 @@ static void clean_directory(const char *dirname)
 static void prepare_required_directories(uid_t uid, gid_t gid) {
     change_dir_ownership(netdata_configured_cache_dir, uid, gid, true);
     change_dir_ownership(netdata_configured_varlib_dir, uid, gid, false);
-    change_dir_ownership(netdata_configured_lock_dir, uid, gid, false);
     change_dir_ownership(netdata_configured_log_dir, uid, gid, false);
     change_dir_ownership(netdata_configured_cloud_dir, uid, gid, false);
 
     char filename[FILENAME_MAX + 1];
     snprintfz(filename, FILENAME_MAX, "%s/registry", netdata_configured_varlib_dir);
     change_dir_ownership(filename, uid, gid, false);
-
-    clean_directory(netdata_configured_lock_dir);
 }
 
 static int become_user(const char *username, int pid_fd) {

src/daemon/environment.c

@@ -49,7 +49,6 @@ void set_environment_for_plugins_and_scripts(void) {
     nd_setenv("NETDATA_WEB_DIR", verify_required_directory(netdata_configured_web_dir), 1);
     nd_setenv("NETDATA_CACHE_DIR", verify_or_create_required_directory(netdata_configured_cache_dir), 1);
     nd_setenv("NETDATA_LIB_DIR", verify_or_create_required_directory(netdata_configured_varlib_dir), 1);
-    nd_setenv("NETDATA_LOCK_DIR", verify_or_create_required_directory(netdata_configured_lock_dir), 1);
     nd_setenv("NETDATA_LOG_DIR", verify_or_create_required_directory(netdata_configured_log_dir), 1);
     nd_setenv("NETDATA_HOST_PREFIX", netdata_configured_host_prefix, 1);
 

src/daemon/h2o-common.c

@@ -10,7 +10,6 @@ const char *netdata_configured_primary_plugins_dir = PLUGINS_DIR;
 const char *netdata_configured_web_dir             = WEB_DIR;
 const char *netdata_configured_cache_dir           = CACHE_DIR;
 const char *netdata_configured_varlib_dir          = VARLIB_DIR;
-const char *netdata_configured_lock_dir            = VARLIB_DIR "/lock";
 const char *netdata_configured_cloud_dir           = VARLIB_DIR "/cloud.d";
 const char *netdata_configured_home_dir            = VARLIB_DIR;
 const char *netdata_configured_host_prefix         = NULL;

src/go/cmd/godplugin/config.go

@@ -19,7 +19,6 @@ type envConfig struct {
 	userDir    string
 	stockDir   string
 	varLibDir  string
-	lockDir    string
 	watchPath  string
 	logLevel   string
 }
@@ -30,15 +29,13 @@ func newEnvConfig() *envConfig {
 		userDir:    os.Getenv("NETDATA_USER_CONFIG_DIR"),
 		stockDir:   os.Getenv("NETDATA_STOCK_CONFIG_DIR"),
 		varLibDir:  os.Getenv("NETDATA_LIB_DIR"),
-		lockDir:    os.Getenv("NETDATA_LOCK_DIR"),
 		watchPath:  os.Getenv("NETDATA_PLUGINS_GOD_WATCH_PATH"),
 		logLevel:   os.Getenv("NETDATA_LOG_LEVEL"),
 	}
 
 	cfg.userDir = cfg.handleDirOnWin(cfg.userDir)
 	cfg.stockDir = cfg.handleDirOnWin(cfg.stockDir)
 	cfg.varLibDir = cfg.handleDirOnWin(cfg.varLibDir)
-	cfg.lockDir = cfg.handleDirOnWin(cfg.lockDir)
 	cfg.watchPath = cfg.handleDirOnWin(cfg.watchPath)
 
 	return cfg
@@ -66,7 +63,6 @@ type config struct {
 	collectorsWatchPath []string
 	serviceDiscoveryDir multipath.MultiPath
 	stateFile           string
-	lockDir             string
 }
 
 func newConfig(opts *cli.Option, env *envConfig) *config {
@@ -79,7 +75,6 @@ func newConfig(opts *cli.Option, env *envConfig) *config {
 	cfg.collectorsWatchPath = cfg.initCollectorsWatchPaths(opts, env)
 	cfg.serviceDiscoveryDir = cfg.initServiceDiscoveryConfigDir()
 	cfg.stateFile = cfg.initStateFile(env)
-	cfg.lockDir = env.lockDir
 
 	return cfg
 }

src/go/cmd/godplugin/main.go

@@ -54,7 +54,6 @@ func main() {
 		ServiceDiscoveryConfigDir: cfg.serviceDiscoveryDir,
 		CollectorsConfigWatchPath: cfg.collectorsWatchPath,
 		StateFile:                 cfg.stateFile,
-		LockDir:                   cfg.lockDir,
 		RunModule:                 opts.Module,
 		MinUpdateEvery:            opts.UpdateEvery,
 	})

src/go/plugin/go.d/agent/agent.go

@@ -18,7 +18,6 @@ import (
 	"github.com/netdata/netdata/go/plugins/pkg/safewriter"
 	"github.com/netdata/netdata/go/plugins/plugin/go.d/agent/confgroup"
 	"github.com/netdata/netdata/go/plugins/plugin/go.d/agent/discovery"
-	"github.com/netdata/netdata/go/plugins/plugin/go.d/agent/filelock"
 	"github.com/netdata/netdata/go/plugins/plugin/go.d/agent/filestatus"
 	"github.com/netdata/netdata/go/plugins/plugin/go.d/agent/functions"
 	"github.com/netdata/netdata/go/plugins/plugin/go.d/agent/jobmgr"
@@ -37,7 +36,6 @@ type Config struct {
 	CollectorsConfigWatchPath []string
 	ServiceDiscoveryConfigDir []string
 	StateFile                 string
-	LockDir                   string
 	ModuleRegistry            module.Registry
 	RunModule                 string
 	MinUpdateEvery            int
@@ -55,7 +53,6 @@ type Agent struct {
 	ServiceDiscoveryConfigDir multipath.MultiPath
 
 	StateFile string
-	LockDir   string
 
 	RunModule      string
 	MinUpdateEvery int
@@ -80,7 +77,6 @@ func New(cfg Config) *Agent {
 		ServiceDiscoveryConfigDir: cfg.ServiceDiscoveryConfigDir,
 		CollectorsConfigWatchPath: cfg.CollectorsConfigWatchPath,
 		StateFile:                 cfg.StateFile,
-		LockDir:                   cfg.LockDir,
 		RunModule:                 cfg.RunModule,
 		MinUpdateEvery:            cfg.MinUpdateEvery,
 		ModuleRegistry:            module.DefaultRegistry,
@@ -205,10 +201,6 @@ func (a *Agent) run(ctx context.Context) {
 		jobMgr.Vnodes = reg
 	}
 
-	if a.LockDir != "" {
-		jobMgr.FileLock = filelock.New(a.LockDir)
-	}
-
 	var fsMgr *filestatus.Manager
 	if !isTerminal && a.StateFile != "" {
 		fsMgr = filestatus.NewManager(a.StateFile)

src/go/plugin/go.d/agent/jobmgr/di.go

@@ -9,12 +9,6 @@ import (
 	"github.com/netdata/netdata/go/plugins/plugin/go.d/agent/vnodes"
 )
 
-type FileLocker interface {
-	Lock(name string) (bool, error)
-	Unlock(name string)
-	UnlockAll()
-}
-
 type FileStatus interface {
 	Save(cfg confgroup.Config, state string)
 	Remove(cfg confgroup.Config)

src/go/plugin/go.d/agent/jobmgr/dyncfg.go

@@ -75,3 +75,13 @@ func unmarshalPayload(dst any, fn functions.Function) error {
 	}
 	return yaml.Unmarshal(fn.Payload, dst)
 }
+
+func getFnSourceValue(fn functions.Function, key string) string {
+	prefix := key + "="
+	for _, part := range strings.Split(fn.Source, ",") {
+		if v, ok := strings.CutPrefix(part, prefix); ok {
+			return strings.TrimSpace(v)
+		}
+	}
+	return ""
+}