From 411f817d644a244cfda073373cb147341f6a9252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=BE=D0=BB=D0=BE=D0=B6=D0=B0=D0=B5=D0=B2=20=D0=94?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=81=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B0?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= <22160647@sigma.sbrf.ru> Date: Fri, 18 Oct 2024 08:31:12 +1000 Subject: [PATCH 01/14] IGNITE-23459 Add console input if password argument presented without value for ./control.sh --- .../internal/commandline/ArgumentParser.java | 161 +++++++------- .../internal/commandline/CommandHandler.java | 42 +--- .../ConnectionAndSslParameters.java | 16 +- .../argument/parser/CLIArgument.java | 199 ++++++++++++++---- .../argument/parser/CLIArgumentParser.java | 86 ++++++-- .../indexreader/IgniteIndexReader.java | 41 ++-- .../CommandHandlerParsingTest.java | 6 +- ...GridCommandHandlerSslWithSecurityTest.java | 123 +++++++++++ .../ignite/util/CacheMetricsCommandTest.java | 2 +- .../apache/ignite/util/CdcCommandTest.java | 4 +- .../GridCommandHandlerClusterByClassTest.java | 9 +- ...GridCommandHandlerFactoryAbstractTest.java | 2 +- .../ignite/util/GridCommandHandlerTest.java | 2 +- .../internal/management/api/Argument.java | 6 + 14 files changed, 490 insertions(+), 209 deletions(-) diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ArgumentParser.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ArgumentParser.java index 5392ae5db51c6..c897892e874c6 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ArgumentParser.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ArgumentParser.java @@ -15,16 +15,15 @@ * limitations under the License. */ - package org.apache.ignite.internal.commandline; import java.lang.reflect.Field; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Deque; -import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.ListIterator; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.BiFunction; @@ -42,15 +41,17 @@ import org.apache.ignite.internal.management.api.CommandUtils; import org.apache.ignite.internal.management.api.CommandsRegistry; import org.apache.ignite.internal.management.api.Positional; +import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteExperimental; -import org.apache.ignite.ssl.SslContextFactory; import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND; import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_HOST; import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_PORT; import static org.apache.ignite.internal.commandline.CommandHandler.UTILITY_NAME; -import static org.apache.ignite.internal.commandline.argument.parser.CLIArgument.optionalArg; +import static org.apache.ignite.internal.commandline.argument.parser.CLIArgument.CLIArgumentBuilder.argument; +import static org.apache.ignite.internal.commandline.argument.parser.CLIArgument.CLIArgumentBuilder.optionalArgument; +import static org.apache.ignite.internal.commandline.argument.parser.CLIArgumentParser.readNextValueToken; import static org.apache.ignite.internal.management.api.CommandUtils.CMD_WORDS_DELIM; import static org.apache.ignite.internal.management.api.CommandUtils.NAME_PREFIX; import static org.apache.ignite.internal.management.api.CommandUtils.PARAM_WORDS_DELIM; @@ -62,7 +63,9 @@ import static org.apache.ignite.internal.management.api.CommandUtils.toFormattedCommandName; import static org.apache.ignite.internal.management.api.CommandUtils.toFormattedFieldName; import static org.apache.ignite.internal.management.api.CommandUtils.visitCommandParams; +import static org.apache.ignite.ssl.SslContextFactory.DFLT_KEY_ALGORITHM; import static org.apache.ignite.ssl.SslContextFactory.DFLT_SSL_PROTOCOL; +import static org.apache.ignite.ssl.SslContextFactory.DFLT_STORE_TYPE; /** * Argument parser. @@ -131,9 +134,6 @@ public class ArgumentParser { /** */ static final String CMD_SSL_FACTORY = "--ssl-factory"; - /** Set of sensitive arguments */ - private static final Set SENSITIVE_ARGUMENTS = new HashSet<>(); - /** */ private static final BiConsumer PORT_VALIDATOR = (name, val) -> { if (val <= 0 || val > 65535) @@ -143,64 +143,46 @@ public class ArgumentParser { /** */ private final List> common = new ArrayList<>(); - static { - SENSITIVE_ARGUMENTS.add(CMD_PASSWORD); - SENSITIVE_ARGUMENTS.add(CMD_KEYSTORE_PASSWORD); - SENSITIVE_ARGUMENTS.add(CMD_TRUSTSTORE_PASSWORD); - } - - /** - * @param arg To check. - * @return True if provided argument is among sensitive one and not should be displayed. - */ - public static boolean isSensitiveArgument(String arg) { - return SENSITIVE_ARGUMENTS.contains(arg); - } + /** Console instance */ + protected final GridConsole console; /** * @param log Logger. * @param registry Supported commands. + * @param console Supported commands. */ - public ArgumentParser(IgniteLogger log, IgniteCommandRegistry registry) { + public ArgumentParser(IgniteLogger log, IgniteCommandRegistry registry, GridConsole console) { this.log = log; this.registry = registry; - - BiConsumer securityWarn = (name, val) -> log.info(String.format("Warning: %s is insecure. " + - "Whenever possible, use interactive prompt for password (just discard %s option).", name, name)); - - arg(CMD_HOST, "HOST_OR_IP", String.class, DFLT_HOST); - arg(CMD_PORT, "PORT", Integer.class, DFLT_PORT, PORT_VALIDATOR); - arg(CMD_USER, "USER", String.class, null); - arg(CMD_PASSWORD, "PASSWORD", String.class, null, (BiConsumer)securityWarn); - arg(CMD_VERBOSE, CMD_VERBOSE, boolean.class, false); - arg(CMD_SSL_PROTOCOL, "SSL_PROTOCOL[, SSL_PROTOCOL_2, ..., SSL_PROTOCOL_N]", String[].class, new String[] {DFLT_SSL_PROTOCOL}); - arg(CMD_SSL_CIPHER_SUITES, "SSL_CIPHER_1[, SSL_CIPHER_2, ..., SSL_CIPHER_N]", String[].class, null); - arg(CMD_SSL_KEY_ALGORITHM, "SSL_KEY_ALGORITHM", String.class, SslContextFactory.DFLT_KEY_ALGORITHM); - arg(CMD_SSL_FACTORY, "SSL_FACTORY_PATH", String.class, null); - arg(CMD_KEYSTORE_TYPE, "KEYSTORE_TYPE", String.class, SslContextFactory.DFLT_STORE_TYPE); - arg(CMD_KEYSTORE, "KEYSTORE_PATH", String.class, null); - arg(CMD_KEYSTORE_PASSWORD, "KEYSTORE_PASSWORD", char[].class, null, (BiConsumer)securityWarn); - arg(CMD_TRUSTSTORE_TYPE, "TRUSTSTORE_TYPE", String.class, SslContextFactory.DFLT_STORE_TYPE); - arg(CMD_TRUSTSTORE, "TRUSTSTORE_PATH", String.class, null); - arg(CMD_TRUSTSTORE_PASSWORD, "TRUSTSTORE_PASSWORD", char[].class, null, (BiConsumer)securityWarn); - arg(CMD_AUTO_CONFIRMATION, CMD_AUTO_CONFIRMATION, boolean.class, false); - arg( - CMD_ENABLE_EXPERIMENTAL, - CMD_ENABLE_EXPERIMENTAL, Boolean.class, - IgniteSystemProperties.getBoolean(IGNITE_ENABLE_EXPERIMENTAL_COMMAND) + this.console = console; + + common.addAll(List.of( + optionalArgument(CMD_HOST, String.class).withUsage("HOST_OR_IP").withDefault(DFLT_HOST).build(), + optionalArgument(CMD_PORT, Integer.class).withUsage("PORT").withDefault(DFLT_PORT).withValidator(PORT_VALIDATOR).build(), + optionalArgument(CMD_USER, String.class).withUsage("USER").build(), + optionalArgument(CMD_PASSWORD, String.class).withUsage("PASSWORD").markSensitive().build(), + optionalArgument(CMD_VERBOSE, boolean.class).withUsage(CMD_VERBOSE).withDefault(false).build(), + optionalArgument(CMD_SSL_PROTOCOL, String[].class) + .withUsage("SSL_PROTOCOL[, SSL_PROTOCOL_2, ..., SSL_PROTOCOL_N]") + .withDefault(t -> new String[] {DFLT_SSL_PROTOCOL}) + .build(), + optionalArgument(CMD_SSL_CIPHER_SUITES, String[].class).withUsage("SSL_CIPHER_1[, SSL_CIPHER_2, ..., SSL_CIPHER_N]").build(), + optionalArgument(CMD_SSL_KEY_ALGORITHM, String.class).withUsage("SSL_KEY_ALGORITHM").withDefault(DFLT_KEY_ALGORITHM).build(), + optionalArgument(CMD_SSL_FACTORY, String.class).withUsage("SSL_FACTORY_PATH").build(), + optionalArgument(CMD_KEYSTORE_TYPE, String.class).withUsage("KEYSTORE_TYPE").withDefault(DFLT_STORE_TYPE).build(), + optionalArgument(CMD_KEYSTORE, String.class).withUsage("KEYSTORE_PATH").build(), + optionalArgument(CMD_KEYSTORE_PASSWORD, char[].class).withUsage("KEYSTORE_PASSWORD").markSensitive().build(), + optionalArgument(CMD_TRUSTSTORE_TYPE, String.class).withUsage("TRUSTSTORE_TYPE").withDefault(DFLT_STORE_TYPE).build(), + optionalArgument(CMD_TRUSTSTORE, String.class).withUsage("TRUSTSTORE_PATH").build(), + optionalArgument(CMD_TRUSTSTORE_PASSWORD, char[].class).withUsage("TRUSTSTORE_PASSWORD").markSensitive().build(), + optionalArgument(CMD_AUTO_CONFIRMATION, boolean.class).withUsage(CMD_AUTO_CONFIRMATION).withDefault(false).build(), + optionalArgument(CMD_ENABLE_EXPERIMENTAL, Boolean.class) + .withUsage(CMD_ENABLE_EXPERIMENTAL) + .withDefault(t -> IgniteSystemProperties.getBoolean(IGNITE_ENABLE_EXPERIMENTAL_COMMAND)) + .build()) ); } - /** */ - private void arg(String name, String usage, Class type, T dflt, BiConsumer validator) { - common.add(optionalArg(name, usage, type, t -> dflt, validator)); - } - - /** */ - private void arg(String name, String usage, Class type, T dflt) { - common.add(optionalArg(name, usage, type, () -> dflt)); - } - /** * Creates list of common utility options. * @@ -236,7 +218,9 @@ public ConnectionAndSslParameters parseA CLIArgumentParser parser = createArgumentParser(); - parser.parse(args.iterator()); + parser.parse(args.listIterator()); + + String argsToStr = convertCommandToString(raw.listIterator(), parser); A arg = (A)argument( cmdPath.peek().argClass(), @@ -253,7 +237,7 @@ public ConnectionAndSslParameters parseA throw new IllegalArgumentException("Experimental commands disabled"); } - return new ConnectionAndSslParameters<>(cmdPath, arg, parser); + return new ConnectionAndSslParameters<>(cmdPath, arg, parser, argsToStr); } /** @@ -323,14 +307,11 @@ private CLIArgumentParser createArgumentParser() { List> positionalArgs = new ArrayList<>(); List> namedArgs = new ArrayList<>(); - BiFunction> toArg = (fld, optional) -> new CLIArgument<>( - toFormattedFieldName(fld).toLowerCase(), - null, - optional, - fld.getType(), - null, - (name, val) -> {} - ); + BiFunction> toArg = + (fld, optional) -> argument(toFormattedFieldName(fld).toLowerCase(), fld.getType()) + .withOptional(optional) + .withSensitive(fld.getAnnotation(Argument.class).sensitive()) + .build(); List> grpdFlds = CommandUtils.argumentGroupsValues(cmdPath.peek().argClass()); @@ -339,14 +320,10 @@ private CLIArgumentParser createArgumentParser() { || fld.getAnnotation(Argument.class).optional()) ); - Consumer positionalArgCb = fld -> positionalArgs.add(new CLIArgument<>( - fld.getName().toLowerCase(), - null, - fld.getAnnotation(Argument.class).optional(), - fld.getType(), - null, - (name, val) -> {} - )); + Consumer positionalArgCb = fld -> positionalArgs.add(argument(fld.getName().toLowerCase(), fld.getType()) + .withOptional(fld.getAnnotation(Argument.class).optional()) + .build() + ); BiConsumer> argGrpCb = (argGrp0, flds) -> flds.forEach(fld -> { if (fld.isAnnotationPresent(Positional.class)) @@ -359,6 +336,42 @@ private CLIArgumentParser createArgumentParser() { namedArgs.addAll(common); - return new CLIArgumentParser(positionalArgs, namedArgs); + return new CLIArgumentParser(positionalArgs, namedArgs, console); + } + + /** + * Create string of command arguments for logging arguments with hidden confidential values + * @param rawIter raw command arguments iterator + * @param parser CLIArgumentParser + * + * @return string of command arguments for logging with hidden confidential values + */ + private String convertCommandToString(ListIterator rawIter, CLIArgumentParser parser) { + SB cmdToStr = new SB(); + + while (rawIter.hasNext()) { + String arg = rawIter.next(); + + CLIArgument cliArg = parser.getCliArg(arg.toLowerCase()); + + cmdToStr.a(arg).a(' '); + + if (cliArg == null || cliArg.isFlag()) + continue; + + String argVal = readNextValueToken(rawIter); + + if (argVal != null) { + if (cliArg.isSensitive()) { + cmdToStr.a("***** "); + log.info(String.format("Warning: %s is insecure. Whenever possible, use interactive " + + "prompt for password (just omit the argument value).", cliArg.name())); + } + else + cmdToStr.a(argVal).a(' '); + } + } + + return cmdToStr.toString(); } } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index 4c733407f6254..b30e5c8b88ce1 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -60,7 +60,6 @@ import org.apache.ignite.internal.util.spring.IgniteSpringHelperImpl; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.X; -import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteExperimental; import org.apache.ignite.ssl.SslContextFactory; @@ -247,7 +246,7 @@ public int execute(List rawArgs) { verbose = F.exist(rawArgs, CMD_VERBOSE::equalsIgnoreCase); - ConnectionAndSslParameters args = new ArgumentParser(logger, registry).parseAndValidate(rawArgs); + ConnectionAndSslParameters args = new ArgumentParser(logger, registry, console).parseAndValidate(rawArgs); cmdName = toFormattedCommandName(args.cmdPath().peekLast().getClass()).toUpperCase(); @@ -272,7 +271,7 @@ public int execute(List rawArgs) { } logger.info("Command [" + cmdName + "] started"); - logger.info("Arguments: " + argumentsToString(rawArgs)); + logger.info("Arguments: " + args.getArgumentsToString()); logger.info(U.DELIM); String deprecationMsg = args.command().deprecationMessage(args.commandArg()); @@ -445,36 +444,6 @@ private boolean isConnectionClosedSilentlyException(Throwable e) { return e instanceof ClientConnectionException && e.getMessage().startsWith("Channel is closed"); } - /** - * Joins user's arguments and hides sensitive information. - * - * @param rawArgs Arguments which user has provided. - * @return String which could be shown in console and pritned to log. - */ - private String argumentsToString(List rawArgs) { - boolean hide = false; - - SB sb = new SB(); - - for (int i = 0; i < rawArgs.size(); i++) { - if (hide) { - sb.a("***** "); - - hide = false; - - continue; - } - - String arg = rawArgs.get(i); - - sb.a(arg).a(' '); - - hide = ArgumentParser.isSensitiveArgument(arg); - } - - return sb.toString(); - } - /** * @param args Common arguments. * @return Thin client configuration to connect to cluster. @@ -651,7 +620,8 @@ private void printHelp(List rawArgs) { "The command has the following syntax:"); logger.info(""); - logger.info(INDENT + join(" ", join(" ", UTILITY_NAME, join(" ", new ArgumentParser(logger, registry).getCommonOptions())), + logger.info(INDENT + join(" ", + join(" ", UTILITY_NAME, join(" ", new ArgumentParser(logger, registry, null).getCommonOptions())), asOptional("command", true), "")); logger.info(""); logger.info(""); @@ -717,8 +687,8 @@ private void printCacheHelpHeader(IgniteLogger logger) { logger.info(INDENT + "The '--cache subcommand' is used to get information about and perform actions" + " with caches. The command has the following syntax:"); logger.info(""); - logger.info(INDENT + join(" ", UTILITY_NAME, join(" ", new ArgumentParser(logger, null).getCommonOptions())) + " " + - "--cache [subcommand] "); + logger.info(INDENT + join(" ", UTILITY_NAME, join(" ", new ArgumentParser(logger, null, null).getCommonOptions())) + + " --cache [subcommand] "); logger.info(""); logger.info(INDENT + "The subcommands that take [nodeId] as an argument ('list', 'find_garbage', " + "'contention' and 'validate_indexes') will be executed on the given node or on all server nodes" + diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ConnectionAndSslParameters.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ConnectionAndSslParameters.java index 1d001a4c19ee3..180dd49b15fca 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ConnectionAndSslParameters.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ConnectionAndSslParameters.java @@ -65,6 +65,9 @@ public class ConnectionAndSslParameters { /** */ private final CLIArgumentParser parser; + /**String builder for logging arguments with hidden confidential values*/ + private final String argumentsToString; + /** * @param cmdPath Path to the command in {@link CommandsRegistry} hierarchy. * @param arg Command argument. @@ -73,11 +76,13 @@ public class ConnectionAndSslParameters { public ConnectionAndSslParameters( Deque> cmdPath, A arg, - CLIArgumentParser parser + CLIArgumentParser parser, + String argumentsToString ) { this.cmdPath = cmdPath; this.arg = arg; this.parser = parser; + this.argumentsToString = argumentsToString; this.user = parser.get(CMD_USER); this.pwd = parser.get(CMD_PASSWORD); @@ -249,4 +254,13 @@ public String sslFactoryConfigPath() { public boolean verbose() { return parser.get(CMD_VERBOSE); } + + /** + * Return sting of arguments for logging with hidden confidential values + * + * @return arguments to string + */ + public String getArgumentsToString() { + return argumentsToString; + } } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgument.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgument.java index 94ce189a7a76a..60064256b2a62 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgument.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgument.java @@ -19,27 +19,21 @@ import java.util.function.BiConsumer; import java.util.function.Function; -import java.util.function.Supplier; +import org.apache.ignite.internal.util.typedef.internal.A; /** * Command line argument. * @param Value type. */ public class CLIArgument { - /** */ - private final BiConsumer EMPTY = (name, val) -> {}; - /** */ private final String name; /** */ - private final String usage; - - /** */ - private final boolean isOptional; + private final Class type; /** */ - private final Class type; + private final String usage; /** */ private final Function dfltValSupplier; @@ -48,48 +42,28 @@ public class CLIArgument { private final BiConsumer validator; /** */ - public static CLIArgument optionalArg(String name, String usage, Class type) { - return new CLIArgument<>(name, usage, true, type, null, null); - } + private final boolean isOptional; /** */ - public static CLIArgument optionalArg(String name, String usage, Class type, Supplier dfltValSupplier) { - return new CLIArgument<>(name, usage, true, type, p -> dfltValSupplier.get(), null); - } + private final boolean isSensitive; /** */ - public static CLIArgument optionalArg( + private CLIArgument( String name, - String usage, Class type, - Function dfltValSupplier, - BiConsumer validator - ) { - return new CLIArgument<>(name, usage, true, type, dfltValSupplier, validator); - } - - /** */ - public static CLIArgument mandatoryArg(String name, String usage, Class type) { - return new CLIArgument<>(name, usage, false, type, null, null); - } - - /** */ - public CLIArgument( - String name, String usage, - boolean isOptional, - Class type, Function dfltValSupplier, - BiConsumer validator + BiConsumer validator, + boolean isOptional, + boolean isSensitive ) { this.name = name; - this.usage = usage; - this.isOptional = isOptional; this.type = type; - this.dfltValSupplier = dfltValSupplier == null - ? (type.equals(Boolean.class) ? p -> (T)Boolean.FALSE : p -> null) - : dfltValSupplier; + this.usage = usage; + this.dfltValSupplier = dfltValSupplier; this.validator = validator; + this.isOptional = isOptional; + this.isSensitive = isSensitive; } /** */ @@ -97,28 +71,161 @@ public String name() { return name; } + /** */ + public Class type() { + return type; + } + /** */ public String usage() { return usage; } + /** */ + public Function defaultValueSupplier() { + return dfltValSupplier; + } + + /** */ + public BiConsumer validator() { + return validator; + } + /** */ public boolean optional() { return isOptional; } /** */ - public Class type() { - return type; + public boolean isSensitive() { + return isSensitive; } /** */ - public Function defaultValueSupplier() { - return dfltValSupplier; + public boolean isFlag() { + return type.equals(Boolean.class) || type.equals(boolean.class); } - /** */ - public BiConsumer validator() { - return validator == null ? EMPTY : validator; + /** + * Command line argument builder. + * @param Value type. + */ + public static class CLIArgumentBuilder { + /** */ + private final String name; + + /** */ + private final Class type; + + /** */ + private String usage; + + /** */ + private Function dfltValSupplier; + + /** */ + private BiConsumer validator; + + /** */ + private boolean isOptional; + + /** */ + private boolean isSensitive; + + /** */ + private CLIArgumentBuilder(String name, Class type, boolean isOptional) { + this.name = name; + this.type = type; + this.isOptional = isOptional; + } + + /** */ + public CLIArgumentBuilder withUsage(String usage) { + this.usage = usage; + + return this; + } + + /** */ + public CLIArgumentBuilder withDefault(T dflt) { + dfltValSupplier = t -> dflt; + + return this; + } + + /** */ + public CLIArgumentBuilder withDefault(Function dfltValSupplier) { + this.dfltValSupplier = dfltValSupplier; + + return this; + } + + /** */ + public CLIArgumentBuilder withValidator(BiConsumer validator) { + this.validator = validator; + + return this; + } + + /** */ + public CLIArgumentBuilder markOptional() { + isOptional = true; + + return this; + } + + /** */ + public CLIArgumentBuilder withOptional(boolean optional) { + isOptional = optional; + + return this; + } + + /** */ + public CLIArgumentBuilder markSensitive() { + isSensitive = true; + + return this; + } + + /** */ + public CLIArgumentBuilder withSensitive(boolean sensitive) { + isSensitive = sensitive; + + return this; + } + + /** */ + public static CLIArgumentBuilder argument(String name, Class type) { + return new CLIArgumentBuilder<>(name, type, false); + } + + /** */ + public static CLIArgumentBuilder optionalArgument(String name, Class type) { + return new CLIArgumentBuilder<>(name, type, true); + } + + /** */ + public CLIArgument build() { + A.notNull(name, "name"); + A.notNull(type, "type"); + + boolean isFlag = type.equals(Boolean.class) || type.equals(boolean.class); + + if (isFlag && isSensitive) + throw new IllegalArgumentException("Flag argument can't be sensitive"); + + Function dfltValSupplier = this.dfltValSupplier; + + if (dfltValSupplier == null) + dfltValSupplier = isFlag ? p -> (T)Boolean.FALSE : p -> null; + + BiConsumer validator = this.validator; + + if (validator == null) + validator = (name, val) -> {}; + + return new CLIArgument<>(name, type, usage, dfltValSupplier, validator, isOptional, isSensitive); + } } } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java index ff7f2f84a7929..758af77771f87 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java @@ -19,14 +19,14 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; +import java.util.ListIterator; import java.util.Map; import java.util.Set; import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.commandline.GridConsole; import org.apache.ignite.internal.util.GridStringBuilder; import static java.util.stream.Collectors.toSet; @@ -49,20 +49,21 @@ public class CLIArgumentParser { /** */ private final Map parsedArgs = new HashMap<>(); - /** */ - public CLIArgumentParser(List> argConfiguration) { - this(Collections.emptyList(), argConfiguration); - } + /** Console instance */ + protected final GridConsole console; /** */ public CLIArgumentParser( - List> positionalArgConfig, - List> argConfiguration + List> positionalArgCfg, + List> argConfiguration, + GridConsole console ) { - this.positionalArgCfg = positionalArgConfig; + this.positionalArgCfg = positionalArgCfg; for (CLIArgument cliArg : argConfiguration) this.argConfiguration.put(cliArg.name(), cliArg); + + this.console = console; } /** @@ -71,7 +72,7 @@ public CLIArgumentParser( * * @param argsIter Iterator. */ - public void parse(Iterator argsIter) { + public void parse(ListIterator argsIter) { Set obligatoryArgs = argConfiguration.values().stream().filter(a -> !a.optional()).map(CLIArgument::name).collect(toSet()); @@ -102,18 +103,10 @@ public void parse(Iterator argsIter) { else if (parsedArgs.get(cliArg.name()) != null) throw new IllegalArgumentException(cliArg.name() + " argument specified twice"); - boolean bool = cliArg.type().equals(Boolean.class) || cliArg.type().equals(boolean.class); - - if (!bool && !argsIter.hasNext()) - throw new IllegalArgumentException("Please specify a value for argument: " + arg); - - String strVal = bool ? "true" : argsIter.next(); - - if (strVal != null && strVal.startsWith(NAME_PREFIX)) - throw new IllegalArgumentException("Unexpected value: " + strVal); + String argVal = readArgumentValue(cliArg, argsIter); try { - Object val = parseVal(strVal, cliArg.type()); + Object val = parseVal(argVal, cliArg.type()); ((CLIArgument)cliArg).validator().accept(cliArg.name(), val); @@ -177,6 +170,16 @@ public T get(int position) { return (T)parsedPositionalArgs.get(position); } + /** + * Get CLIArgument. + * + * @param name Argument name. + * @return CLIArgument. + */ + public CLIArgument getCliArg(String name) { + return argConfiguration.get(name); + } + /** * Returns usage description. * @@ -214,4 +217,47 @@ private String argNameForUsage(CLIArgument arg) { else return arg.name(); } + + /** */ + private String readArgumentValue(CLIArgument arg, ListIterator argsIter) { + if (arg.isFlag()) + return "true"; + + String argVal = readNextValueToken(argsIter); + + if (argVal == null) { + if (console != null && arg.isSensitive()) + argVal = new String(requestPasswordFromConsole(arg.name().substring(NAME_PREFIX.length()) + ": ")); + else + throw new IllegalArgumentException("Please specify a value for argument: " + arg.name()); + } + + return argVal; + } + + /** */ + public static String readNextValueToken(ListIterator argsIter) { + if (!argsIter.hasNext()) + return null; + + String val = argsIter.next(); + + if (val.startsWith(NAME_PREFIX)) { + argsIter.previous(); + + return null; + } + + return val; + } + + /** + * Requests password from console with message. + * + * @param msg Message. + * @return Password. + */ + private char[] requestPasswordFromConsole(String msg) { + return console.readPassword(msg); + } } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java index 94af0f28227ac..4ffd28a6c6d2a 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java @@ -65,7 +65,6 @@ import org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; -import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreV2; import org.apache.ignite.internal.processors.cache.persistence.file.FileVersionCheckingFactory; import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO; import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO; @@ -104,8 +103,8 @@ import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE; -import static org.apache.ignite.internal.commandline.argument.parser.CLIArgument.mandatoryArg; -import static org.apache.ignite.internal.commandline.argument.parser.CLIArgument.optionalArg; +import static org.apache.ignite.internal.commandline.argument.parser.CLIArgument.CLIArgumentBuilder.argument; +import static org.apache.ignite.internal.commandline.argument.parser.CLIArgument.CLIArgumentBuilder.optionalArgument; import static org.apache.ignite.internal.management.SystemViewTask.SimpleType.NUMBER; import static org.apache.ignite.internal.management.SystemViewTask.SimpleType.STRING; import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; @@ -118,6 +117,7 @@ import static org.apache.ignite.internal.pagemem.PageIdUtils.partId; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_TEMPLATE; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreV2.VERSION; import static org.apache.ignite.internal.util.GridUnsafe.allocateBuffer; import static org.apache.ignite.internal.util.GridUnsafe.bufferAddress; import static org.apache.ignite.internal.util.GridUnsafe.freeBuffer; @@ -300,20 +300,27 @@ public IgniteIndexReader( public static void main(String[] args) { System.out.println("THIS UTILITY MUST BE LAUNCHED ON PERSISTENT STORE WHICH IS NOT UNDER RUNNING GRID!"); - CLIArgumentParser p = new CLIArgumentParser(asList( - mandatoryArg( - DIR_ARG, - "partition directory, where " + INDEX_FILE_NAME + " and (optionally) partition files are located.", - String.class + CLIArgumentParser p = new CLIArgumentParser( + Collections.emptyList(), + asList( + argument(DIR_ARG, String.class) + .withUsage("partition directory, where " + INDEX_FILE_NAME + " and (optionally) partition files are located.") + .build(), + optionalArgument(PART_CNT_ARG, Integer.class).withUsage("full partitions count in cache group.").withDefault(0).build(), + optionalArgument(PAGE_SIZE_ARG, Integer.class).withUsage("page size.").withDefault(DFLT_PAGE_SIZE).build(), + optionalArgument(PAGE_STORE_VER_ARG, Integer.class).withUsage("page store version.").withDefault(VERSION).build(), + optionalArgument(INDEXES_ARG, String[].class) + .withUsage("you can specify index tree names that will be processed, separated by comma without " + + "spaces, other index trees will be skipped.") + .withDefault(U.EMPTY_STRS) + .build(), + optionalArgument(CHECK_PARTS_ARG, Boolean.class) + .withUsage("check cache data tree in partition files and it's consistency with indexes.") + .withDefault(false) + .build() ), - optionalArg(PART_CNT_ARG, "full partitions count in cache group.", Integer.class, () -> 0), - optionalArg(PAGE_SIZE_ARG, "page size.", Integer.class, () -> DFLT_PAGE_SIZE), - optionalArg(PAGE_STORE_VER_ARG, "page store version.", Integer.class, () -> FilePageStoreV2.VERSION), - optionalArg(INDEXES_ARG, "you can specify index tree names that will be processed, separated by comma " + - "without spaces, other index trees will be skipped.", String[].class, () -> U.EMPTY_STRS), - optionalArg(CHECK_PARTS_ARG, - "check cache data tree in partition files and it's consistency with indexes.", Boolean.class, () -> false) - )); + null + ); if (args.length == 0) { System.out.println(p.usage()); @@ -321,7 +328,7 @@ public static void main(String[] args) { return; } - p.parse(asList(args).iterator()); + p.parse(asList(args).listIterator()); Set idxs = new HashSet<>(asList(p.get(INDEXES_ARG))); diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java index 8a816611c24a1..ada5db815771e 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java @@ -968,7 +968,7 @@ public void testIndexForceRebuildWrongArgs() { "--group-names", "--some-other-arg" )), IllegalArgumentException.class, - "Unexpected value: --some-other-arg" + "Please specify a value for argument: --group-names" ); GridTestUtils.assertThrows( @@ -979,7 +979,7 @@ public void testIndexForceRebuildWrongArgs() { "--cache-names", "--some-other-arg" )), IllegalArgumentException.class, - "Unexpected value: --some-other-arg" + "Please specify a value for argument: --cache-names" ); GridTestUtils.assertThrows( @@ -1339,7 +1339,7 @@ private void checkEnumDescription(Command cmd) { * @return Common parameters container object. */ private ConnectionAndSslParameters parseArgs(List args) { - return new ArgumentParser(setupTestLogger(), new IgniteCommandRegistry()).parseAndValidate(args); + return new ArgumentParser(setupTestLogger(), new IgniteCommandRegistry(), null).parseAndValidate(args); } /** diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/processors/security/GridCommandHandlerSslWithSecurityTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/processors/security/GridCommandHandlerSslWithSecurityTest.java index b3fe7c8e38aea..9275f7227f2e9 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/processors/security/GridCommandHandlerSslWithSecurityTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/processors/security/GridCommandHandlerSslWithSecurityTest.java @@ -200,4 +200,127 @@ public void testConnector() throws Exception { assertContains(log, testOutput, "--keystore-password *****"); assertContains(log, testOutput, "--truststore-password *****"); } + + /** + * Verify that the command work correctly when request starts with the --password argument + * without value that invoke console password input for user, and that it is requested only once. + * + * @throws Exception If failed. + */ + @Test + public void testInputKeyUserPwdOnlyOncePwdArgStart() throws Exception { + performTest(Arrays.asList( + "--password", + "--state", + "--user", login, + "--keystore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "thinClient" : "connectorServer"), + "--keystore-password", keyStorePassword(), + "--truststore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "trusttwo" : "trustthree"), + "--truststore-password", keyStorePassword())); + } + + /** + * Verify that the command work correctly when request contains the --password argument inside + * without value that invoke console password input for user, and that it is requested only once. + * + * @throws Exception If failed. + */ + @Test + public void testInputKeyUserPwdOnlyOncePwdArgMiddle() throws Exception { + performTest(Arrays.asList( + "--state", + "--user", login, + "--password", + "--keystore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "thinClient" : "connectorServer"), + "--keystore-password", keyStorePassword(), + "--truststore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "trusttwo" : "trustthree"), + "--truststore-password", keyStorePassword())); + } + + /** + * Verify that the command work correctly when request ends with the --password argument + * without value that invoke console password input for user, and that it is requested only once. + * + * @throws Exception If failed. + */ + @Test + public void testInputKeyUserPwdOnlyOncePwdArgEnd() throws Exception { + performTest(Arrays.asList( + "--state", + "--user", login, + "--keystore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "thinClient" : "connectorServer"), + "--keystore-password", keyStorePassword(), + "--truststore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "trusttwo" : "trustthree"), + "--truststore-password", keyStorePassword(), + "--password")); + } + + /** + * Perform the test with prepared List arguments + * + * @param args List of query arguments. + * @throws Exception If failed. + */ + private void performTest(List args) throws Exception { + IgniteEx crd = startGrid(); + + crd.cluster().state(ACTIVE); + + TestCommandHandler hnd = newCommandHandler(); + + AtomicInteger pwdCnt = new AtomicInteger(); + + ((CommandHandler)GridTestUtils.getFieldValue(hnd, "hnd")).console = new NoopConsole() { + @Override public char[] readPassword(String fmt, Object... args) { + pwdCnt.incrementAndGet(); + + return pwd.toCharArray(); + } + }; + + int exitCode = hnd.execute(args); + + assertEquals(EXIT_CODE_OK, exitCode); + assertEquals(1, pwdCnt.get()); + } + + /** + * Verify that the command work correctly when request few arguments + * without value that invoke console input. + * + * @throws Exception If failed. + */ + @Test + public void testInputKeyForFewRequestedArguments() throws Exception { + IgniteEx crd = startGrid(); + + crd.cluster().state(ACTIVE); + + TestCommandHandler hnd = newCommandHandler(); + + AtomicInteger reqCnt = new AtomicInteger(); + + ((CommandHandler)GridTestUtils.getFieldValue(hnd, "hnd")).console = new NoopConsole() { + @Override public char[] readPassword(String fmt, Object... args) { + reqCnt.incrementAndGet(); + if (reqCnt.get() == 1) + return keyStorePassword().toCharArray(); + else + return pwd.toCharArray(); + } + }; + + int exitCode = hnd.execute(Arrays.asList( + "--state", + "--user", login, + "--verbose", + "--keystore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "thinClient" : "connectorServer"), + "--keystore-password", + "--truststore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "trusttwo" : "trustthree"), + "--truststore-password", keyStorePassword(), + "--password")); + + assertEquals(EXIT_CODE_OK, exitCode); + assertEquals(2, reqCnt.get()); + } } diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/CacheMetricsCommandTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/CacheMetricsCommandTest.java index 5f36276a957c0..ab5f6581c0004 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/CacheMetricsCommandTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/CacheMetricsCommandTest.java @@ -184,7 +184,7 @@ public void testInvalidArguments() { CACHE_ONE); // Check when after --caches argument extra argument is passed instead of list of caches - checkInvalidArguments("Unexpected value: --all-caches", ENABLE, CACHES_ARGUMENT, ALL_CACHES_ARGUMENT); + checkInvalidArguments("Please specify a value for argument: --caches", ENABLE, CACHES_ARGUMENT, ALL_CACHES_ARGUMENT); } /** diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/CdcCommandTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/CdcCommandTest.java index 90359af8c7fca..c53bfcb869777 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/CdcCommandTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/CdcCommandTest.java @@ -177,7 +177,7 @@ public void testParseDeleteLostSegmentLinks() { assertContains(log, executeCommand(EXIT_CODE_INVALID_ARGUMENTS, CDC, DELETE_LOST_SEGMENT_LINKS, NODE_ID), - "Unexpected value: --yes"); + "Please specify a value for argument: --node-id"); assertContains(log, executeCommand(EXIT_CODE_INVALID_ARGUMENTS, CDC, DELETE_LOST_SEGMENT_LINKS, NODE_ID, "10"), @@ -300,7 +300,7 @@ public void testParseResend() { assertContains(log, executeCommand(EXIT_CODE_INVALID_ARGUMENTS, CDC, RESEND, CACHES), - "Unexpected value: --yes"); + "Please specify a value for argument: --caches"); } /** */ diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java index c6dceefd994c1..80ce1ccc2078a 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java @@ -1362,9 +1362,7 @@ public void testCacheCreate() { assertContains( log, executeCommand(EXIT_CODE_INVALID_ARGUMENTS, "--cache", CREATE, SPRING_XML_CONFIG), - !sslEnabled() - ? "Please specify a value for argument: --springXmlConfig" - : "Unexpected value: --keystore" + "Please specify a value for argument: --springXmlConfig" ); autoConfirmation = true; @@ -1651,10 +1649,7 @@ public void testCacheScan() { assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute("--cache", SCAN, "cache", "--limit")); - if (sslEnabled()) // Extra arguments at the end added. - assertContains(log, testOut.toString(), "Unexpected value: --keystore"); - else - assertContains(log, testOut.toString(), "Please specify a value for argument: --limit"); + assertContains(log, testOut.toString(), "Please specify a value for argument: --limit"); assertEquals(EXIT_CODE_INVALID_ARGUMENTS, execute("--cache", SCAN, "cache", "--limit", "test")); diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerFactoryAbstractTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerFactoryAbstractTest.java index 5c84a7030e971..07a98d279ec24 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerFactoryAbstractTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerFactoryAbstractTest.java @@ -206,7 +206,7 @@ public JmxCommandHandler(@Nullable IgniteLogger log) { String cmdName = null; try { - ArgumentParser parser = new ArgumentParser(log, new IgniteCommandRegistry()); + ArgumentParser parser = new ArgumentParser(log, new IgniteCommandRegistry(), null); ConnectionAndSslParameters p = parser.parseAndValidate(value); diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java index 7cb8d582ba757..dedbfe978d514 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java @@ -3786,7 +3786,7 @@ public void testIncrementalSnapshotRestore() throws Exception { assertContains( log, testOut.toString(), - !sslEnabled() ? "Please specify a value for argument: --increment" : "Unexpected value: " + "Please specify a value for argument: --increment" ); // Wrong params. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/api/Argument.java b/modules/core/src/main/java/org/apache/ignite/internal/management/api/Argument.java index 3ca12071ba199..67e424d9621fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/api/Argument.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/api/Argument.java @@ -52,4 +52,10 @@ * @return {@code True} if parameter name expected without "--" prefix. */ public boolean withoutPrefix() default false; + + /** + * @return {@code True} if the argument value is sensitive and needs value to be requested from console, + * in case this value is not specified, {@code false} if not needed. + */ + public boolean sensitive() default false; } From a8d5ee0deb5eca1aa717c403c192df0f0f99e2c3 Mon Sep 17 00:00:00 2001 From: EAFomin <150327438+EAFomin@users.noreply.github.com> Date: Wed, 29 Jan 2025 12:00:18 +0300 Subject: [PATCH 02/14] IGNITE-23749 Removed redundant permission check to cancel services from control utility (#11815) --- ...SecurityCommandHandlerPermissionsTest.java | 44 +++++++++++++++++++ .../management/kill/CancelServiceTask.java | 10 +++++ 2 files changed, 54 insertions(+) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/SecurityCommandHandlerPermissionsTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/SecurityCommandHandlerPermissionsTest.java index 9587f2a1b2245..f288a4f791420 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/SecurityCommandHandlerPermissionsTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/SecurityCommandHandlerPermissionsTest.java @@ -27,12 +27,15 @@ import org.apache.ignite.Ignite; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.client.thin.ServicesTest; import org.apache.ignite.internal.processors.security.impl.TestSecurityData; import org.apache.ignite.internal.processors.security.impl.TestSecurityPluginProvider; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.plugin.security.SecurityPermission; import org.apache.ignite.plugin.security.SecurityPermissionSet; import org.apache.ignite.plugin.security.SecurityPermissionSetBuilder; +import org.apache.ignite.services.ServiceConfiguration; +import org.apache.ignite.services.ServiceDescriptor; import org.apache.ignite.util.GridCommandHandlerAbstractTest; import org.junit.Test; import org.junit.runners.Parameterized; @@ -47,6 +50,7 @@ import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_DESTROY; import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_READ; import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_REMOVE; +import static org.apache.ignite.plugin.security.SecurityPermission.SERVICE_CANCEL; import static org.apache.ignite.plugin.security.SecurityPermissionSetBuilder.ALL_PERMISSIONS; import static org.apache.ignite.plugin.security.SecurityPermissionSetBuilder.NO_PERMISSIONS; import static org.apache.ignite.plugin.security.SecurityPermissionSetBuilder.systemPermissions; @@ -134,6 +138,38 @@ public void testCacheCreate() throws Exception { ); } + /** */ + @Test + public void testServiceCancel() throws Exception { + String srvcName = "testService"; + Collection cmdArgs = asList("--kill", "service", srvcName); + + Ignite ignite = startGrid( + 0, + userData(TEST_NO_PERMISSIONS_LOGIN, NO_PERMISSIONS), + userData(TEST_LOGIN, servicePermission(srvcName, SERVICE_CANCEL)) + ); + + ServiceConfiguration srvcCfg = new ServiceConfiguration(); + + srvcCfg.setName(srvcName); + srvcCfg.setMaxPerNodeCount(1); + srvcCfg.setTotalCount(1); + srvcCfg.setService(new ServicesTest.TestService()); + + ignite.services().deploy(srvcCfg); + + Collection svcs = ignite.services().serviceDescriptors(); + + assertEquals(EXIT_CODE_UNEXPECTED_ERROR, execute(enrichWithConnectionArguments(cmdArgs, TEST_NO_PERMISSIONS_LOGIN))); + assertEquals(1, svcs.size()); + + assertEquals(EXIT_CODE_OK, execute(enrichWithConnectionArguments(cmdArgs, TEST_LOGIN))); + + svcs = ignite.services().serviceDescriptors(); + assertEquals(0, svcs.size()); + } + /** */ protected IgniteEx startGrid(int idx, TestSecurityData... userData) throws Exception { String login = getTestIgniteInstanceName(idx); @@ -186,6 +222,14 @@ private SecurityPermissionSet cachePermission(SecurityPermission... perms) { .build(); } + /** */ + private SecurityPermissionSet servicePermission(String name, SecurityPermission... perms) { + return SecurityPermissionSetBuilder.create() + .defaultAllowAll(false) + .appendServicePermissions(name, perms) + .build(); + } + /** */ private TestSecurityData userData(String login, SecurityPermissionSet perms) { return new TestSecurityData( diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/kill/CancelServiceTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/kill/CancelServiceTask.java index 2cd4f24717d2b..e68eaa6c27d59 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/kill/CancelServiceTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/kill/CancelServiceTask.java @@ -22,6 +22,9 @@ import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; +import org.apache.ignite.plugin.security.SecurityPermissionSet; + +import static org.apache.ignite.plugin.security.SecurityPermissionSetBuilder.NO_PERMISSIONS; /** * Task for cancel services with specified name. @@ -60,6 +63,13 @@ protected CancelServiceJob(KillServiceCommandArg arg, boolean debug) { return null; } + /** {@inheritDoc} */ + @Override public SecurityPermissionSet requiredPermissions() { + // This task does nothing but delegates the call to the Ignite public API. + // Therefore, it is safe to execute task without any additional permissions check. + return NO_PERMISSIONS; + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(CancelServiceJob.class, this); From dbf1d5c0769c94302822de9271c87bebacd2bd7c Mon Sep 17 00:00:00 2001 From: Maksim Davydov <70368398+maksaska@users.noreply.github.com> Date: Wed, 29 Jan 2025 23:37:02 +0300 Subject: [PATCH 03/14] IGNITE-24093 Remove PARTIAL_COUNTERS_MAP_SINCE (#11820) --- ...dDhtPartitionsStateValidatorBenchmark.java | 12 +-- .../GridCachePartitionExchangeManager.java | 50 ++-------- .../CachePartitionFullCountersMap.java | 36 -------- .../CachePartitionPartialCountersMap.java | 23 ----- .../GridDhtPartitionsExchangeFuture.java | 33 ++----- .../GridDhtPartitionsFullMessage.java | 92 +++---------------- .../GridDhtPartitionsSingleMessage.java | 22 +---- .../IgniteDhtPartitionCountersMap.java | 20 ++-- .../IgniteDhtPartitionCountersMap2.java | 69 -------------- .../GridDhtPartitionsStateValidator.java | 8 +- .../resources/META-INF/classnames.properties | 1 - ...CachePartitionsStateValidatorSelfTest.java | 31 ++++--- 12 files changed, 69 insertions(+), 328 deletions(-) delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap2.java diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java index 6f9f4ff309a2d..ce29a8eaeb320 100644 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java @@ -27,12 +27,12 @@ import org.apache.ignite.internal.benchmarks.jmh.JmhAbstractBenchmark; import org.apache.ignite.internal.benchmarks.jmh.runner.JmhIdeBenchmarkRunner; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionsStateValidator; -import org.apache.ignite.internal.util.typedef.T2; import org.jetbrains.annotations.Nullable; import org.mockito.Matchers; import org.mockito.Mockito; @@ -91,7 +91,7 @@ private GridDhtLocalPartition partitionMock(int id, long updateCounter, long siz * @return Message with specified {@code countersMap} and {@code sizeMap}. */ private GridDhtPartitionsSingleMessage from( - @Nullable Map> countersMap, + @Nullable CachePartitionPartialCountersMap countersMap, @Nullable Map sizesMap ) { GridDhtPartitionsSingleMessage msg = new GridDhtPartitionsSingleMessage(); @@ -117,14 +117,14 @@ public void setup() { List locPartitions = Lists.newArrayList(); - Map> updateCountersMap = new HashMap<>(); + CachePartitionPartialCountersMap cntrMap = new CachePartitionPartialCountersMap(PARTS); Map cacheSizesMap = new HashMap<>(); IntStream.range(0, PARTS).forEach(k -> { locPartitions.add(partitionMock(k, k + 1, k + 1)); long us = k > 20 && k <= 30 ? 0 : k + 2L; - updateCountersMap.put(k, new T2<>(k + 2L, us)); + cntrMap.add(k, k + 2L, us); cacheSizesMap.put(k, us); }); @@ -137,10 +137,10 @@ public void setup() { for (int n = 0; n < NODES; ++n) { UUID remoteNode = UUID.randomUUID(); - msgs.put(remoteNode, from(updateCountersMap, cacheSizesMap)); + msgs.put(remoteNode, from(cntrMap, cacheSizesMap)); } - msgs.put(ignoreNode, from(updateCountersMap, cacheSizesMap)); + msgs.put(ignoreNode, from(cntrMap, cacheSizesMap)); validator = new GridDhtPartitionsStateValidator(cctxMock); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index 78c7b0c317fff..d7a9b56653673 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -85,8 +85,6 @@ import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache; -import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionFullCountersMap; -import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.FinishPreloadingTask; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.ForceRebalanceExchangeTask; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage; @@ -172,7 +170,6 @@ import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT; import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL; import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE; -import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap.PARTIAL_COUNTERS_MAP_SINCE; import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.nextDumpTimeout; import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader.DFLT_PRELOAD_RESEND_TIMEOUT; import static org.apache.ignite.internal.processors.metric.GridMetricManager.CLUSTER_METRICS; @@ -1393,7 +1390,7 @@ private void sendAllPartitions( ) { long time = System.currentTimeMillis(); - GridDhtPartitionsFullMessage m = createPartitionsFullMessage(true, false, null, null, null, null, grps); + GridDhtPartitionsFullMessage m = createPartitionsFullMessage(true, null, null, null, null, grps); m.topologyVersion(msgTopVer); @@ -1449,7 +1446,6 @@ private void sendAllPartitions( * * @param compress {@code True} if possible to compress message (properly work only if prepareMarshall/ * finishUnmarshall methods are called). - * @param newCntrMap {@code True} if possible to use {@link CachePartitionFullCountersMap}. * @param exchId Non-null exchange ID if message is created for exchange. * @param lastVer Last version. * @param partHistSuppliers Partition history suppliers map. @@ -1458,7 +1454,6 @@ private void sendAllPartitions( */ public GridDhtPartitionsFullMessage createPartitionsFullMessage( boolean compress, - boolean newCntrMap, @Nullable final GridDhtPartitionExchangeId exchId, @Nullable GridCacheVersion lastVer, @Nullable IgniteDhtPartitionHistorySuppliersMap partHistSuppliers, @@ -1466,7 +1461,7 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage( ) { Collection grps = cctx.cache().cacheGroups(); - return createPartitionsFullMessage(compress, newCntrMap, exchId, lastVer, partHistSuppliers, partsToReload, grps); + return createPartitionsFullMessage(compress, exchId, lastVer, partHistSuppliers, partsToReload, grps); } /** @@ -1474,7 +1469,6 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage( * * @param compress {@code True} if possible to compress message (properly work only if prepareMarshall/ * finishUnmarshall methods are called). - * @param newCntrMap {@code True} if possible to use {@link CachePartitionFullCountersMap}. * @param exchId Non-null exchange ID if message is created for exchange. * @param lastVer Last version. * @param partHistSuppliers Partition history suppliers map. @@ -1484,7 +1478,6 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage( */ public GridDhtPartitionsFullMessage createPartitionsFullMessage( boolean compress, - boolean newCntrMap, @Nullable final GridDhtPartitionExchangeId exchId, @Nullable GridCacheVersion lastVer, @Nullable IgniteDhtPartitionHistorySuppliersMap partHistSuppliers, @@ -1523,14 +1516,7 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage( partsSizes.put(grp.groupId(), partSizesMap); if (exchId != null) { - CachePartitionFullCountersMap cntrsMap = grp.topology().fullUpdateCounters(); - - if (newCntrMap) - m.addPartitionUpdateCounters(grp.groupId(), cntrsMap); - else { - m.addPartitionUpdateCounters(grp.groupId(), - CachePartitionFullCountersMap.toCountersMap(cntrsMap)); - } + m.addPartitionUpdateCounters(grp.groupId(), grp.topology().fullUpdateCounters()); // Lost partitions can be skipped on node left or activation. m.addLostPartitions(grp.groupId(), grp.topology().lostPartitions()); @@ -1545,12 +1531,7 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage( addFullPartitionsMap(m, dupData, compress, top.groupId(), map, top.similarAffinityKey()); if (exchId != null) { - CachePartitionFullCountersMap cntrsMap = top.fullUpdateCounters(); - - if (newCntrMap) - m.addPartitionUpdateCounters(top.groupId(), cntrsMap); - else - m.addPartitionUpdateCounters(top.groupId(), CachePartitionFullCountersMap.toCountersMap(cntrsMap)); + m.addPartitionUpdateCounters(top.groupId(), top.fullUpdateCounters()); Map partSizesMap = top.globalPartSizes(); @@ -1621,7 +1602,6 @@ private void sendLocalPartitions( createPartitionsSingleMessage(id, cctx.kernalContext().clientNode(), false, - node.version().compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0, null, grps); @@ -1647,19 +1627,17 @@ private void sendLocalPartitions( * @param exchangeId Exchange ID. * @param clientOnlyExchange Client exchange flag. * @param sndCounters {@code True} if need send partition update counters. - * @param newCntrMap {@code True} if possible to use {@link CachePartitionPartialCountersMap}. * @return Message. */ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage( @Nullable GridDhtPartitionExchangeId exchangeId, boolean clientOnlyExchange, boolean sndCounters, - boolean newCntrMap, ExchangeActions exchActions ) { Collection grps = cctx.cache().cacheGroups(); - return createPartitionsSingleMessage(exchangeId, clientOnlyExchange, sndCounters, newCntrMap, exchActions, grps); + return createPartitionsSingleMessage(exchangeId, clientOnlyExchange, sndCounters, exchActions, grps); } /** @@ -1668,7 +1646,6 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage( * @param exchangeId Exchange ID. * @param clientOnlyExchange Client exchange flag. * @param sndCounters {@code True} if need send partition update counters. - * @param newCntrMap {@code True} if possible to use {@link CachePartitionPartialCountersMap}. * @param grps Selected cache groups. * @return Message. */ @@ -1676,7 +1653,6 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage( @Nullable GridDhtPartitionExchangeId exchangeId, boolean clientOnlyExchange, boolean sndCounters, - boolean newCntrMap, ExchangeActions exchActions, Collection grps ) { @@ -1698,12 +1674,8 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage( locMap, grp.affinity().similarAffinityKey()); - if (sndCounters) { - CachePartitionPartialCountersMap cntrsMap = grp.topology().localUpdateCounters(true); - - m.addPartitionUpdateCounters(grp.groupId(), - newCntrMap ? cntrsMap : CachePartitionPartialCountersMap.toCountersMap(cntrsMap)); - } + if (sndCounters) + m.addPartitionUpdateCounters(grp.groupId(), grp.topology().localUpdateCounters(true)); m.addPartitionSizes(grp.groupId(), grp.topology().partitionSizes()); } @@ -1722,12 +1694,8 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage( locMap, top.similarAffinityKey()); - if (sndCounters) { - CachePartitionPartialCountersMap cntrsMap = top.localUpdateCounters(true); - - m.addPartitionUpdateCounters(top.groupId(), - newCntrMap ? cntrsMap : CachePartitionPartialCountersMap.toCountersMap(cntrsMap)); - } + if (sndCounters) + m.addPartitionUpdateCounters(top.groupId(), top.localUpdateCounters(true)); m.addPartitionSizes(top.groupId(), top.partitionSizes()); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java index ebc993c3b8123..1384a558e4769 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java @@ -19,9 +19,6 @@ import java.io.Serializable; import java.util.Arrays; -import java.util.Map; -import org.apache.ignite.internal.util.typedef.T2; -import org.apache.ignite.internal.util.typedef.internal.U; /** * @@ -99,37 +96,4 @@ public void clear() { Arrays.fill(initialUpdCntrs, 0); Arrays.fill(updCntrs, 0); } - - /** - * @param map Full counters map. - * @return Regular java map with counters. - */ - public static Map> toCountersMap(CachePartitionFullCountersMap map) { - int partsCnt = map.updCntrs.length; - - Map> map0 = U.newHashMap(partsCnt); - - for (int p = 0; p < partsCnt; p++) - map0.put(p, new T2<>(map.initialUpdCntrs[p], map.updCntrs[p])); - - return map0; - } - - /** - * @param map Regular java map with counters. - * @param partsCnt Total cache partitions. - * @return Full counters map. - */ - static CachePartitionFullCountersMap fromCountersMap(Map> map, int partsCnt) { - CachePartitionFullCountersMap map0 = new CachePartitionFullCountersMap(partsCnt); - - for (Map.Entry> e : map.entrySet()) { - T2 cntrs = e.getValue(); - - map0.initialUpdCntrs[e.getKey()] = cntrs.get1(); - map0.updCntrs[e.getKey()] = cntrs.get2(); - } - - return map0; - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java index 9a549177cec64..bc9871bb5fbf1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java @@ -21,10 +21,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.Map; -import java.util.TreeMap; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.lang.IgniteProductVersion; /** * @@ -33,9 +31,6 @@ public class CachePartitionPartialCountersMap implements Serializable { /** */ private static final long serialVersionUID = 0L; - /** */ - public static final IgniteProductVersion PARTIAL_COUNTERS_MAP_SINCE = IgniteProductVersion.fromString("2.1.4"); - /** */ public static final CachePartitionPartialCountersMap EMPTY = new CachePartitionPartialCountersMap(); @@ -215,24 +210,6 @@ public static Map> toCountersMap(CachePartitionPartialCo return res; } - /** - * @param map Partition ID to partition counters map. - * @param partsCnt Total cache partitions. - * @return Partial local counters map. - */ - static CachePartitionPartialCountersMap fromCountersMap(Map> map, int partsCnt) { - CachePartitionPartialCountersMap map0 = new CachePartitionPartialCountersMap(partsCnt); - - TreeMap> sorted = new TreeMap<>(map); - - for (Map.Entry> e : sorted.entrySet()) - map0.add(e.getKey(), e.getValue().get1(), e.getValue().get2()); - - map0.trim(); - - return map0; - } - /** {@inheritDoc} */ @Override public String toString() { StringBuilder sb = new StringBuilder("CachePartitionPartialCountersMap {"); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 9a8d610d171c0..896893d437ef9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -127,7 +127,6 @@ import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteInClosure; -import org.apache.ignite.lang.IgniteProductVersion; import org.apache.ignite.lang.IgniteRunnable; import org.jetbrains.annotations.Nullable; @@ -146,7 +145,6 @@ import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL; import static org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents.serverJoinEvent; import static org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents.serverLeftEvent; -import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap.PARTIAL_COUNTERS_MAP_SINCE; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.isSnapshotOperation; import static org.apache.ignite.internal.processors.security.SecurityUtils.remoteSecurityContext; import static org.apache.ignite.internal.util.IgniteUtils.doInParallel; @@ -2131,7 +2129,6 @@ private void sendLocalPartitions(ClusterNode node) throws IgniteCheckedException msg = cctx.exchange().createPartitionsSingleMessage(exchangeId(), false, true, - node.version().compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0, exchActions); Map> partHistReserved0 = partHistReserved; @@ -2174,15 +2171,13 @@ else if (localJoinExchange()) /** * @param compress Message compress flag. - * @param newCntrMap {@code True} if possible to use {@link CachePartitionFullCountersMap}. * @return Message. */ - private GridDhtPartitionsFullMessage createPartitionsMessage(boolean compress, boolean newCntrMap) { + private GridDhtPartitionsFullMessage createPartitionsMessage(boolean compress) { GridCacheVersion last = lastVer.get(); GridDhtPartitionsFullMessage m = cctx.exchange().createPartitionsFullMessage( compress, - newCntrMap, exchangeId(), last != null ? last : cctx.versions().last(), partHistSuppliers, @@ -2962,7 +2957,7 @@ public void forceClientReconnect(ClusterNode node, GridDhtPartitionsSingleMessag onDone(null, reconnectEx); - GridDhtPartitionsFullMessage fullMsg = createPartitionsMessage(true, false); + GridDhtPartitionsFullMessage fullMsg = createPartitionsMessage(true); fullMsg.setErrorsMap(exchangeGlobalExceptions); @@ -3123,8 +3118,7 @@ public void waitAndReplyToNode(final UUID nodeId, final GridDhtPartitionsSingleM return; } - GridDhtPartitionsFullMessage msg = - createPartitionsMessage(true, node.version().compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0); + GridDhtPartitionsFullMessage msg = createPartitionsMessage(true); msg.rebalanced(rebalanced()); @@ -3293,7 +3287,7 @@ private void onAffinityInitialized(IgniteInternalFuture>> assignmentChange = fut.get(); - GridDhtPartitionsFullMessage m = createPartitionsMessage(false, false); + GridDhtPartitionsFullMessage m = createPartitionsMessage(false); CacheAffinityChangeMessage msg = new CacheAffinityChangeMessage(exchId, m, assignmentChange); @@ -3350,8 +3344,7 @@ private List assignPartitionStates(GridDhtPartitionTopology Map> varCntrs = new HashMap<>(); for (Map.Entry e : msgs.entrySet()) { - CachePartitionPartialCountersMap nodeCntrs = e.getValue().partitionUpdateCounters(top.groupId(), - top.partitions()); + CachePartitionPartialCountersMap nodeCntrs = e.getValue().partitionUpdateCounters(top.groupId()); assert nodeCntrs != null; @@ -3903,10 +3896,7 @@ else if (exchCtx.events().hasServerLeft()) cctx.versions().onExchange(lastVer.get().order()); - IgniteProductVersion minVer = exchCtx.events().discoveryCache().minimumNodeVersion(); - - GridDhtPartitionsFullMessage msg = createPartitionsMessage(true, - minVer.compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0); + GridDhtPartitionsFullMessage msg = createPartitionsMessage(true); if (!cctx.affinity().rebalanceRequired() && !deactivateCluster()) msg.rebalanced(true); @@ -4079,7 +4069,7 @@ private void processSingleMessageOnCrdFinish( ? grp.topology() : cctx.exchange().clientTopology(grpId, events().discoveryCache()); - CachePartitionPartialCountersMap cntrs = msg.partitionUpdateCounters(grpId, top.partitions()); + CachePartitionPartialCountersMap cntrs = msg.partitionUpdateCounters(grpId); if (cntrs != null) top.collectUpdateCounters(cntrs); @@ -4512,7 +4502,6 @@ private void processSinglePartitionRequest(ClusterNode node, GridDhtPartitionsSi msg.restoreExchangeId(), cctx.kernalContext().clientNode(), true, - node.version().compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0, exchActions); if (localJoinExchange() && finishState0 == null) @@ -4756,12 +4745,11 @@ private void updatePartitionFullMap(AffinityTopologyVersion resTopVer, GridDhtPa parallelismLvl, cctx.kernalContext().pools().getSystemExecutorService(), msg.partitions().keySet(), grpId -> { + CachePartitionFullCountersMap cntrMap = msg.partitionUpdateCounters(grpId); + CacheGroupContext grp = cctx.cache().cacheGroup(grpId); if (grp != null) { - CachePartitionFullCountersMap cntrMap = msg.partitionUpdateCounters(grpId, - grp.topology().partitions()); - grp.topology().update(resTopVer, msg.partitions().get(grpId), cntrMap, @@ -4774,9 +4762,6 @@ private void updatePartitionFullMap(AffinityTopologyVersion resTopVer, GridDhtPa else { GridDhtPartitionTopology top = cctx.exchange().clientTopology(grpId, events().discoveryCache()); - CachePartitionFullCountersMap cntrMap = msg.partitionUpdateCounters(grpId, - top.partitions()); - top.update(resTopVer, msg.partitions().get(grpId), cntrMap, diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java index 346a6ab717622..8027584bc5e47 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java @@ -45,7 +45,6 @@ import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType; @@ -86,14 +85,6 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa /** Serialized partitions counters. */ private byte[] partCntrsBytes; - /** Partitions update counters. */ - @GridToStringInclude - @GridDirectTransient - private IgniteDhtPartitionCountersMap2 partCntrs2; - - /** Serialized partitions counters. */ - private byte[] partCntrsBytes2; - /** Partitions history suppliers. */ @GridToStringInclude @GridDirectTransient @@ -198,8 +189,6 @@ public GridDhtPartitionsFullMessage(@Nullable GridDhtPartitionExchangeId id, cp.partsBytes = partsBytes; cp.partCntrs = partCntrs; cp.partCntrsBytes = partCntrsBytes; - cp.partCntrs2 = partCntrs2; - cp.partCntrsBytes2 = partCntrsBytes2; cp.partHistSuppliers = partHistSuppliers; cp.partHistSuppliersBytes = partHistSuppliersBytes; cp.partsToReload = partsToReload; @@ -323,24 +312,13 @@ public void addFullPartitionsMap(int grpId, GridDhtPartitionFullMap fullMap, @Nu * @param grpId Cache group ID. * @param cntrMap Partition update counters. */ - public void addPartitionUpdateCounters(int grpId, Map> cntrMap) { + public void addPartitionUpdateCounters(int grpId, CachePartitionFullCountersMap cntrMap) { if (partCntrs == null) partCntrs = new IgniteDhtPartitionCountersMap(); partCntrs.putIfAbsent(grpId, cntrMap); } - /** - * @param grpId Cache group ID. - * @param cntrMap Partition update counters. - */ - public void addPartitionUpdateCounters(int grpId, CachePartitionFullCountersMap cntrMap) { - if (partCntrs2 == null) - partCntrs2 = new IgniteDhtPartitionCountersMap2(); - - partCntrs2.putIfAbsent(grpId, cntrMap); - } - /** * @param grpId Group id. * @param lostParts Lost parts. @@ -373,19 +351,10 @@ public void addLostPartitions(int grpId, Collection lostParts) { /** * @param grpId Cache group ID. - * @param partsCnt Total cache partitions. * @return Partition update counters. */ - public CachePartitionFullCountersMap partitionUpdateCounters(int grpId, int partsCnt) { - if (partCntrs2 != null) - return partCntrs2.get(grpId); - - if (partCntrs == null) - return null; - - Map> map = partCntrs.get(grpId); - - return map != null ? CachePartitionFullCountersMap.fromCountersMap(map, partsCnt) : null; + public CachePartitionFullCountersMap partitionUpdateCounters(int grpId) { + return partCntrs == null ? null : partCntrs.get(grpId); } /** @@ -482,7 +451,6 @@ public void rebalanced(boolean rebalanced) { boolean marshal = (!F.isEmpty(parts) && partsBytes == null) || (partCntrs != null && !partCntrs.empty() && partCntrsBytes == null) || - (partCntrs2 != null && !partCntrs2.empty() && partCntrsBytes2 == null) || (partHistSuppliers != null && partHistSuppliersBytes == null) || (partsToReload != null && partsToReloadBytes == null) || (!F.isEmpty(errs) && errsBytes == null); @@ -499,9 +467,6 @@ public void rebalanced(boolean rebalanced) { if (partCntrs != null && !partCntrs.empty() && partCntrsBytes == null) objectsToMarshall.add(partCntrs); - if (partCntrs2 != null && !partCntrs2.empty() && partCntrsBytes2 == null) - objectsToMarshall.add(partCntrs2); - if (partHistSuppliers != null && partHistSuppliersBytes == null) objectsToMarshall.add(partHistSuppliers); @@ -534,9 +499,6 @@ public void rebalanced(boolean rebalanced) { if (partCntrs != null && !partCntrs.empty() && partCntrsBytes == null) partCntrsBytes = iter.next(); - if (partCntrs2 != null && !partCntrs2.empty() && partCntrsBytes2 == null) - partCntrsBytes2 = iter.next(); - if (partHistSuppliers != null && partHistSuppliersBytes == null) partHistSuppliersBytes = iter.next(); @@ -579,9 +541,6 @@ public void topologyVersion(AffinityTopologyVersion topVer) { if (partCntrsBytes != null && partCntrs == null) objectsToUnmarshall.add(partCntrsBytes); - if (partCntrsBytes2 != null && partCntrs2 == null) - objectsToUnmarshall.add(partCntrsBytes2); - if (partHistSuppliersBytes != null && partHistSuppliers == null) objectsToUnmarshall.add(partHistSuppliersBytes); @@ -640,9 +599,6 @@ public void topologyVersion(AffinityTopologyVersion topVer) { if (partCntrsBytes != null && partCntrs == null) partCntrs = (IgniteDhtPartitionCountersMap)iter.next(); - if (partCntrsBytes2 != null && partCntrs2 == null) - partCntrs2 = (IgniteDhtPartitionCountersMap2)iter.next(); - if (partHistSuppliersBytes != null && partHistSuppliers == null) partHistSuppliers = (IgniteDhtPartitionHistorySuppliersMap)iter.next(); @@ -658,9 +614,6 @@ public void topologyVersion(AffinityTopologyVersion topVer) { if (partCntrs == null) partCntrs = new IgniteDhtPartitionCountersMap(); - if (partCntrs2 == null) - partCntrs2 = new IgniteDhtPartitionCountersMap2(); - if (partHistSuppliers == null) partHistSuppliers = new IgniteDhtPartitionHistorySuppliersMap(); @@ -729,42 +682,36 @@ public void topologyVersion(AffinityTopologyVersion topVer) { writer.incrementState(); case 13: - if (!writer.writeByteArray("partCntrsBytes2", partCntrsBytes2)) - return false; - - writer.incrementState(); - - case 14: if (!writer.writeByteArray("partHistSuppliersBytes", partHistSuppliersBytes)) return false; writer.incrementState(); - case 15: + case 14: if (!writer.writeByteArray("partsBytes", partsBytes)) return false; writer.incrementState(); - case 16: + case 15: if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes)) return false; writer.incrementState(); - case 17: + case 16: if (!writer.writeByteArray("partsToReloadBytes", partsToReloadBytes)) return false; writer.incrementState(); - case 18: + case 17: if (!writer.writeAffinityTopologyVersion("resTopVer", resTopVer)) return false; writer.incrementState(); - case 19: + case 18: if (!writer.writeAffinityTopologyVersion("topVer", topVer)) return false; @@ -843,14 +790,6 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); case 13: - partCntrsBytes2 = reader.readByteArray("partCntrsBytes2"); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); - - case 14: partHistSuppliersBytes = reader.readByteArray("partHistSuppliersBytes"); if (!reader.isLastRead()) @@ -858,7 +797,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 15: + case 14: partsBytes = reader.readByteArray("partsBytes"); if (!reader.isLastRead()) @@ -866,7 +805,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 16: + case 15: partsSizesBytes = reader.readByteArray("partsSizesBytes"); if (!reader.isLastRead()) @@ -874,7 +813,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 17: + case 16: partsToReloadBytes = reader.readByteArray("partsToReloadBytes"); if (!reader.isLastRead()) @@ -882,7 +821,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 18: + case 17: resTopVer = reader.readAffinityTopologyVersion("resTopVer"); if (!reader.isLastRead()) @@ -890,7 +829,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { reader.incrementState(); - case 19: + case 18: topVer = reader.readAffinityTopologyVersion("topVer"); if (!reader.isLastRead()) @@ -910,7 +849,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 20; + return 19; } /** {@inheritDoc} */ @@ -955,9 +894,8 @@ public void merge(GridDhtPartitionsFullMessage other, GridDiscoveryManager disco */ public void cleanUp() { partsBytes = null; - partCntrs2 = null; + partCntrs = null; partCntrsBytes = null; - partCntrsBytes2 = null; partHistSuppliersBytes = null; partsToReloadBytes = null; partsSizesBytes = null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java index 77f8d5f7d5a69..e5d792d739c45 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java @@ -32,7 +32,6 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType; @@ -64,7 +63,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes /** Partitions update counters. */ @GridToStringInclude @GridDirectTransient - private Map partCntrs; + private Map partCntrs; /** Serialized partitions counters. */ private byte[] partCntrsBytes; @@ -201,7 +200,7 @@ public void addLocalPartitionMap(int cacheId, GridDhtPartitionMap locMap, @Nulla * @param grpId Cache group ID. * @param cntrMap Partition update counters. */ - public void addPartitionUpdateCounters(int grpId, Object cntrMap) { + public void addPartitionUpdateCounters(int grpId, CachePartitionPartialCountersMap cntrMap) { if (partCntrs == null) partCntrs = new HashMap<>(); @@ -210,23 +209,12 @@ public void addPartitionUpdateCounters(int grpId, Object cntrMap) { /** * @param grpId Cache group ID. - * @param partsCnt Total cache partitions. * @return Partition update counters. */ - public CachePartitionPartialCountersMap partitionUpdateCounters(int grpId, int partsCnt) { - Object res = partCntrs == null ? null : partCntrs.get(grpId); + public CachePartitionPartialCountersMap partitionUpdateCounters(int grpId) { + CachePartitionPartialCountersMap res = partCntrs == null ? null : partCntrs.get(grpId); - if (res == null) - return CachePartitionPartialCountersMap.EMPTY; - - if (res instanceof CachePartitionPartialCountersMap) - return (CachePartitionPartialCountersMap)res; - - assert res instanceof Map : res; - - Map> map = (Map>)res; - - return CachePartitionPartialCountersMap.fromCountersMap(map, partsCnt); + return res == null ? CachePartitionPartialCountersMap.EMPTY : res; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap.java index 4a22abd9a988c..e7954d960ce44 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap.java @@ -19,11 +19,8 @@ package org.apache.ignite.internal.processors.cache.distributed.dht.preloader; import java.io.Serializable; -import java.util.Collections; import java.util.HashMap; import java.util.Map; -import org.apache.ignite.internal.util.typedef.T2; -import org.apache.ignite.internal.util.typedef.internal.S; /** * Partition counters map. @@ -33,7 +30,7 @@ public class IgniteDhtPartitionCountersMap implements Serializable { private static final long serialVersionUID = 0L; /** */ - private Map>> map; + private Map map; /** * @return {@code True} if map is empty. @@ -46,7 +43,7 @@ public synchronized boolean empty() { * @param cacheId Cache ID. * @param cntrMap Counters map. */ - public synchronized void putIfAbsent(int cacheId, Map> cntrMap) { + public synchronized void putIfAbsent(int cacheId, CachePartitionFullCountersMap cntrMap) { if (map == null) map = new HashMap<>(); @@ -58,20 +55,15 @@ public synchronized void putIfAbsent(int cacheId, Map> c * @param cacheId Cache ID. * @return Counters map. */ - public synchronized Map> get(int cacheId) { + public synchronized CachePartitionFullCountersMap get(int cacheId) { if (map == null) - map = new HashMap<>(); + return null; - Map> cntrMap = map.get(cacheId); + CachePartitionFullCountersMap cntrMap = map.get(cacheId); if (cntrMap == null) - return Collections.emptyMap(); + return null; return cntrMap; } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(IgniteDhtPartitionCountersMap.class, this); - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap2.java deleted file mode 100644 index d1e6d99c4aa20..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionCountersMap2.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.ignite.internal.processors.cache.distributed.dht.preloader; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; - -/** - * Partition counters map. - */ -public class IgniteDhtPartitionCountersMap2 implements Serializable { - /** */ - private static final long serialVersionUID = 0L; - - /** */ - private Map map; - - /** - * @return {@code True} if map is empty. - */ - public synchronized boolean empty() { - return map == null || map.isEmpty(); - } - - /** - * @param cacheId Cache ID. - * @param cntrMap Counters map. - */ - public synchronized void putIfAbsent(int cacheId, CachePartitionFullCountersMap cntrMap) { - if (map == null) - map = new HashMap<>(); - - if (!map.containsKey(cacheId)) - map.put(cacheId, cntrMap); - } - - /** - * @param cacheId Cache ID. - * @return Counters map. - */ - public synchronized CachePartitionFullCountersMap get(int cacheId) { - if (map == null) - return null; - - CachePartitionFullCountersMap cntrMap = map.get(cacheId); - - if (cntrMap == null) - return null; - - return cntrMap; - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java index ed038b247e59e..b0078d678d592 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java @@ -183,8 +183,6 @@ public Map> validatePartitionsUpdateCounters( updateCountersAndNodesByPartitions.put(part.id(), new AbstractMap.SimpleEntry<>(cctx.localNodeId(), part.updateCounter())); } - int partitions = top.partitions(); - // Then process and validate counters from other nodes. for (Map.Entry e : messages.entrySet()) { UUID nodeId = e.getKey(); @@ -193,7 +191,7 @@ public Map> validatePartitionsUpdateCounters( final GridDhtPartitionsSingleMessage msg = e.getValue(); - CachePartitionPartialCountersMap countersMap = msg.partitionUpdateCounters(top.groupId(), partitions); + CachePartitionPartialCountersMap countersMap = msg.partitionUpdateCounters(top.groupId()); Map sizesMap = msg.partitionSizes(top.groupId()); @@ -244,8 +242,6 @@ public Map> validatePartitionsSizes( sizesAndNodesByPartitions.put(part.id(), new AbstractMap.SimpleEntry<>(cctx.localNodeId(), part.fullSize())); } - int partitions = top.partitions(); - // Then process and validate sizes from other nodes. for (Map.Entry e : messages.entrySet()) { UUID nodeId = e.getKey(); @@ -254,7 +250,7 @@ public Map> validatePartitionsSizes( final GridDhtPartitionsSingleMessage msg = e.getValue(); - CachePartitionPartialCountersMap countersMap = msg.partitionUpdateCounters(top.groupId(), partitions); + CachePartitionPartialCountersMap countersMap = msg.partitionUpdateCounters(top.groupId()); Map sizesMap = msg.partitionSizes(top.groupId()); diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index fc27ad5fe8322..6a7559946de58 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -1205,7 +1205,6 @@ org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPre org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloaderAssignments org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionCountersMap -org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionCountersMap2 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionHistorySuppliersMap org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionsToReloadMap org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteHistoricalIterator diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java index ec580fb8ea040..b1b05fe49c1d4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java @@ -24,12 +24,12 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionsStateValidator; -import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.Nullable; import org.junit.Assert; @@ -88,7 +88,10 @@ private GridDhtLocalPartition partitionMock(int id, long updateCounter, long siz * @param sizesMap Sizes map. * @return Message with specified {@code countersMap} and {@code sizeMap}. */ - private GridDhtPartitionsSingleMessage from(@Nullable Map> countersMap, @Nullable Map sizesMap) { + private GridDhtPartitionsSingleMessage from( + @Nullable CachePartitionPartialCountersMap countersMap, + @Nullable Map sizesMap + ) { GridDhtPartitionsSingleMessage msg = new GridDhtPartitionsSingleMessage(); if (countersMap != null) @@ -109,10 +112,10 @@ public void testPartitionCountersValidation() { UUID ignoreNode = UUID.randomUUID(); // For partitions 0 and 2 we have inconsistent update counters. - Map> updateCountersMap = new HashMap<>(); - updateCountersMap.put(0, new T2<>(2L, 2L)); - updateCountersMap.put(1, new T2<>(2L, 2L)); - updateCountersMap.put(2, new T2<>(5L, 5L)); + CachePartitionPartialCountersMap cntrMap = new CachePartitionPartialCountersMap(3); + cntrMap.add(0, 2L, 2L); + cntrMap.add(1, 2L, 2L); + cntrMap.add(2, 5L, 5L); // For partitions 0 and 2 we have inconsistent cache sizes. Map cacheSizesMap = new HashMap<>(); @@ -122,8 +125,8 @@ public void testPartitionCountersValidation() { // Form single messages map. Map msgs = new HashMap<>(); - msgs.put(remoteNode, from(updateCountersMap, cacheSizesMap)); - msgs.put(ignoreNode, from(updateCountersMap, cacheSizesMap)); + msgs.put(remoteNode, from(cntrMap, cacheSizesMap)); + msgs.put(ignoreNode, from(cntrMap, cacheSizesMap)); GridDhtPartitionsStateValidator validator = new GridDhtPartitionsStateValidator(cctxMock); @@ -150,10 +153,10 @@ public void testPartitionCacheSizesValidation() { UUID ignoreNode = UUID.randomUUID(); // For partitions 0 and 2 we have inconsistent update counters. - Map> updateCountersMap = new HashMap<>(); - updateCountersMap.put(0, new T2<>(2L, 2L)); - updateCountersMap.put(1, new T2<>(2L, 2L)); - updateCountersMap.put(2, new T2<>(5L, 5L)); + CachePartitionPartialCountersMap cntrMap = new CachePartitionPartialCountersMap(3); + cntrMap.add(0, 2L, 2L); + cntrMap.add(1, 2L, 2L); + cntrMap.add(2, 5L, 5L); // For partitions 0 and 2 we have inconsistent cache sizes. Map cacheSizesMap = new HashMap<>(); @@ -163,8 +166,8 @@ public void testPartitionCacheSizesValidation() { // Form single messages map. Map msgs = new HashMap<>(); - msgs.put(remoteNode, from(updateCountersMap, cacheSizesMap)); - msgs.put(ignoreNode, from(updateCountersMap, cacheSizesMap)); + msgs.put(remoteNode, from(cntrMap, cacheSizesMap)); + msgs.put(ignoreNode, from(cntrMap, cacheSizesMap)); GridDhtPartitionsStateValidator validator = new GridDhtPartitionsStateValidator(cctxMock); From a5a3785d9a33434c187c728969e8c10af2c32bb2 Mon Sep 17 00:00:00 2001 From: Vladislav Novikov <47698772+vladnovoren@users.noreply.github.com> Date: Wed, 29 Jan 2025 23:38:27 +0300 Subject: [PATCH 04/14] IGNITE-24101 Remove SEPARATE_CACHE_PER_NON_COLLOCATED_SET_SINCE (#11810) --- .../datastructures/DataStructuresProcessor.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java index 00d333ef86f03..cf3614ebc7e1e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java @@ -93,7 +93,6 @@ import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgnitePredicate; -import org.apache.ignite.lang.IgniteProductVersion; import org.apache.ignite.spi.systemview.view.datastructures.AtomicLongView; import org.apache.ignite.spi.systemview.view.datastructures.AtomicReferenceView; import org.apache.ignite.spi.systemview.view.datastructures.AtomicSequenceView; @@ -197,10 +196,6 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen /** */ private static final String SEQUENCES_VIEW_DESC = "Data structure atomic sequences"; - /** Non collocated IgniteSet will use separate cache if all nodes in cluster is not older then specified version. */ - private static final IgniteProductVersion SEPARATE_CACHE_PER_NON_COLLOCATED_SET_SINCE = - IgniteProductVersion.fromString("2.7.0"); - /** Initial capacity. */ private static final int INITIAL_CAPACITY = 10; @@ -1775,14 +1770,12 @@ else if (reentrantLock != null) { final boolean create = cfg != null; final boolean collocated = isCollocated(cfg); - final boolean separated = !collocated && - U.isOldestNodeVersionAtLeast(SEPARATE_CACHE_PER_NON_COLLOCATED_SET_SINCE, ctx.grid().cluster().nodes()); return getCollection(new CX1>() { @Override public IgniteSet applyx(GridCacheContext cctx) throws IgniteCheckedException { - return cctx.dataStructures().set(name, collocated, create, separated); + return cctx.dataStructures().set(name, collocated, create, !collocated); } - }, cfg, name, grpName, SET, create, separated); + }, cfg, name, grpName, SET, create, !collocated); } /** From ab74d038f1dbe2157d9768c4a9ff48986e6d2da2 Mon Sep 17 00:00:00 2001 From: Aleksey Plekhanov Date: Thu, 30 Jan 2025 09:21:14 +0300 Subject: [PATCH 05/14] IGNITE-24323 SQL Calcite: Add query blocking tasks executor (allows to execute SQL inside UDF) - Fixes #11833. Signed-off-by: Aleksey Plekhanov --- .../query/calcite/CalciteQueryProcessor.java | 22 +- .../query/calcite/exec/rel/AbstractNode.java | 9 - .../exec/task/AbstractQueryTaskExecutor.java | 89 +++++ .../calcite/exec/task/QueryAwareTask.java | 26 ++ .../exec/task/QueryBlockingTaskExecutor.java | 111 ++++++ .../query/calcite/exec/task/QueryKey.java | 63 +++ .../calcite/exec/task/QueryTasksQueue.java | 369 ++++++++++++++++++ .../StripedQueryTaskExecutor.java} | 53 +-- .../exec/rel/AbstractExecutionTest.java | 67 +++- .../task/QueryBlockingTaskExecutorTest.java | 118 ++++++ .../exec/task/QueryTasksQueueTest.java | 279 +++++++++++++ .../AbstractBasicIntegrationTest.java | 8 +- ...ryBlockingTaskExecutorIntegrationTest.java | 76 ++++ .../SqlDiagnosticIntegrationTest.java | 4 +- .../UserDefinedFunctionsIntegrationTest.java | 48 +++ .../calcite/planner/AbstractPlannerTest.java | 6 +- .../query/calcite/planner/PlannerTest.java | 4 +- .../testsuites/IgniteCalciteTestSuite.java | 11 +- .../testsuites/IntegrationTestSuite.java | 2 + .../ignite/testsuites/UtilTestSuite.java | 44 +++ 20 files changed, 1309 insertions(+), 100 deletions(-) create mode 100644 modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/AbstractQueryTaskExecutor.java create mode 100644 modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryAwareTask.java create mode 100644 modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryBlockingTaskExecutor.java create mode 100644 modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryKey.java create mode 100644 modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryTasksQueue.java rename modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/{QueryTaskExecutorImpl.java => task/StripedQueryTaskExecutor.java} (61%) create mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryBlockingTaskExecutorTest.java create mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryTasksQueueTest.java create mode 100644 modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/QueryBlockingTaskExecutorIntegrationTest.java create mode 100644 modules/calcite/src/test/java/org/apache/ignite/testsuites/UtilTestSuite.java diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessor.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessor.java index 7e9763e161fe7..83f1c7e7418f2 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessor.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessor.java @@ -79,10 +79,11 @@ import org.apache.ignite.internal.processors.query.calcite.exec.MailboxRegistry; import org.apache.ignite.internal.processors.query.calcite.exec.MailboxRegistryImpl; import org.apache.ignite.internal.processors.query.calcite.exec.QueryTaskExecutor; -import org.apache.ignite.internal.processors.query.calcite.exec.QueryTaskExecutorImpl; import org.apache.ignite.internal.processors.query.calcite.exec.TimeoutService; import org.apache.ignite.internal.processors.query.calcite.exec.TimeoutServiceImpl; import org.apache.ignite.internal.processors.query.calcite.exec.exp.RexExecutorImpl; +import org.apache.ignite.internal.processors.query.calcite.exec.task.QueryBlockingTaskExecutor; +import org.apache.ignite.internal.processors.query.calcite.exec.task.StripedQueryTaskExecutor; import org.apache.ignite.internal.processors.query.calcite.hint.HintsConfig; import org.apache.ignite.internal.processors.query.calcite.message.MessageService; import org.apache.ignite.internal.processors.query.calcite.message.MessageServiceImpl; @@ -124,6 +125,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.jetbrains.annotations.Nullable; +import static org.apache.ignite.IgniteSystemProperties.getBoolean; import static org.apache.ignite.IgniteSystemProperties.getLong; import static org.apache.ignite.events.EventType.EVT_SQL_QUERY_EXECUTION; @@ -141,6 +143,15 @@ public class CalciteQueryProcessor extends GridProcessorAdapter implements Query defaults = "" + DFLT_IGNITE_CALCITE_PLANNER_TIMEOUT) public static final String IGNITE_CALCITE_PLANNER_TIMEOUT = "IGNITE_CALCITE_PLANNER_TIMEOUT"; + /** + * Use query blocking executor property name. + */ + @SystemProperty(value = "Calcite-based SQL engine. Use query blocking task executor instead of striped task " + + "executor. Query blocking executor allows to run SQL queries inside user-defined functions at the cost of " + + "some performance penalty", defaults = "" + false) + public static final String IGNITE_CALCITE_USE_QUERY_BLOCKING_TASK_EXECUTOR = + "IGNITE_CALCITE_USE_QUERY_BLOCKING_TASK_EXECUTOR"; + /** */ public static final FrameworkConfig FRAMEWORK_CONFIG = Frameworks.newConfigBuilder() .executor(new RexExecutorImpl(DataContexts.EMPTY)) @@ -186,8 +197,7 @@ public class CalciteQueryProcessor extends GridProcessorAdapter implements Query private final FrameworkConfig frameworkCfg; /** Query planner timeout. */ - private final long queryPlannerTimeout = getLong(IGNITE_CALCITE_PLANNER_TIMEOUT, - DFLT_IGNITE_CALCITE_PLANNER_TIMEOUT); + private final long qryPlannerTimeout = getLong(IGNITE_CALCITE_PLANNER_TIMEOUT, DFLT_IGNITE_CALCITE_PLANNER_TIMEOUT); /** */ private final QueryPlanCache qryPlanCache; @@ -254,7 +264,9 @@ public CalciteQueryProcessor(GridKernalContext ctx) { qryPlanCache = new QueryPlanCacheImpl(ctx); parserMetrics = new QueryParserMetricsHolder(ctx.metric()); mailboxRegistry = new MailboxRegistryImpl(ctx); - taskExecutor = new QueryTaskExecutorImpl(ctx); + taskExecutor = getBoolean(IGNITE_CALCITE_USE_QUERY_BLOCKING_TASK_EXECUTOR) + ? new QueryBlockingTaskExecutor(ctx) + : new StripedQueryTaskExecutor(ctx); executionSvc = new ExecutionServiceImpl<>(ctx, ArrayRowHandler.INSTANCE); partSvc = new AffinityServiceImpl(ctx); msgSvc = new MessageServiceImpl(ctx); @@ -666,7 +678,7 @@ private T processQuery( exchangeSvc, (q, ex) -> qryReg.unregister(q.id(), ex), log, - queryPlannerTimeout, + qryPlannerTimeout, timeout ); diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/rel/AbstractNode.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/rel/AbstractNode.java index 5450992ac050c..f27548c0eb7c9 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/rel/AbstractNode.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/rel/AbstractNode.java @@ -49,9 +49,6 @@ public abstract class AbstractNode implements Node { /** */ protected static final int IO_BATCH_CNT = IgniteSystemProperties.getInteger(IGNITE_CALCITE_EXEC_IO_BATCH_CNT, 4); - /** for debug purpose */ - private volatile Thread thread; - /** * {@link Inbox} node may not have proper context at creation time in case it * creates on first message received from a remote source. This case the context @@ -186,12 +183,6 @@ protected void checkState() throws Exception { throw new QueryCancelledException("The query was timed out."); if (Thread.interrupted()) throw new IgniteInterruptedCheckedException("Thread was interrupted."); - if (!U.assertionsEnabled()) - return; - if (thread == null) - thread = Thread.currentThread(); - else - assert thread == Thread.currentThread(); } /** */ diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/AbstractQueryTaskExecutor.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/AbstractQueryTaskExecutor.java new file mode 100644 index 0000000000000..ffed07350852c --- /dev/null +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/AbstractQueryTaskExecutor.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.exec.task; + +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.processors.query.calcite.exec.QueryTaskExecutor; +import org.apache.ignite.internal.processors.query.calcite.util.AbstractService; +import org.apache.ignite.internal.processors.security.SecurityContext; +import org.apache.ignite.internal.util.typedef.internal.U; + +/** + * Abstract query task executor. + */ +public abstract class AbstractQueryTaskExecutor extends AbstractService implements QueryTaskExecutor, Thread.UncaughtExceptionHandler { + /** */ + public static final String THREAD_POOL_NAME = "CalciteQueryExecutor"; + + /** */ + protected final GridKernalContext ctx; + + /** */ + protected Thread.UncaughtExceptionHandler eHnd; + + /** */ + protected AbstractQueryTaskExecutor(GridKernalContext ctx) { + super(ctx); + this.ctx = ctx; + } + + /** {@inheritDoc} */ + @Override public void uncaughtException(Thread t, Throwable e) { + if (eHnd != null) + eHnd.uncaughtException(t, e); + } + + /** {@inheritDoc} */ + @Override public void onStart(GridKernalContext ctx) { + eHnd = ctx.uncaughtExceptionHandler(); + + super.onStart(ctx); + } + + /** */ + protected class SecurityAwareTask implements Runnable { + /** */ + private final SecurityContext secCtx; + + /** */ + private final Runnable qryTask; + + /** */ + public SecurityAwareTask(SecurityContext secCtx, Runnable qryTask) { + this.secCtx = secCtx; + this.qryTask = qryTask; + } + + /** {@inheritDoc} */ + @Override public void run() { + try (AutoCloseable ignored = ctx.security().withContext(secCtx)) { + qryTask.run(); + } + catch (Throwable e) { + U.warn(log, "Uncaught exception", e); + + /* + * No exceptions are rethrown here to preserve the current thread from being destroyed, + * because other queries may be pinned to the current thread id. + * However, unrecoverable errors must be processed by FailureHandler. + */ + uncaughtException(Thread.currentThread(), e); + } + } + } +} diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryAwareTask.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryAwareTask.java new file mode 100644 index 0000000000000..b04f6706306fa --- /dev/null +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryAwareTask.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.exec.task; + +/** + * Query aware task. + */ +interface QueryAwareTask extends Runnable { + /** */ + public QueryKey queryKey(); +} diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryBlockingTaskExecutor.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryBlockingTaskExecutor.java new file mode 100644 index 0000000000000..c19b3dad31a85 --- /dev/null +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryBlockingTaskExecutor.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.exec.task; + +import java.util.UUID; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.managers.communication.GridIoPolicy; +import org.apache.ignite.internal.processors.security.SecurityContext; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.thread.IgniteThreadPoolExecutor; + +import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName; +import static org.apache.ignite.internal.processors.pool.PoolProcessor.THREAD_POOLS; + +/** + * Query task executor based on queue with query blocking. + */ +public class QueryBlockingTaskExecutor extends AbstractQueryTaskExecutor { + /** */ + private final QueryTasksQueue tasksQueue = new QueryTasksQueue(); + + /** */ + private IgniteThreadPoolExecutor executor; + + /** */ + public QueryBlockingTaskExecutor(GridKernalContext ctx) { + super(ctx); + } + + /** {@inheritDoc} */ + @Override public void execute(UUID qryId, long fragmentId, Runnable qryTask) { + SecurityContext secCtx = ctx.security().securityContext(); + + QueryKey qryKey = new QueryKey(qryId, fragmentId); + + executor.execute(new QueryAndSecurityAwareTask(qryKey, secCtx, qryTask)); + } + + /** {@inheritDoc} */ + @Override public void onStart(GridKernalContext ctx) { + super.onStart(ctx); + + executor = new IgniteThreadPoolExecutor( + "calciteQry", + ctx.igniteInstanceName(), + ctx.config().getQueryThreadPoolSize(), + ctx.config().getQueryThreadPoolSize(), + IgniteConfiguration.DFLT_THREAD_KEEP_ALIVE_TIME, + tasksQueue.blockingQueue(), + GridIoPolicy.CALLER_THREAD, + eHnd + ) { + @Override protected void afterExecute(Runnable r, Throwable t) { + tasksQueue.unblockQuery(((QueryAwareTask)r).queryKey()); + + super.afterExecute(r, t); + } + }; + + // Prestart threads to ensure that all threads always use queue to poll tasks (without this call worker can + // get its first task directly from 'execute' method, bypassing tasks queue). + executor.prestartAllCoreThreads(); + + executor.registerMetrics(ctx.metric().registry(metricName(THREAD_POOLS, THREAD_POOL_NAME))); + } + + /** {@inheritDoc} */ + @Override public void tearDown() { + U.shutdownNow(getClass(), executor, log); + } + + /** */ + private class QueryAndSecurityAwareTask extends SecurityAwareTask implements QueryAwareTask { + /** */ + private final QueryKey qryKey; + + /** */ + public QueryAndSecurityAwareTask(QueryKey qryKey, SecurityContext secCtx, Runnable qryTask) { + super(secCtx, qryTask); + + this.qryKey = qryKey; + } + + /** {@inheritDoc} */ + @Override public QueryKey queryKey() { + return qryKey; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(QueryAndSecurityAwareTask.class, this); + } + } +} diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryKey.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryKey.java new file mode 100644 index 0000000000000..411b0fdc52a07 --- /dev/null +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryKey.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.exec.task; + +import java.util.Objects; +import java.util.UUID; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; + +/** + * Query key. + */ +class QueryKey { + /** */ + private final UUID qryId; + + /** */ + private final long fragmentId; + + /** */ + QueryKey(UUID qryId, long fragmentId) { + this.qryId = qryId; + this.fragmentId = fragmentId; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + QueryKey key = (QueryKey)o; + + return fragmentId == key.fragmentId && Objects.equals(qryId, key.qryId); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return U.safeAbs(31 * (31 + (qryId != null ? qryId.hashCode() : 0)) + Long.hashCode(fragmentId)); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(QueryKey.class, this); + } +} diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryTasksQueue.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryTasksQueue.java new file mode 100644 index 0000000000000..6bdc044076a1b --- /dev/null +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryTasksQueue.java @@ -0,0 +1,369 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.exec.task; + +import java.lang.reflect.Array; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * A tasks queue with filtering (based on linked nodes). + */ +class QueryTasksQueue { + /** + * Linked list node class. + */ + private static class Node { + /** */ + QueryAwareTask item; + + /** Next node in chain. */ + Node next; + + /** */ + Node(QueryAwareTask item) { + this.item = item; + } + } + + /** Current number of elements */ + private final AtomicInteger cnt = new AtomicInteger(); + + /** Head of linked list. */ + Node head; + + /** Tail of linked list. */ + private Node last; + + /** */ + private final ReentrantLock lock = new ReentrantLock(); + + /** Wait condition for waiting takes. */ + private final Condition notEmpty = lock.newCondition(); + + /** Set of blocked (currently running) queries. */ + private final Set blockedQrys = new HashSet<>(); + + /** + * Creates a {@code LinkedBlockingQueue}. + */ + QueryTasksQueue() { + last = head = new Node(null); + } + + /** Queue size. */ + public int size() { + return cnt.get(); + } + + /** Add a task to the queue. */ + public void addTask(QueryAwareTask task) { + lock.lock(); + + try { + assert last.next == null : "Unexpected last.next: " + last.next; + + last = last.next = new Node(task); + + cnt.getAndIncrement(); + + notEmpty.signal(); + } + finally { + lock.unlock(); + } + } + + /** Poll task and block query. */ + public QueryAwareTask pollTaskAndBlockQuery(long timeout, TimeUnit unit) throws InterruptedException { + lock.lockInterruptibly(); + + try { + QueryAwareTask res; + + long nanos = unit.toNanos(timeout); + + while (cnt.get() == 0 || (res = dequeue()) == null) { + if (nanos <= 0L) + return null; + + nanos = notEmpty.awaitNanos(nanos); + } + + boolean added = blockedQrys.add(res.queryKey()); + + assert added; + + return res; + } + finally { + lock.unlock(); + } + } + + /** + * Removes a first non-blocked task from the head of the queue. + * + * @return The task. + */ + private QueryAwareTask dequeue() { + assert lock.isHeldByCurrentThread(); + assert head.item == null : "Unexpected head.item: " + head.item; + + for (Node pred = head, cur = pred.next; cur != null; pred = cur, cur = cur.next) { + if (!blockedQrys.contains(cur.item.queryKey())) { // Skip tasks for blocked queries. + QueryAwareTask res = cur.item; + + unlink(pred, cur); + + if (cnt.decrementAndGet() > 0) + notEmpty.signal(); + + return res; + } + } + + return null; + } + + /** Unblock query. */ + public void unblockQuery(QueryKey qryKey) { + lock.lock(); + + try { + boolean removed = blockedQrys.remove(qryKey); + + assert removed; + + if (cnt.get() > 0) + notEmpty.signal(); + } + finally { + lock.unlock(); + } + } + + /** Remove task */ + public boolean removeTask(QueryAwareTask task) { + if (task == null) + return false; + + lock.lock(); + + try { + for (Node pred = head, cur = pred.next; cur != null; pred = cur, cur = cur.next) { + if (task.equals(cur.item)) { + unlink(pred, cur); + + cnt.getAndDecrement(); + + return true; + } + } + + return false; + } + finally { + lock.unlock(); + } + } + + /** + * Unlinks interior Node cur with predecessor pred. + */ + private void unlink(Node pred, Node cur) { + cur.item = null; + pred.next = cur.next; + + if (last == cur) + last = pred; + } + + /** */ + public T[] toArray(T[] a) { + lock.lock(); + + try { + int size = cnt.get(); + + if (a.length < size) + a = (T[])Array.newInstance(a.getClass().getComponentType(), size); + + int k = 0; + + for (Node cur = head.next; cur != null; cur = cur.next) + a[k++] = (T)cur.item; + + while (a.length > k) + a[k++] = null; + + return a; + } + finally { + lock.unlock(); + } + } + + /** */ + public int drainTo(Collection c, int maxElements) { + Objects.requireNonNull(c); + + if (maxElements <= 0) + return 0; + + lock.lock(); + + try { + int n = Math.min(maxElements, cnt.get()); + int i = 0; + + for (Node cur = head.next; i < n && cur != null; cur = cur.next, i++) { + c.add(cur.item); + + unlink(head, cur); + + cnt.getAndDecrement(); + } + + return i; + } + finally { + lock.unlock(); + } + } + + /** + * @return {@code BlockingQueue} on top of {@code QueryTasksQueue}. This blocking queue implements only methods + * required by {@code ThreadPoolExecutor}. + */ + public BlockingQueue blockingQueue() { + return new BlockingQueue<>() { + @Override public boolean add(@NotNull Runnable runnable) { + addTask((QueryAwareTask)runnable); + + return true; + } + + @Override public boolean offer(@NotNull Runnable runnable) { + return add(runnable); + } + + @Override public boolean offer(Runnable runnable, long timeout, @NotNull TimeUnit unit) { + return add(runnable); + } + + @Override public void put(@NotNull Runnable runnable) { + add(runnable); + } + + @Override public int remainingCapacity() { + return Integer.MAX_VALUE; + } + + @Override public boolean remove(Object o) { + return removeTask((QueryAwareTask)o); + } + + @Override public @NotNull Runnable take() throws InterruptedException { + return pollTaskAndBlockQuery(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } + + @Override public @Nullable Runnable poll(long timeout, @NotNull TimeUnit unit) throws InterruptedException { + return pollTaskAndBlockQuery(0, TimeUnit.NANOSECONDS); + } + + @Override public Runnable remove() { + throw new UnsupportedOperationException(); + } + + @Override public Runnable poll() { + throw new UnsupportedOperationException(); + } + + @Override public int size() { + return QueryTasksQueue.this.size(); + } + + @Override public boolean isEmpty() { + return size() == 0; + } + + @Override public @NotNull Object[] toArray() { + return toArray(new Object[size()]); + } + + @Override public @NotNull T[] toArray(@NotNull T[] a) { + return QueryTasksQueue.this.toArray(a); + } + + @Override public int drainTo(@NotNull Collection c) { + return drainTo(c, Integer.MAX_VALUE); + } + + @Override public int drainTo(@NotNull Collection c, int maxElements) { + return QueryTasksQueue.this.drainTo(c, maxElements); + } + + @Override public boolean contains(Object o) { + throw new UnsupportedOperationException(); + } + + @Override public Runnable element() { + throw new UnsupportedOperationException(); + } + + @Override public Runnable peek() { + throw new UnsupportedOperationException(); + } + + @Override public @NotNull Iterator iterator() { + throw new UnsupportedOperationException(); + } + + @Override public boolean containsAll(@NotNull Collection c) { + throw new UnsupportedOperationException(); + } + + @Override public boolean addAll(@NotNull Collection c) { + throw new UnsupportedOperationException(); + } + + @Override public boolean removeAll(@NotNull Collection c) { + throw new UnsupportedOperationException(); + } + + @Override public boolean retainAll(@NotNull Collection c) { + throw new UnsupportedOperationException(); + } + + @Override public void clear() { + throw new UnsupportedOperationException(); + } + }; + } +} diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/QueryTaskExecutorImpl.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/StripedQueryTaskExecutor.java similarity index 61% rename from modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/QueryTaskExecutorImpl.java rename to modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/StripedQueryTaskExecutor.java index b489d05192cd1..213c7cbdda033 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/QueryTaskExecutorImpl.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/task/StripedQueryTaskExecutor.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.ignite.internal.processors.query.calcite.exec; +package org.apache.ignite.internal.processors.query.calcite.exec.task; import java.util.UUID; import org.apache.ignite.internal.GridKernalContext; -import org.apache.ignite.internal.processors.query.calcite.util.AbstractService; import org.apache.ignite.internal.processors.security.SecurityContext; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.thread.IgniteStripedThreadPoolExecutor; @@ -28,25 +27,15 @@ import static org.apache.ignite.internal.processors.pool.PoolProcessor.THREAD_POOLS; /** - * Query task executor. + * Query task executor based on striped pool. */ -public class QueryTaskExecutorImpl extends AbstractService implements QueryTaskExecutor, Thread.UncaughtExceptionHandler { - /** */ - public static final String THREAD_POOL_NAME = "CalciteQueryExecutor"; - - /** */ - private final GridKernalContext ctx; - +public class StripedQueryTaskExecutor extends AbstractQueryTaskExecutor { /** */ private IgniteStripedThreadPoolExecutor stripedThreadPoolExecutor; /** */ - private Thread.UncaughtExceptionHandler eHnd; - - /** */ - public QueryTaskExecutorImpl(GridKernalContext ctx) { + public StripedQueryTaskExecutor(GridKernalContext ctx) { super(ctx); - this.ctx = ctx; } /** @@ -56,40 +45,16 @@ public void stripedThreadPoolExecutor(IgniteStripedThreadPoolExecutor stripedThr this.stripedThreadPoolExecutor = stripedThreadPoolExecutor; } - /** - * @param eHnd Uncaught exception handler. - */ - public void exceptionHandler(Thread.UncaughtExceptionHandler eHnd) { - this.eHnd = eHnd; - } - /** {@inheritDoc} */ @Override public void execute(UUID qryId, long fragmentId, Runnable qryTask) { SecurityContext secCtx = ctx.security().securityContext(); - stripedThreadPoolExecutor.execute( - () -> { - try (AutoCloseable ignored = ctx.security().withContext(secCtx)) { - qryTask.run(); - } - catch (Throwable e) { - U.warn(log, "Uncaught exception", e); - - /* - * No exceptions are rethrown here to preserve the current thread from being destroyed, - * because other queries may be pinned to the current thread id. - * However, unrecoverable errors must be processed by FailureHandler. - */ - uncaughtException(Thread.currentThread(), e); - } - }, - hash(qryId, fragmentId) - ); + stripedThreadPoolExecutor.execute(new SecurityAwareTask(secCtx, qryTask), hash(qryId, fragmentId)); } /** {@inheritDoc} */ @Override public void onStart(GridKernalContext ctx) { - exceptionHandler(ctx.uncaughtExceptionHandler()); + super.onStart(ctx); IgniteStripedThreadPoolExecutor executor = new IgniteStripedThreadPoolExecutor( ctx.config().getQueryThreadPoolSize(), @@ -110,12 +75,6 @@ public void exceptionHandler(Thread.UncaughtExceptionHandler eHnd) { U.shutdownNow(getClass(), stripedThreadPoolExecutor, log); } - /** {@inheritDoc} */ - @Override public void uncaughtException(Thread t, Throwable e) { - if (eHnd != null) - eHnd.uncaughtException(t, e); - } - /** */ private static int hash(UUID qryId, long fragmentId) { // inlined Objects.hash(...) diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/exec/rel/AbstractExecutionTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/exec/rel/AbstractExecutionTest.java index 6e416d64295ec..03275e190368a 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/exec/rel/AbstractExecutionTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/exec/rel/AbstractExecutionTest.java @@ -33,7 +33,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; - import com.google.common.collect.ImmutableMap; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; @@ -48,8 +47,10 @@ import org.apache.ignite.internal.processors.query.calcite.exec.MailboxRegistry; import org.apache.ignite.internal.processors.query.calcite.exec.MailboxRegistryImpl; import org.apache.ignite.internal.processors.query.calcite.exec.QueryTaskExecutor; -import org.apache.ignite.internal.processors.query.calcite.exec.QueryTaskExecutorImpl; import org.apache.ignite.internal.processors.query.calcite.exec.TimeoutServiceImpl; +import org.apache.ignite.internal.processors.query.calcite.exec.task.AbstractQueryTaskExecutor; +import org.apache.ignite.internal.processors.query.calcite.exec.task.QueryBlockingTaskExecutor; +import org.apache.ignite.internal.processors.query.calcite.exec.task.StripedQueryTaskExecutor; import org.apache.ignite.internal.processors.query.calcite.exec.tracker.NoOpIoTracker; import org.apache.ignite.internal.processors.query.calcite.exec.tracker.NoOpMemoryTracker; import org.apache.ignite.internal.processors.query.calcite.message.CalciteMessage; @@ -78,16 +79,16 @@ @RunWith(Parameterized.class) public class AbstractExecutionTest extends GridCommonAbstractTest { /** Last parameter number. */ - protected static final int LAST_PARAM_NUM = 0; + protected static final int LAST_PARAM_NUM = 1; /** Params string. */ - protected static final String PARAMS_STRING = "Execution strategy = {0}"; + protected static final String PARAMS_STRING = "Task executor = {0}, Execution strategy = {1}"; /** */ private Throwable lastE; /** */ - private Map taskExecutors; + private Map taskExecutors; /** */ private Map exchangeServices; @@ -101,6 +102,15 @@ public class AbstractExecutionTest extends GridCommonAbstractTest { /** */ protected int nodesCnt = 3; + /** */ + enum TaskExecutorType { + /** */ + STRIPED, + + /** */ + QUERY_BLOCKING + } + /** */ enum ExecutionStrategy { /** */ @@ -141,9 +151,19 @@ public T2 nextTask(Deque> tasks) { /** */ @Parameterized.Parameters(name = PARAMS_STRING) public static List parameters() { - return Stream.of(ExecutionStrategy.values()).map(s -> new Object[]{s}).collect(Collectors.toList()); + List params = Stream.of(ExecutionStrategy.values()) + .map(s -> new Object[] {TaskExecutorType.STRIPED, s}) + .collect(Collectors.toList()); + + params.add(new Object[] {TaskExecutorType.QUERY_BLOCKING, ExecutionStrategy.FIFO}); + + return params; } + /** Task executor. */ + @Parameterized.Parameter + public TaskExecutorType taskExecutorType; + /** Execution direction. */ @Parameterized.Parameter(LAST_PARAM_NUM) public ExecutionStrategy execStgy; @@ -167,16 +187,29 @@ public void setup() throws Exception { kernal.add(new NoOpIgniteSecurityProcessor(kernal)); kernal.add(new GridCacheProcessor(kernal)); - QueryTaskExecutorImpl taskExecutor = new QueryTaskExecutorImpl(kernal); - taskExecutor.stripedThreadPoolExecutor(new IgniteTestStripedThreadPoolExecutor( - execStgy, - kernal.config().getQueryThreadPoolSize(), - kernal.igniteInstanceName(), - "calciteQry", - this::handle, - true, - DFLT_THREAD_KEEP_ALIVE_TIME - )); + AbstractQueryTaskExecutor taskExecutor; + + if (taskExecutorType == TaskExecutorType.STRIPED) { + StripedQueryTaskExecutor executor = new StripedQueryTaskExecutor(kernal); + + executor.stripedThreadPoolExecutor(new IgniteTestStripedThreadPoolExecutor( + execStgy, + kernal.config().getQueryThreadPoolSize(), + kernal.igniteInstanceName(), + "calciteQry", + this::handle, + true, + DFLT_THREAD_KEEP_ALIVE_TIME + )); + + taskExecutor = executor; + } + else { + taskExecutor = new QueryBlockingTaskExecutor(kernal); + + taskExecutor.onStart(kernal); + } + taskExecutors.put(uuid, taskExecutor); MailboxRegistryImpl mailboxRegistry = new MailboxRegistryImpl(kernal); @@ -263,7 +296,7 @@ public IgniteTestStripedThreadPoolExecutor( /** */ @After public void tearDown() { - taskExecutors.values().forEach(QueryTaskExecutorImpl::tearDown); + taskExecutors.values().forEach(AbstractQueryTaskExecutor::tearDown); if (lastE != null) throw new AssertionError(lastE); diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryBlockingTaskExecutorTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryBlockingTaskExecutorTest.java new file mode 100644 index 0000000000000..9f4a7a6c8a1fe --- /dev/null +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryBlockingTaskExecutorTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.exec.task; + +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.security.NoOpIgniteSecurityProcessor; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.GridTestKernalContext; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +/** */ +public class QueryBlockingTaskExecutorTest extends GridCommonAbstractTest { + /** Tests that tasks for different queries can be executed concurrently. */ + @Test + public void testConcurrentTasks() throws Exception { + GridTestKernalContext ctx = newContext(new IgniteConfiguration().setQueryThreadPoolSize(10)); + ctx.add(new NoOpIgniteSecurityProcessor(ctx)); + QueryBlockingTaskExecutor executor = new QueryBlockingTaskExecutor(ctx); + executor.onStart(ctx); + + CountDownLatch latch = new CountDownLatch(1); + + AtomicInteger cnt = new AtomicInteger(); + AtomicBoolean fail = new AtomicBoolean(); + + UUID qryId1 = UUID.randomUUID(); + UUID qryId2 = UUID.randomUUID(); + + Runnable task = () -> { + cnt.incrementAndGet(); + + try { + if (!latch.await(1_000L, TimeUnit.MILLISECONDS)) + fail.set(true); + } + catch (InterruptedException e) { + fail.set(true); + } + }; + + executor.execute(qryId1, 0, task); + executor.execute(qryId1, 1, task); + executor.execute(qryId2, 0, task); + + assertTrue("Failed to wait for tasks completion", + GridTestUtils.waitForCondition(() -> cnt.get() == 3, 1_000L)); + + latch.countDown(); + + assertFalse("Failed to wait for latch", fail.get()); + } + + /** Tests that tasks for the same query can't be executed concurrently. */ + @Test + public void testSameQueryTasks() throws Exception { + GridTestKernalContext ctx = newContext(new IgniteConfiguration().setQueryThreadPoolSize(10)); + ctx.add(new NoOpIgniteSecurityProcessor(ctx)); + QueryBlockingTaskExecutor executor = new QueryBlockingTaskExecutor(ctx); + executor.onStart(ctx); + + int qryCnt = 20; + int taskCnt = 10_000; + + UUID[] qryIds = new UUID[qryCnt]; + AtomicBoolean[] blocked = new AtomicBoolean[qryCnt]; + + for (int i = 0; i < qryCnt; i += 2) + qryIds[i] = qryIds[i + 1] = UUID.randomUUID(); + + for (int i = 0; i < qryCnt; i++) + blocked[i] = new AtomicBoolean(); + + AtomicInteger cnt = new AtomicInteger(); + AtomicBoolean fail = new AtomicBoolean(); + + for (int i = 0; i < taskCnt; i++) { + int qryIdx = ThreadLocalRandom.current().nextInt(qryCnt); + + executor.execute(qryIds[qryIdx], qryIdx, () -> { + if (!blocked[qryIdx].compareAndSet(false, true)) + fail.set(true); + + doSleep(ThreadLocalRandom.current().nextLong(10L)); + + blocked[qryIdx].set(false); + + cnt.incrementAndGet(); + }); + } + + assertTrue("Failed to wait for tasks completion", + GridTestUtils.waitForCondition(() -> cnt.get() == taskCnt, getTestTimeout())); + + assertFalse("Tasks for the same query executed concurrently", fail.get()); + } +} diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryTasksQueueTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryTasksQueueTest.java new file mode 100644 index 0000000000000..134bf1d15e81c --- /dev/null +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/exec/task/QueryTasksQueueTest.java @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.exec.task; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +/** Tests QueryTasksQueue data structure. */ +public class QueryTasksQueueTest extends GridCommonAbstractTest { + /** */ + @Test + public void testQueryBlockingUnblocking() throws Exception { + long waitTimeout = 10_000L; + + QueryTasksQueue queue = new QueryTasksQueue(); + UUID qryId1 = UUID.randomUUID(); + UUID qryId2 = UUID.randomUUID(); + QueryKey qryKey1 = new QueryKey(qryId1, 0); + QueryKey qryKey2 = new QueryKey(qryId2, 0); + QueryKey qryKey3 = new QueryKey(qryId1, 1); + + queue.addTask(new TestQueryAwareTask(qryKey1)); + queue.addTask(new TestQueryAwareTask(qryKey1)); + queue.addTask(new TestQueryAwareTask(qryKey2)); + queue.addTask(new TestQueryAwareTask(qryKey2)); + queue.addTask(new TestQueryAwareTask(qryKey1)); + queue.addTask(new TestQueryAwareTask(qryKey3)); + + QueryAwareTask task = queue.pollTaskAndBlockQuery(waitTimeout, TimeUnit.MILLISECONDS); + assertEquals(qryKey1, task.queryKey()); + + task = queue.pollTaskAndBlockQuery(waitTimeout, TimeUnit.MILLISECONDS); + assertEquals(qryKey2, task.queryKey()); + + task = queue.pollTaskAndBlockQuery(waitTimeout, TimeUnit.MILLISECONDS); + assertEquals(qryKey3, task.queryKey()); + + // Test threads parking and unparking. + QueryAwareTask[] res = new TestQueryAwareTask[1]; + + Runnable pollAndStoreResult = () -> { + try { + res[0] = queue.pollTaskAndBlockQuery(waitTimeout, TimeUnit.MILLISECONDS); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + }; + + // Unparking on unblock query. + Thread thread1 = new Thread(pollAndStoreResult); + + thread1.start(); + + assertTrue(GridTestUtils.waitForCondition(() -> thread1.getState() == Thread.State.TIMED_WAITING, waitTimeout)); + + queue.unblockQuery(qryKey2); + + thread1.join(waitTimeout); + + assertFalse(thread1.isAlive()); + + assertEquals(qryKey2, res[0].queryKey()); + + // Unparking on new task. + queue.unblockQuery(qryKey3); + + Thread thread2 = new Thread(pollAndStoreResult); + + thread2.start(); + + assertTrue(GridTestUtils.waitForCondition(() -> thread2.getState() == Thread.State.TIMED_WAITING, waitTimeout)); + + queue.addTask(new TestQueryAwareTask(qryKey3)); + + thread2.join(waitTimeout); + + assertFalse(thread2.isAlive()); + + assertEquals(qryKey3, res[0].queryKey()); + + // Get next task after unblocking all pending locks. + queue.unblockQuery(qryKey1); + queue.unblockQuery(qryKey2); + queue.unblockQuery(qryKey3); + + task = queue.pollTaskAndBlockQuery(waitTimeout, TimeUnit.MILLISECONDS); + assertEquals(qryKey1, task.queryKey()); + + // Unparking on unblock query second time. + Thread thread3 = new Thread(pollAndStoreResult); + + thread3.start(); + + assertTrue(GridTestUtils.waitForCondition(() -> thread3.getState() == Thread.State.TIMED_WAITING, waitTimeout)); + + queue.unblockQuery(qryKey1); + + thread3.join(waitTimeout); + + assertFalse(thread3.isAlive()); + + assertEquals(qryKey1, res[0].queryKey()); + + assertEquals(0, queue.size()); + } + + /** */ + @Test + public void testToArray() { + QueryTasksQueue queue = new QueryTasksQueue(); + + QueryKey qryKey1 = new QueryKey(UUID.randomUUID(), 0); + QueryKey qryKey2 = new QueryKey(UUID.randomUUID(), 1); + QueryKey qryKey3 = new QueryKey(UUID.randomUUID(), 2); + + queue.addTask(new TestQueryAwareTask(qryKey1)); + queue.addTask(new TestQueryAwareTask(qryKey2)); + queue.addTask(new TestQueryAwareTask(qryKey3)); + queue.addTask(new TestQueryAwareTask(qryKey1)); + + assertEquals(4, queue.size()); + + QueryAwareTask[] tasks = queue.toArray(new TestQueryAwareTask[2]); + + assertEquals(4, queue.size()); + assertEquals(4, tasks.length); + assertEquals(qryKey1, tasks[0].queryKey()); + assertEquals(qryKey2, tasks[1].queryKey()); + assertEquals(qryKey3, tasks[2].queryKey()); + assertEquals(qryKey1, tasks[3].queryKey()); + + tasks = queue.toArray(new TestQueryAwareTask[] { + null, null, null, null, + new TestQueryAwareTask(qryKey1), + new TestQueryAwareTask(qryKey2) + }); + + assertEquals(4, queue.size()); + assertEquals(6, tasks.length); + assertEquals(qryKey1, tasks[0].queryKey()); + assertEquals(qryKey2, tasks[1].queryKey()); + assertEquals(qryKey3, tasks[2].queryKey()); + assertEquals(qryKey1, tasks[3].queryKey()); + assertNull(tasks[4]); + assertNull(tasks[5]); + } + + /** */ + @Test + public void testDrainTo() { + QueryTasksQueue queue = new QueryTasksQueue(); + + QueryKey qryKey1 = new QueryKey(UUID.randomUUID(), 0); + QueryKey qryKey2 = new QueryKey(UUID.randomUUID(), 1); + QueryKey qryKey3 = new QueryKey(UUID.randomUUID(), 2); + + queue.addTask(new TestQueryAwareTask(qryKey1)); + queue.addTask(new TestQueryAwareTask(qryKey2)); + queue.addTask(new TestQueryAwareTask(qryKey3)); + queue.addTask(new TestQueryAwareTask(qryKey1)); + queue.addTask(new TestQueryAwareTask(qryKey2)); + queue.addTask(new TestQueryAwareTask(qryKey3)); + + List tasks = new ArrayList<>(); + assertEquals(2, queue.drainTo(tasks, 2)); + + assertEquals(4, queue.size()); + assertEquals(2, tasks.size()); + assertEquals(qryKey1, tasks.get(0).queryKey()); + assertEquals(qryKey2, tasks.get(1).queryKey()); + + assertEquals(1, queue.drainTo(tasks, 1)); + + assertEquals(3, queue.size()); + assertEquals(3, tasks.size()); + assertEquals(qryKey3, tasks.get(2).queryKey()); + + tasks.clear(); + + assertEquals(3, queue.drainTo(tasks, Integer.MAX_VALUE)); + + assertEquals(0, queue.size()); + assertEquals(3, tasks.size()); + assertEquals(qryKey1, tasks.get(0).queryKey()); + assertEquals(qryKey2, tasks.get(1).queryKey()); + assertEquals(qryKey3, tasks.get(2).queryKey()); + } + + /** */ + @Test + public void testRemove() { + QueryTasksQueue queue = new QueryTasksQueue(); + + QueryKey qryKey1 = new QueryKey(UUID.randomUUID(), 0); + QueryKey qryKey2 = new QueryKey(UUID.randomUUID(), 1); + QueryAwareTask task1 = new TestQueryAwareTask(qryKey1); + QueryAwareTask task2 = new TestQueryAwareTask(qryKey1); + QueryAwareTask task3 = new TestQueryAwareTask(qryKey2); + QueryAwareTask task4 = new TestQueryAwareTask(qryKey2); + + queue.addTask(task1); + queue.addTask(task2); + queue.addTask(task3); + queue.addTask(task4); + + assertEquals(4, queue.size()); + + queue.removeTask(task2); + + assertEquals(3, queue.size()); + + assertEqualsArraysAware(new QueryAwareTask[] {task1, task3, task4}, queue.toArray(new TestQueryAwareTask[3])); + + queue.removeTask(task1); + + assertEquals(2, queue.size()); + + assertEqualsArraysAware(new QueryAwareTask[] {task3, task4}, queue.toArray(new TestQueryAwareTask[2])); + + queue.removeTask(task4); + + assertEquals(1, queue.size()); + + assertEqualsArraysAware(new QueryAwareTask[] {task3}, queue.toArray(new TestQueryAwareTask[1])); + + queue.removeTask(task3); + + assertEquals(0, queue.size()); + + // Check add after remove of all tasks. + queue.addTask(task1); + queue.addTask(task2); + assertEquals(2, queue.size()); + + assertEqualsArraysAware(new QueryAwareTask[] {task1, task2}, queue.toArray(new TestQueryAwareTask[2])); + } + + /** */ + private static class TestQueryAwareTask implements QueryAwareTask { + /** */ + private final QueryKey qryKey; + + /** */ + public TestQueryAwareTask(QueryKey qryKey) { + this.qryKey = qryKey; + } + + /** {@inheritDoc} */ + @Override public QueryKey queryKey() { + return qryKey; + } + + /** */ + @Override public void run() { + // No-op. + } + } +} diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/AbstractBasicIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/AbstractBasicIntegrationTest.java index 81e6024b56bc5..81c724c396275 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/AbstractBasicIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/AbstractBasicIntegrationTest.java @@ -187,12 +187,12 @@ protected void assertThrows(String sql, Class cls, String m } /** */ - protected IgniteCache createAndPopulateTable() { - return createAndPopulateTable(client, 2, CacheMode.PARTITIONED); + protected void createAndPopulateTable() { + createAndPopulateTable(client, 2, CacheMode.PARTITIONED); } /** */ - protected IgniteCache createAndPopulateTable(Ignite ignite, int backups, CacheMode cacheMode) { + protected void createAndPopulateTable(Ignite ignite, int backups, CacheMode cacheMode) { IgniteCache person = ignite.getOrCreateCache(this.cacheConfiguration() .setName(TABLE_NAME) .setSqlSchema("PUBLIC") @@ -212,8 +212,6 @@ protected IgniteCache createAndPopulateTable(Ignite ignite, i put(ignite, person, idx++, new Employer("Ilya", 15d)); put(ignite, person, idx++, new Employer("Roma", 10d)); put(ignite, person, idx, new Employer("Roma", 10d)); - - return person; } /** */ diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/QueryBlockingTaskExecutorIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/QueryBlockingTaskExecutorIntegrationTest.java new file mode 100644 index 0000000000000..c31b287caa8d3 --- /dev/null +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/QueryBlockingTaskExecutorIntegrationTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.integration; + +import org.apache.ignite.internal.processors.query.calcite.QueryChecker; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.WithSystemProperty; +import org.junit.Test; + +import static org.apache.ignite.internal.processors.query.calcite.CalciteQueryProcessor.IGNITE_CALCITE_USE_QUERY_BLOCKING_TASK_EXECUTOR; + +/** + * Integration test with common queries for query blocking task executor instead of striped task executor. + */ +@WithSystemProperty(key = IGNITE_CALCITE_USE_QUERY_BLOCKING_TASK_EXECUTOR, value = "true") +public class QueryBlockingTaskExecutorIntegrationTest extends AbstractBasicIntegrationTransactionalTest { + /** */ + @Test + public void testSimpleScan() { + createAndPopulateTable(); + + assertQuery("SELECT * FROM person").resultSize(5).check(); + } + + /** */ + @Test + public void testJoinRehash() throws Exception { + sql("CREATE TABLE order_items (id varchar, orderId int, amount int, PRIMARY KEY (id))\n" + + " WITH \"cache_name=order_items,backups=1," + atomicity() + "\""); + + sql("CREATE TABLE orders (id int, region varchar, PRIMARY KEY (id))\n" + + " WITH \"cache_name=orders,backups=1," + atomicity() + "\""); + + sql("CREATE INDEX order_items_orderId ON order_items (orderId ASC)"); + sql("CREATE INDEX orders_region ON orders (region ASC)"); + + for (int i = 0; i < 500; i++) { + sql("INSERT INTO orders VALUES(?, ?)", i, "region" + i % 10); + for (int j = 0; j < 20; j++) + sql("INSERT INTO order_items VALUES(?, ?, ?)", i + "_" + j, i, j); + } + + String sql = "SELECT sum(i.amount)" + + " FROM order_items i JOIN orders o ON o.id=i.orderId" + + " WHERE o.region = ?"; + + assertQuery(sql) + .withParams("region0") + .matches(QueryChecker.containsSubPlan("IgniteMergeJoin")) + .matches(QueryChecker.containsSubPlan("IgniteExchange(distribution=[affinity")) + .returns(9500L) // 50 * sum(0 .. 19) + .check(); + + // Create concurrent queries with a lot of tasks (join rehashing also add a concurrency, + // since send batches between nodes) + GridTestUtils.runMultiThreaded(() -> { + for (int i = 0; i < 100; i++) + assertQuery(sql).withParams("region" + (i % 10)).returns(9500L).check(); + }, 10, "query_starter"); + } +} diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlDiagnosticIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlDiagnosticIntegrationTest.java index b24e8e66d3d22..1c33edd3b06f6 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlDiagnosticIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/SqlDiagnosticIntegrationTest.java @@ -59,7 +59,7 @@ import org.apache.ignite.internal.processors.query.QueryUtils; import org.apache.ignite.internal.processors.query.calcite.Query; import org.apache.ignite.internal.processors.query.calcite.QueryRegistry; -import org.apache.ignite.internal.processors.query.calcite.exec.QueryTaskExecutorImpl; +import org.apache.ignite.internal.processors.query.calcite.exec.task.AbstractQueryTaskExecutor; import org.apache.ignite.internal.processors.security.SecurityContext; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.U; @@ -312,7 +312,7 @@ public void testUserQueriesMetrics() throws Exception { /** */ @Test public void testThreadPoolMetrics() { - String regName = metricName(PoolProcessor.THREAD_POOLS, QueryTaskExecutorImpl.THREAD_POOL_NAME); + String regName = metricName(PoolProcessor.THREAD_POOLS, AbstractQueryTaskExecutor.THREAD_POOL_NAME); MetricRegistry mreg = client.context().metric().registry(regName); LongMetric tasksCnt = mreg.findMetric("CompletedTaskCount"); diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/UserDefinedFunctionsIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/UserDefinedFunctionsIntegrationTest.java index 2f5796393cf29..6772c0030dfc7 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/UserDefinedFunctionsIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/UserDefinedFunctionsIntegrationTest.java @@ -18,18 +18,35 @@ package org.apache.ignite.internal.processors.query.calcite.integration; import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cache.query.annotations.QuerySqlFunction; +import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.WithSystemProperty; import org.junit.Test; +import static org.apache.ignite.internal.processors.query.calcite.CalciteQueryProcessor.IGNITE_CALCITE_USE_QUERY_BLOCKING_TASK_EXECUTOR; + /** * Integration test for user defined functions. */ +@WithSystemProperty(key = IGNITE_CALCITE_USE_QUERY_BLOCKING_TASK_EXECUTOR, value = "true") public class UserDefinedFunctionsIntegrationTest extends AbstractBasicIntegrationTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.getSqlConfiguration().setQueryEnginesConfiguration(new CalciteQueryEngineConfiguration()); + + return cfg; + } + /** */ @Test public void testFunctions() throws Exception { @@ -132,6 +149,25 @@ public static int addFour(int a, int b, int c, int d) { } } + /** */ + @Test + public void testInnerSql() { + IgniteCache emp4 = client.getOrCreateCache(this.cacheConfiguration() + .setName("emp4") + .setSqlFunctionClasses(InnerSqlFunctionsLibrary.class) + .setSqlSchema("PUBLIC") + .setQueryEntities(F.asList(new QueryEntity(Integer.class, Employer.class).setTableName("emp4"))) + ); + + for (int i = 0; i < 100; i++) + put(client, emp4, i, new Employer("Name" + i, (double)i)); + + assertQuery(grid(0), "SELECT sum(salary(?, _key)) FROM emp4") + .withParams(grid(0).name()) + .returns(4950d) + .check(); + } + /** */ public static class MulFunctionsLibrary { /** */ @@ -167,4 +203,16 @@ public static String echo(String s) { return s; } } + + /** */ + public static class InnerSqlFunctionsLibrary { + /** */ + @QuerySqlFunction + public static double salary(String ignite, int key) { + return (double)Ignition.ignite(ignite) + .cache("emp4") + .query(new SqlFieldsQuery("SELECT salary FROM emp4 WHERE _key = ?").setArgs(key)) + .getAll().get(0).get(0); + } + } } diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/planner/AbstractPlannerTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/planner/AbstractPlannerTest.java index 56c23704a510b..95a8c3454e45c 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/planner/AbstractPlannerTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/planner/AbstractPlannerTest.java @@ -62,8 +62,8 @@ import org.apache.ignite.internal.processors.failure.FailureProcessor; import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor; import org.apache.ignite.internal.processors.query.calcite.exec.ExecutionContext; -import org.apache.ignite.internal.processors.query.calcite.exec.QueryTaskExecutorImpl; import org.apache.ignite.internal.processors.query.calcite.exec.RowHandler; +import org.apache.ignite.internal.processors.query.calcite.exec.task.StripedQueryTaskExecutor; import org.apache.ignite.internal.processors.query.calcite.externalize.RelJsonReader; import org.apache.ignite.internal.processors.query.calcite.message.CalciteMessage; import org.apache.ignite.internal.processors.query.calcite.message.MessageServiceImpl; @@ -123,7 +123,7 @@ public abstract class AbstractPlannerTest extends GridCommonAbstractTest { protected List nodes; /** */ - protected List executors; + protected List executors; /** */ protected volatile Throwable lastE; @@ -147,7 +147,7 @@ public void setup() { @After public void tearDown() throws Throwable { if (!F.isEmpty(executors)) - executors.forEach(QueryTaskExecutorImpl::tearDown); + executors.forEach(StripedQueryTaskExecutor::tearDown); if (lastE != null) throw lastE; diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/planner/PlannerTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/planner/PlannerTest.java index 512861139f514..b3b52d4b840dd 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/planner/PlannerTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/planner/PlannerTest.java @@ -39,10 +39,10 @@ import org.apache.ignite.internal.processors.query.calcite.exec.ExecutionContext; import org.apache.ignite.internal.processors.query.calcite.exec.LogicalRelImplementor; import org.apache.ignite.internal.processors.query.calcite.exec.MailboxRegistryImpl; -import org.apache.ignite.internal.processors.query.calcite.exec.QueryTaskExecutorImpl; import org.apache.ignite.internal.processors.query.calcite.exec.rel.Node; import org.apache.ignite.internal.processors.query.calcite.exec.rel.Outbox; import org.apache.ignite.internal.processors.query.calcite.exec.rel.RootNode; +import org.apache.ignite.internal.processors.query.calcite.exec.task.StripedQueryTaskExecutor; import org.apache.ignite.internal.processors.query.calcite.exec.tracker.NoOpIoTracker; import org.apache.ignite.internal.processors.query.calcite.exec.tracker.NoOpMemoryTracker; import org.apache.ignite.internal.processors.query.calcite.message.MessageServiceImpl; @@ -380,7 +380,7 @@ private Node implementFragment( kernal.add(new NoOpIgniteSecurityProcessor(kernal)); kernal.add(new GridCacheProcessor(kernal)); - QueryTaskExecutorImpl taskExecutor = new QueryTaskExecutorImpl(kernal); + StripedQueryTaskExecutor taskExecutor = new StripedQueryTaskExecutor(kernal); taskExecutor.stripedThreadPoolExecutor(new IgniteStripedThreadPoolExecutor( kernal.config().getQueryThreadPoolSize(), kernal.igniteInstanceName(), diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java index 5bdf5d417b510..b9f1914f9e10d 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IgniteCalciteTestSuite.java @@ -17,13 +17,8 @@ package org.apache.ignite.testsuites; -import org.apache.ignite.internal.processors.query.calcite.QueryCheckerTest; -import org.apache.ignite.internal.processors.query.calcite.exec.ClosableIteratorsHolderTest; -import org.apache.ignite.internal.processors.query.calcite.exec.KeyFilteringCursorTest; import org.apache.ignite.internal.processors.query.calcite.exec.LogicalRelImplementorTest; import org.apache.ignite.internal.processors.query.calcite.exec.NumericTypesPrecisionsTest; -import org.apache.ignite.internal.processors.query.calcite.exec.exp.IgniteSqlFunctionsTest; -import org.apache.ignite.internal.processors.query.calcite.exec.tracker.MemoryTrackerTest; import org.apache.ignite.internal.processors.query.calcite.jdbc.JdbcConnectionEnabledPropertyTest; import org.apache.ignite.internal.processors.query.calcite.jdbc.JdbcSetClientInfoTest; import org.apache.ignite.internal.processors.query.calcite.jdbc.JdbcThinTransactionalSelfTest; @@ -43,13 +38,10 @@ PlannerTestSuite.class, ExecutionTestSuite.class, IntegrationTestSuite.class, + UtilTestSuite.class, - ClosableIteratorsHolderTest.class, - MemoryTrackerTest.class, - QueryCheckerTest.class, SqlCustomParserTest.class, SqlReservedWordsTest.class, - IgniteSqlFunctionsTest.class, LogicalRelImplementorTest.class, ScriptTestSuite.class, @@ -57,7 +49,6 @@ NumericTypesPrecisionsTest.class, - KeyFilteringCursorTest.class, SqlTransactionsIsolationTest.class, SqlTransactionsUnsupportedModesTest.class, diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java index 8e64b50f6d923..50032d9253f5e 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/IntegrationTestSuite.java @@ -54,6 +54,7 @@ import org.apache.ignite.internal.processors.query.calcite.integration.OperatorsExtensionIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.PartitionPruneTest; import org.apache.ignite.internal.processors.query.calcite.integration.PartitionsReservationIntegrationTest; +import org.apache.ignite.internal.processors.query.calcite.integration.QueryBlockingTaskExecutorIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.QueryEngineConfigurationIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.QueryMetadataIntegrationTest; import org.apache.ignite.internal.processors.query.calcite.integration.QueryWithPartitionsIntegrationTest; @@ -151,6 +152,7 @@ OperatorsExtensionIntegrationTest.class, SessionContextSqlFunctionTest.class, SqlPlanHistoryIntegrationTest.class, + QueryBlockingTaskExecutorIntegrationTest.class, }) public class IntegrationTestSuite { } diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/UtilTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/UtilTestSuite.java new file mode 100644 index 0000000000000..527f0240060d6 --- /dev/null +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/UtilTestSuite.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import org.apache.ignite.internal.processors.query.calcite.QueryCheckerTest; +import org.apache.ignite.internal.processors.query.calcite.exec.ClosableIteratorsHolderTest; +import org.apache.ignite.internal.processors.query.calcite.exec.KeyFilteringCursorTest; +import org.apache.ignite.internal.processors.query.calcite.exec.exp.IgniteSqlFunctionsTest; +import org.apache.ignite.internal.processors.query.calcite.exec.task.QueryBlockingTaskExecutorTest; +import org.apache.ignite.internal.processors.query.calcite.exec.task.QueryTasksQueueTest; +import org.apache.ignite.internal.processors.query.calcite.exec.tracker.MemoryTrackerTest; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * Calcite utility classes tests. + */ +@RunWith(Suite.class) +@Suite.SuiteClasses({ + ClosableIteratorsHolderTest.class, + MemoryTrackerTest.class, + QueryCheckerTest.class, + IgniteSqlFunctionsTest.class, + KeyFilteringCursorTest.class, + QueryBlockingTaskExecutorTest.class, + QueryTasksQueueTest.class, +}) +public class UtilTestSuite { +} From 9147c1cb8424a49f36727d9df853bb949d15adb9 Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 31 Jan 2025 10:00:55 +0300 Subject: [PATCH 06/14] IGNITE-24336 SQL Calcite: Add support of user-defined table functions - Fixes #11832. Signed-off-by: Aleksey Plekhanov --- docs/_docs/SQL/custom-sql-func.adoc | 17 + .../org/apache/ignite/snippets/SqlAPI.java | 29 ++ .../calcite/exec/LogicalRelImplementor.java | 4 +- .../query/calcite/exec/TableFunctionScan.java | 31 +- .../exp/IgniteReflectiveFunctionBase.java | 38 ++ .../exec/exp/IgniteScalarFunction.java | 15 +- .../calcite/exec/exp/IgniteTableFunction.java | 135 +++++++ .../calcite/schema/SchemaHolderImpl.java | 16 + .../AbstractBasicIntegrationTest.java | 13 + .../UserDefinedFunctionsIntegrationTest.java | 352 ++++++++++++++++++ .../calcite/jdbc/JdbcSetClientInfoTest.java | 20 + .../query/annotations/QuerySqlFunction.java | 1 + .../annotations/QuerySqlTableFunction.java | 84 +++++ .../schema/AbstractSchemaChangeListener.java | 11 + .../query/schema/SchemaChangeListener.java | 19 +- .../schema/management/SchemaManager.java | 37 +- .../processors/query/h2/H2SchemaManager.java | 11 + 17 files changed, 807 insertions(+), 26 deletions(-) create mode 100644 modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteReflectiveFunctionBase.java create mode 100644 modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteTableFunction.java create mode 100644 modules/core/src/main/java/org/apache/ignite/cache/query/annotations/QuerySqlTableFunction.java diff --git a/docs/_docs/SQL/custom-sql-func.adoc b/docs/_docs/SQL/custom-sql-func.adoc index c531fc64c7d67..7c9540c87ca92 100644 --- a/docs/_docs/SQL/custom-sql-func.adoc +++ b/docs/_docs/SQL/custom-sql-func.adoc @@ -46,4 +46,21 @@ Once you have deployed a cache with the above configuration, you can call the cu include::{javaFile}[tags=sql-function-query, indent=0] ---- + +Custom SQL function can be a table function. Result of table function is treated as a row set (a table) and can be used +by other SQL operators. Custom SQL function is also a `public` method marked by annotation `@QuerySqlTableFunction`. +Table function must return an `Iterable` as a row set. Each row can be represented by an `Object[]` or by a `Collection`. +Row length must match the defined number of column types. Row value types must match the defined column types or be able +assigned to them. + +[source,java] +---- +include::{javaFile}[tags=sql-table-function-example, indent=0] +---- +[source,java] +---- +include::{javaFile}[tags=sql-table-function-config-query, indent=0] +---- +NOTE: The table functions also are available currently only with link:SQL/sql-calcite[Calcite, window=_blank]. + NOTE: Classes registered with `CacheConfiguration.setSqlFunctionClasses(...)` must be added to the classpath of all the nodes where the defined custom functions might be executed. Otherwise, you will get a `ClassNotFoundException` error when trying to execute the custom function. diff --git a/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/SqlAPI.java b/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/SqlAPI.java index a6dab34559d9a..9de660bf2dbf3 100644 --- a/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/SqlAPI.java +++ b/docs/_docs/code-snippets/java/src/main/java/org/apache/ignite/snippets/SqlAPI.java @@ -165,6 +165,18 @@ public static int sqr(int x) { // end::sql-function-example[] + // tag::sql-table-function-example[] + static class SqlTableFunctions { + @QuerySqlTableFunction(columnTypes = {Integer.class, String.class}, columnNames = {"INT_COL", "STR_COL"}) + public static Iterable table_function(int i) { + return Arrays.asList( + new Object[] {i, "" + i}, + new Object[] {i * 10, "empty"} + ); + } + } + // end::sql-table-function-example[] + @Test IgniteCache setSqlFunction(Ignite ignite) { @@ -181,6 +193,23 @@ IgniteCache setSqlFunction(Ignite ignite) { return cache; } + @Test + IgniteCache testSqlTableFunction(Ignite ignite) { + // tag::sql-table-function-config-query[] + CacheConfiguration cfg = new CacheConfiguration("myCache"); + + cfg.setSqlFunctionClasses(SqlTableFunctions.class); + + IgniteCache cache = ignite.createCache(cfg); + + SqlFieldsQuery query = new SqlFieldsQuery("SELECT STR_COL FROM TABLE_FUNCTION(10) WHERE INT_COL > 50"); + + cache.query(query).getAll(); + // end::sql-table-function-config-query[] + + return cache; + } + void call(IgniteCache cache) { // tag::sql-function-query[] diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/LogicalRelImplementor.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/LogicalRelImplementor.java index 30737feed7a78..e2e72b05b9325 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/LogicalRelImplementor.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/LogicalRelImplementor.java @@ -662,13 +662,13 @@ else if (rel instanceof Intersect) /** {@inheritDoc} */ @Override public Node visit(IgniteTableFunctionScan rel) { - Supplier> dataSupplier = expressionFactory.execute(rel.getCall()); + Supplier> dataSupplier = expressionFactory.execute(rel.getCall()); RelDataType rowType = rel.getRowType(); RowFactory rowFactory = ctx.rowHandler().factory(ctx.getTypeFactory(), rowType); - return new ScanNode<>(ctx, rowType, new TableFunctionScan<>(dataSupplier, rowFactory)); + return new ScanNode<>(ctx, rowType, new TableFunctionScan<>(rowType, dataSupplier, rowFactory)); } /** {@inheritDoc} */ diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/TableFunctionScan.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/TableFunctionScan.java index a8531842def1b..b29f91d6a7fe8 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/TableFunctionScan.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/TableFunctionScan.java @@ -17,30 +17,55 @@ package org.apache.ignite.internal.processors.query.calcite.exec; +import java.util.Collection; import java.util.Iterator; import java.util.function.Supplier; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.calcite.exec.RowHandler.RowFactory; import org.apache.ignite.internal.util.typedef.F; /** */ public class TableFunctionScan implements Iterable { /** */ - private final Supplier> dataSupplier; + private final RelDataType rowType; + + /** */ + private final Supplier> dataSupplier; /** */ private final RowFactory rowFactory; /** */ public TableFunctionScan( - Supplier> dataSupplier, + RelDataType rowType, + Supplier> dataSupplier, RowFactory rowFactory ) { + this.rowType = rowType; this.dataSupplier = dataSupplier; this.rowFactory = rowFactory; } /** {@inheritDoc} */ @Override public Iterator iterator() { - return F.iterator(dataSupplier.get(), rowFactory::create, true); + return F.iterator(dataSupplier.get(), this::convertToRow, true); + } + + /** */ + private Row convertToRow(Object rowContainer) { + if (rowContainer.getClass() != Object[].class && !Collection.class.isAssignableFrom(rowContainer.getClass())) + throw new IgniteSQLException("Unable to process table function data: row type is neither Collection or Object[]."); + + Object[] rowArr = rowContainer.getClass() == Object[].class + ? (Object[])rowContainer + : ((Collection)rowContainer).toArray(); + + if (rowArr.length != rowType.getFieldCount()) { + throw new IgniteSQLException("Unable to process table function data: row length [" + rowArr.length + + "] doesn't match defined columns number [" + rowType.getFieldCount() + "]."); + } + + return rowFactory.create(rowArr); } } diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteReflectiveFunctionBase.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteReflectiveFunctionBase.java new file mode 100644 index 0000000000000..1a5dcf8b04578 --- /dev/null +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteReflectiveFunctionBase.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.query.calcite.exec.exp; + +import java.lang.reflect.Method; +import org.apache.calcite.schema.impl.ReflectiveFunctionBase; + +/** A base for outer java-method functions. */ +abstract class IgniteReflectiveFunctionBase extends ReflectiveFunctionBase implements ImplementableFunction { + /** */ + protected final CallImplementor implementor; + + /** */ + protected IgniteReflectiveFunctionBase(Method method, CallImplementor implementor) { + super(method); + + this.implementor = implementor; + } + + /** {@inheritDoc} */ + @Override public CallImplementor getImplementor() { + return implementor; + } +} diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteScalarFunction.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteScalarFunction.java index 41655ef16d5f1..8c43f5a8620eb 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteScalarFunction.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteScalarFunction.java @@ -21,22 +21,16 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.schema.ScalarFunction; -import org.apache.calcite.schema.impl.ReflectiveFunctionBase; /** * Implementation of {@link ScalarFunction} for Ignite user defined functions. */ -public class IgniteScalarFunction extends ReflectiveFunctionBase implements ScalarFunction, ImplementableFunction { - /** Implementor. */ - private final CallImplementor implementor; - +public class IgniteScalarFunction extends IgniteReflectiveFunctionBase implements ScalarFunction { /** * Private constructor. */ private IgniteScalarFunction(Method method, CallImplementor implementor) { - super(method); - - this.implementor = implementor; + super(method, implementor); } /** @@ -56,9 +50,4 @@ public static ScalarFunction create(Method method) { @Override public RelDataType getReturnType(RelDataTypeFactory typeFactory) { return typeFactory.createJavaType(method.getReturnType()); } - - /** {@inheritDoc} */ - @Override public CallImplementor getImplementor() { - return implementor; - } } diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteTableFunction.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteTableFunction.java new file mode 100644 index 0000000000000..e9ece1ee9a740 --- /dev/null +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteTableFunction.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.query.calcite.exec.exp; + +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.java.JavaTypeFactory; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.TableFunction; +import org.apache.ignite.cache.query.annotations.QuerySqlTableFunction; +import org.apache.ignite.internal.processors.query.IgniteSQLException; +import org.apache.ignite.internal.util.typedef.F; + +/** + * Holder of user-defined table function. + * + * @see QuerySqlTableFunction + */ +public class IgniteTableFunction extends IgniteReflectiveFunctionBase implements TableFunction { + /** Column types of the returned table representation. */ + private final Class[] colTypes; + + /** Column names of the returned table representation. */ + private final List colNames; + + /** + * Creates user-defined table function holder. + * + * @param method The implementatng method. + * @param colTypes Column types of the returned table representation. + * @param colNames Column names of the returned table representation. + * @param implementor Call implementor. + */ + private IgniteTableFunction(Method method, Class[] colTypes, String[] colNames, CallImplementor implementor) { + super(method, implementor); + + validate(method, colTypes, colNames); + + this.colTypes = colTypes; + this.colNames = Arrays.asList(colNames); + } + + /** + * Creates user-defined table function implementor and holder. + * + * @param method The implementating method. + * @param colTypes Column types of the returned table representation. + * @param colNames Column names of the returned table representation. + */ + public static IgniteTableFunction create(Method method, Class[] colTypes, String[] colNames) { + NotNullImplementor implementor = new ReflectiveCallNotNullImplementor(method); + + CallImplementor impl = RexImpTable.createImplementor(implementor, NullPolicy.NONE, false); + + return new IgniteTableFunction(method, colTypes, colNames, impl); + } + + /** {@inheritDoc} */ + @Override public RelDataType getRowType(RelDataTypeFactory typeFactory, List arguments) { + JavaTypeFactory tf = (JavaTypeFactory)typeFactory; + + List converted = Stream.of(colTypes).map(cl -> tf.toSql(tf.createType(cl))).collect(Collectors.toList()); + + return typeFactory.createStructType(converted, colNames); + } + + /** {@inheritDoc} */ + @Override public Type getElementType(List arguments) { + // Calcite's {@link TableFunctionImpl} does real invocation ({@link TableFunctionImpl#apply(List)}) to determine + // the type. The call might be long, 'heavy', may affect some metrics and should not be executed at validation/planning. + // We may check the argument number here but not their types. The types might be wrong, but converted further. + if (F.isEmpty(arguments) && !F.isEmpty(method.getParameterTypes()) + || F.isEmpty(method.getParameterTypes()) && !F.isEmpty(arguments) + || method.getParameterTypes().length != arguments.size()) { + throw new IllegalArgumentException("Wrong arguments number: " + arguments.size() + ". Expected: " + + method.getParameterTypes().length + '.'); + } + + return Iterable.class; + } + + /** Validates the parameters and throws an exception if it finds an incorrect parameter. */ + private static void validate(Method mtd, Class[] colTypes, String[] colNames) { + if (F.isEmpty(colTypes)) + raiseValidationError(mtd, "Column types cannot be empty."); + + if (F.isEmpty(colNames)) + raiseValidationError(mtd, "Column names cannot be empty."); + + if (colTypes.length != colNames.length) { + raiseValidationError(mtd, "Number of the table column names [" + colNames.length + + "] must match the number of column types [" + colTypes.length + "]."); + } + + if (new HashSet<>(Arrays.asList(colNames)).size() != colNames.length) + raiseValidationError(mtd, "One or more column names is not unique."); + + if (!Iterable.class.isAssignableFrom(mtd.getReturnType())) + raiseValidationError(mtd, "The method is expected to return a collection (iterable)."); + } + + /** + * Throws a parameter validation exception with a standard text prefix. + * + * @param method A java-method implementing related user-defined table function. + * @param errPostfix Error text postfix. + */ + private static void raiseValidationError(Method method, String errPostfix) { + String mtdSign = method.getName() + '(' + Stream.of(method.getParameterTypes()).map(Class::getSimpleName) + .collect(Collectors.joining(", ")) + ')'; + + throw new IgniteSQLException("Unable to create table function for method '" + mtdSign + "'. " + errPostfix); + } +} diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/SchemaHolderImpl.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/SchemaHolderImpl.java index a3d78d68fd04e..80cbee8d300c7 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/SchemaHolderImpl.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/SchemaHolderImpl.java @@ -42,6 +42,7 @@ import org.apache.ignite.internal.processors.query.QueryField; import org.apache.ignite.internal.processors.query.QueryUtils; import org.apache.ignite.internal.processors.query.calcite.exec.exp.IgniteScalarFunction; +import org.apache.ignite.internal.processors.query.calcite.exec.exp.IgniteTableFunction; import org.apache.ignite.internal.processors.query.calcite.trait.TraitUtils; import org.apache.ignite.internal.processors.query.calcite.type.IgniteTypeFactory; import org.apache.ignite.internal.processors.query.calcite.util.AbstractService; @@ -357,6 +358,21 @@ private static Object affinityIdentity(CacheConfiguration ccfg) { rebuild(); } + /** {@inheritDoc} */ + @Override public void onTableFunctionCreated( + String schemaName, + String name, + Method method, + Class[] colTypes, + String[] colNames + ) { + IgniteSchema schema = igniteSchemas.computeIfAbsent(schemaName, IgniteSchema::new); + + schema.addFunction(name.toUpperCase(), IgniteTableFunction.create(method, colTypes, colNames)); + + rebuild(); + } + /** {@inheritDoc} */ @Override public void onSystemViewCreated(String schemaName, SystemView sysView) { IgniteSchema schema = igniteSchemas.computeIfAbsent(schemaName, IgniteSchema::new); diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/AbstractBasicIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/AbstractBasicIntegrationTest.java index 81c724c396275..59c00cd32bc91 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/AbstractBasicIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/AbstractBasicIntegrationTest.java @@ -334,5 +334,18 @@ public Employer(String name, Double salary) { this.name = name; this.salary = salary; } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + Employer employer = (Employer)o; + + return name.equals(employer.name) && salary.equals(employer.salary); + } } } diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/UserDefinedFunctionsIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/UserDefinedFunctionsIntegrationTest.java index 6772c0030dfc7..422d3a8c2db86 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/UserDefinedFunctionsIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/UserDefinedFunctionsIntegrationTest.java @@ -17,17 +17,25 @@ package org.apache.ignite.internal.processors.query.calcite.integration; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import org.apache.calcite.sql.validate.SqlValidatorException; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.Ignition; import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cache.query.annotations.QuerySqlFunction; +import org.apache.ignite.cache.query.annotations.QuerySqlTableFunction; import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.ListeningTestLogger; +import org.apache.ignite.testframework.LogListener; import org.apache.ignite.testframework.junits.WithSystemProperty; import org.junit.Test; @@ -38,12 +46,18 @@ */ @WithSystemProperty(key = IGNITE_CALCITE_USE_QUERY_BLOCKING_TASK_EXECUTOR, value = "true") public class UserDefinedFunctionsIntegrationTest extends AbstractBasicIntegrationTest { + /** Log listener. */ + private static final ListeningTestLogger listeningLog = new ListeningTestLogger(log); + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); cfg.getSqlConfiguration().setQueryEnginesConfiguration(new CalciteQueryEngineConfiguration()); + if (igniteInstanceName.endsWith("0")) + cfg.setGridLogger(listeningLog); + return cfg; } @@ -122,12 +136,350 @@ public void testFunctions() throws Exception { assertQuery("SELECT sq(4)").returns(16d).check(); } + /** */ + @Test + public void testTableFunctions() throws Exception { + IgniteCache emp = client.getOrCreateCache(new CacheConfiguration("emp") + .setSqlSchema("PUBLIC") + .setSqlFunctionClasses(TableFunctionsLibrary.class) + .setQueryEntities(F.asList(new QueryEntity(Integer.class, Employer.class).setTableName("emp"))) + ); + + emp.put(1, new Employer("Igor1", 1d)); + emp.put(2, new Employer("Roman1", 2d)); + + awaitPartitionMapExchange(); + + assertQuery("SELECT * from collectionRow(?)").withParams(1) + .returns(2, 3, 4) + .returns(5, 6, 7) + .returns(8, 9, 10) + .check(); + + assertQuery("SELECT * from collectionRow(?) WHERE COL_2=4").withParams(2) + .returns(3, 4, 5) + .check(); + + assertQuery("SELECT COL_1, COL_3 from collectionRow(?) WHERE COL_2=3").withParams(1) + .returns(2, 4) + .check(); + + // Overrides. + assertQuery("SELECT * from collectionRow(?, 2, ?)").withParams(1, 3) + .returns(11, 22, 33) + .returns(41, 52, 63) + .returns(71, 82, 93) + .check(); + + assertQuery("SELECT * from arrayRow(?)").withParams(1) + .returns(2, 3, 4) + .returns(5, 6, 7) + .returns(8, 9, 10) + .check(); + + assertQuery("SELECT COL_1, COL_3 from arrayRow_and_it(1) WHERE COL_1>4 AND COL_2>? AND COL_3>6").withParams(5) + .returns(5, 7) + .returns(8, 10) + .check(); + + assertQuery("SELECT * from boxingUnboxing(1, ?, ?, 4.0::FLOAT)").withParams(1, 4.0d) + .returns(1, 1, 4.0d, 4.0d) + .check(); + + assertQuery("SELECT * from boxingUnboxing(1, 1, 2, 2)") + .returns(1, 1, 2.0d, 2.0d) + .check(); + + assertQuery("SELECT * from boxingUnboxing(?, ?, ?, ?)").withParams(1, 1, 2.0d, 2.0d) + .returns(1, 1, 2.0d, 2.0d) + .check(); + + assertQuery("SELECT * from emp WHERE SALARY >= (SELECT COL_1 from collectionRow(1) WHERE COL_2=3)") + .returns("Roman1", 2d) + .check(); + + assertQuery("SELECT * from aliasedName(?)").withParams(1) + .returns(2, 3, 4) + .returns(5, 6, 7) + .check(); + + assertQuery("SELECT * from raiseException(?, ?, ?)").withParams(1, "test", false) + .returns(2, "test2") + .returns(3, "test3") + .check(); + + assertThrows("SELECT * from raiseException(?, ?, ?)", IgniteSQLException.class, "An error occurred while query executing", + 1, "test", true); + + assertThrows("SELECT * from raiseException(?, ?, ?)", RuntimeException.class, "Test exception", + 1, "test", true); + + // Object type. + assertQuery("SELECT * from withObjectType(1)") + .returns(1, new Employer("emp1", 1000d)) + .returns(10, new Employer("emp10", 10000d)) + .check(); + + assertQuery("SELECT * from withObjectType(1) where EMP=?") + .withParams(new Employer("emp10", 10000d)) + .returns(10, new Employer("emp10", 10000d)) + .check(); + } + + /** */ + @Test + public void testIncorrectTableFunctions() throws Exception { + LogListener logChecker0 = LogListener.matches("One or more column names is not unique") + .andMatches("must match the number of column types") + .andMatches("The method is expected to return a collection (iterable)") + .andMatches("Column types cannot be empty") + .andMatches("Column names cannot be empty") + .build(); + + listeningLog.registerListener(logChecker0); + + IgniteCache emp = client.getOrCreateCache(new CacheConfiguration("emp") + .setSqlSchema("PUBLIC") + .setSqlFunctionClasses(IncorrectTableFunctionsLibrary.class) + .setQueryEntities(F.asList(new QueryEntity(Integer.class, Employer.class).setTableName("emp"))) + ); + + emp.put(1, new Employer("Igor1", 1d)); + emp.put(2, new Employer("Roman1", 2d)); + + awaitPartitionMapExchange(); + + // Ensure the cache SQL is OK. + assertQuery("SELECT * from emp WHERE SALARY >= 2") + .returns("Roman1", 2d) + .check(); + + assertThrows("SELECT * FROM wrongRowLength(1)", IgniteSQLException.class, + "row length [2] doesn't match defined columns number [3]"); + + assertThrows("SELECT * FROM wrongRowType(1)", IgniteSQLException.class, + "row type is neither Collection or Object[]"); + + logChecker0.check(getTestTimeout()); + + String errTxt = "No match found for function signature"; + + assertThrows("SELECT * FROM duplicateColumnName(1, 'a')", SqlValidatorException.class, errTxt); + + assertThrows("SELECT * FROM wrongColumnNamesNumber(1, 'a')", SqlValidatorException.class, errTxt); + + assertThrows("SELECT * FROM wrongReturnType(1, 'a')", SqlValidatorException.class, errTxt); + + assertThrows("SELECT * FROM noReturnType(1, 'a')", SqlValidatorException.class, errTxt); + + assertThrows("SELECT * FROM noColumnTypes(1, 'a')", SqlValidatorException.class, errTxt); + + assertThrows("SELECT * FROM noColumnNames(1, 'a')", SqlValidatorException.class, errTxt); + } + + /** */ + @Test + public void testUnregistrableTableFunctions() { + GridTestUtils.assertThrowsAnyCause( + null, + () -> client.getOrCreateCache(new CacheConfiguration("emp") + .setSqlFunctionClasses(UnregistrableTableFunctionsLibrary.class)), + IgniteCheckedException.class, + "both table and non-table function variants are defined" + ); + } + /** */ @SuppressWarnings("ThrowableNotThrown") private void assertThrows(String sql) { GridTestUtils.assertThrowsWithCause(() -> assertQuery(sql).check(), IgniteSQLException.class); } + /** */ + public static final class TableFunctionsLibrary { + /** */ + private TableFunctionsLibrary() { + // No-op. + } + + /** Trivial test. Returns collections as row holders. */ + @QuerySqlTableFunction(columnTypes = {int.class, int.class, int.class}, columnNames = {"COL_1", "COL_2", "COL_3"}) + public static Iterable> collectionRow(int x) { + return Arrays.asList( + Arrays.asList(x + 1, x + 2, x + 3), + Arrays.asList(x + 4, x + 5, x + 6), + Arrays.asList(x + 7, x + 8, x + 9) + ); + } + + /** Overrides. */ + @QuerySqlTableFunction(columnTypes = {int.class, int.class, int.class}, columnNames = {"COL_1", "COL_2", "COL_3"}) + public static Collection> collectionRow(int x, int y, int z) { + return Arrays.asList( + Arrays.asList(x + 10, y + 20, z + 30), + Arrays.asList(x + 40, y + 50, z + 60), + Arrays.asList(x + 70, y + 80, z + 90) + ); + } + + /** Returns arrays as row holders. */ + @QuerySqlTableFunction(columnTypes = {int.class, int.class, int.class}, columnNames = {"COL_1", "COL_2", "COL_3"}) + public static Iterable arrayRow(int x) { + return Arrays.asList( + new Object[] {x + 1, x + 2, x + 3}, + new Object[] {x + 4, x + 5, x + 6}, + new Object[] {x + 7, x + 8, x + 9} + ); + } + + /** Returns mixed row holders. */ + @QuerySqlTableFunction(columnTypes = {int.class, int.class, int.class}, columnNames = {"COL_1", "COL_2", "COL_3"}) + public static Collection arrayRow_and_it(int x) { + return Arrays.asList( + new Object[] {x + 1, x + 2, x + 3}, + Arrays.asList(x + 4, x + 5, x + 6), + new Object[] {x + 7, x + 8, x + 9} + ); + } + + /** Boxed/unboxed test. */ + @QuerySqlTableFunction(columnTypes = {Integer.class, int.class, Double.class, double.class}, + columnNames = {"COL_1", "COL_2", "COL_3", "COL_4"}) + public static Collection> boxingUnboxing(int i1, Integer i2, double d1, Double d2) { + return List.of(Arrays.asList(i1, i2, d1, d2)); + } + + /** Alias test. */ + @QuerySqlTableFunction(columnTypes = {int.class, int.class, int.class}, columnNames = {"COL_1", "COL_2", "COL_3"}, + alias = "aliasedName") + public static Iterable> alias(int x) { + return Arrays.asList( + Arrays.asList(x + 1, x + 2, x + 3), + Arrays.asList(x + 4, x + 5, x + 6) + ); + } + + /** Can raise a user-side exception. */ + @QuerySqlTableFunction(columnTypes = {int.class, String.class}, columnNames = {"COL_1", "COL_2"}) + public static Iterable> raiseException(int i, String str, boolean doThrow) { + if (doThrow) + throw new RuntimeException("Test exception."); + + return Arrays.asList( + Arrays.asList(i + 1, str + (i + 1)), + Arrays.asList(i + 2, str + (i + 2)) + ); + } + + /** User exception test. */ + @QuerySqlTableFunction(columnTypes = {int.class, Object.class}, columnNames = {"ID", "EMP"}) + public static Iterable> withObjectType(int i) { + return Arrays.asList( + Arrays.asList(i, new Employer("emp" + i, i * 1000d)), + Arrays.asList(i * 10, new Employer("emp" + i * 10, i * 10000d)) + ); + } + } + + /** */ + public static final class IncorrectTableFunctionsLibrary { + /** */ + private IncorrectTableFunctionsLibrary() { + // No-op. + } + + /** Duplicated column names. */ + @QuerySqlTableFunction(columnTypes = {Integer.class, String.class}, columnNames = {"INT_COL", "INT_COL"}) + public static Iterable duplicateColumnName(int i, String s) { + return Arrays.asList( + Arrays.asList(i, s + i), + Arrays.asList(i * 10, s + (i * 10)) + ); + } + + /** Non-matching number of the column names. */ + @QuerySqlTableFunction(columnTypes = {Integer.class, String.class}, columnNames = {"INT_COL"}) + public static Iterable wrongColumnNamesNumber(int i, String s) { + return Arrays.asList( + Arrays.asList(i, s + i), + Arrays.asList(i * 10, s + (i * 10)) + ); + } + + /** Wrong return type 1. */ + @QuerySqlTableFunction(columnTypes = {Integer.class, Integer.class}, columnNames = {"COL_1", "COL_2"}) + public static Object[] wrongReturnType(int i) { + return new Object[] { + Arrays.asList(i * 2, i * 2 + 1), + Arrays.asList(i * 3, i * 3 + 1) + }; + } + + /** Wrong return type 2. */ + @QuerySqlTableFunction(columnTypes = {Integer.class}, columnNames = {"COL_1"}) + public static void noReturnType(int i) { + System.err.println("Test value: " + i); + } + + /** Empty column types. */ + @QuerySqlTableFunction(columnTypes = {}, columnNames = {"INT_COL", "STR_COL"}) + public static Collection noColumnTypes(int i, String s) { + return Arrays.asList( + Arrays.asList(i, s + i), + Arrays.asList(i * 10, s + (i * 10)) + ); + } + + /** Empty column names. */ + @QuerySqlTableFunction(columnTypes = {Integer.class, String.class}, columnNames = {}) + public static Collection noColumnNames(int i, String s) { + return Arrays.asList( + Arrays.asList(i, s + i), + Arrays.asList(i * 10, s + (i * 10)) + ); + } + + /** Incorrect row length. */ + @QuerySqlTableFunction(columnTypes = {int.class, String.class, double.class}, columnNames = {"INT_COL", "STR_COL", "DBL_COL"}) + public static Collection wrongRowLength(int i) { + return Arrays.asList( + Arrays.asList(i, "code_" + i, i * 1000.0d), + Arrays.asList(i + 15, "code_" + (i + 15), (i + 15) * 1000.0d), + // Mismatched length. + Arrays.asList(i * 100, i * 100000.0d) + ); + } + + /** Incorrect row type. */ + @QuerySqlTableFunction(columnTypes = {int.class, String.class, double.class}, columnNames = {"INT_COL", "STR_COL", "DBL_COL"}) + public static Collection wrongRowType(int i) { + return Arrays.asList( + Arrays.asList(i, "code_" + i, i * 1000.0d), + "wrongType", + // Mismatched length. + Arrays.asList(i * 100, i * 100000.0d) + ); + } + } + + /** */ + public static final class UnregistrableTableFunctionsLibrary { + /** */ + private UnregistrableTableFunctionsLibrary() { + // No-op. + } + + /** Both variants. */ + @QuerySqlFunction + @QuerySqlTableFunction(columnTypes = {Integer.class, String.class}, columnNames = {"INT_COL", "STR_COL"}) + public static Collection bothVariants(int i, String s) { + return Arrays.asList( + Arrays.asList(i, s + i), + Arrays.asList(i * 10, s + (i * 10)) + ); + } + } + /** */ public static class AddFunctionsLibrary { /** */ diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/jdbc/JdbcSetClientInfoTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/jdbc/JdbcSetClientInfoTest.java index 6b47c5ce7d607..89e36df4f89ed 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/jdbc/JdbcSetClientInfoTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/jdbc/JdbcSetClientInfoTest.java @@ -22,10 +22,13 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Properties; import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cache.query.annotations.QuerySqlFunction; +import org.apache.ignite.cache.query.annotations.QuerySqlTableFunction; import org.apache.ignite.calcite.CalciteQueryEngineConfiguration; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -272,6 +275,15 @@ private void checkSessionId(Connection conn, @Nullable String sesId) throws Exce String actSesId = set.getString("SESSION_ID"); assertEquals(sesId, actSesId); + + // Ensure that the context works also with a table function. + set = jdbcQuery(conn, "select SID from sessionIdTbl();"); + + set.next(); + + actSesId = set.getString(1); + + assertEquals(sesId, actSesId); } /** */ @@ -287,5 +299,13 @@ public String sessionId() { return sesCtx == null ? null : sesCtx.getAttribute(SESSION_ID); } + + /** */ + @QuerySqlTableFunction(columnTypes = {String.class}, columnNames = {"SID"}) + public Collection sessionIdTbl() { + SessionContext sesCtx = sesCtxProv.getSessionContext(); + + return Collections.singletonList(new Object[] {sesCtx == null ? null : sesCtx.getAttribute(SESSION_ID)}); + } } } diff --git a/modules/core/src/main/java/org/apache/ignite/cache/query/annotations/QuerySqlFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/query/annotations/QuerySqlFunction.java index 1f86c6192c353..fdd2c188cad7c 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/query/annotations/QuerySqlFunction.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/query/annotations/QuerySqlFunction.java @@ -60,6 +60,7 @@ * Note, accessing to the attributes is available in the Calcite query engine only. In a such case a class must have public * zero-args constructor. * + * @see QuerySqlTableFunction * @see SessionContextProviderResource */ @Documented diff --git a/modules/core/src/main/java/org/apache/ignite/cache/query/annotations/QuerySqlTableFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/query/annotations/QuerySqlTableFunction.java new file mode 100644 index 0000000000000..7c41bf954ef22 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/cache/query/annotations/QuerySqlTableFunction.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.cache.query.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.apache.ignite.resources.SessionContextProviderResource; + +/** + * Annotates public methods in classes to be used in SQL queries as custom table functions. Annotated class must be + * registered using {@link org.apache.ignite.configuration.CacheConfiguration#setSqlFunctionClasses(Class[])}. + *

+ * Usage example: + * 

+ *     public class MyTableFunctions {
+ *         @QuerySqlTableFunction(columnTypes = {Float.class, String.class}, columnNames = {"F_VAL", "S_VAL"})
+ *         public static Iterable<Object[]> my_table(int i, Float f, String str) {
+ *             return Arrays.asList(
+ *                 new Object[]{f, "code_" + (i * 10) + ": " + str},
+ *                 new Object[]{null, "code_" + (i * 20) + ": " + str},
+ *                 new Object[]{f,  "code_" + (i * 30) + ": " + str}
+ *             );
+ *         }
+ *     }
+ *
+ *     // Register in CacheConfiguration.
+ *     cacheCfg.setSqlFunctionClasses(MyTableFunctions.class);
+ *
+ *     // And use in queries.
+ *     cache.query(new SqlFieldsQuery("select S_VAL from MY_TABLE(1, 5.0f, "ext") where F_VAL is not null"));
+ *
+ *

+ * Table function must return an {@code Iterable} as a row set. Each row can be represented by an {@code Object[]} or + * by an {@code Collection}. Row length must match the defined number of column types. Row value types must match the + * defined column types or be able assigned to them. + *

+ * Note, the table functions are available currently only with Calcite. + * + * @see QuerySqlFunction + * @see SessionContextProviderResource + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +public @interface QuerySqlTableFunction { + /** + * Specifies alias for the table function name to be used form SQL queries. If no alias provided, the method name is used. + * + * @return Alias for table function name. + */ + String alias() default ""; + + /** + * Defines column types of the returned SQL table representation. Number of the types must match number of {@link #columnNames()}. + * + * @return Table column types. + */ + Class[] columnTypes(); + + /** + * Defines column names of the returned SQL table representation. Number of the names must match number of {@link #columnTypes()}. + * + * @return Table column names. + */ + String[] columnNames(); +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/AbstractSchemaChangeListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/AbstractSchemaChangeListener.java index ede1e036c2f7d..7049d2d327a43 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/AbstractSchemaChangeListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/AbstractSchemaChangeListener.java @@ -107,6 +107,17 @@ public abstract class AbstractSchemaChangeListener implements SchemaChangeListen // No-op. } + /** {@inheritDoc} */ + @Override public void onTableFunctionCreated( + String schemaName, + String name, + Method method, + Class[] colTypes, + String[] colNames + ) { + // No-op. + } + /** {@inheritDoc} */ @Override public void onSystemViewCreated(String schemaName, SystemView sysView) { // No-op. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaChangeListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaChangeListener.java index 2e88a0ee495d2..d28243bf422af 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaChangeListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaChangeListener.java @@ -136,10 +136,27 @@ public void onColumnsDropped( * @param schemaName Schema name. * @param name Function name. * @param deterministic Specifies if the function is deterministic (result depends only on input parameters) - * @param method Public static method, implementing this function. + * @param method Public method, implementing this function. */ public void onFunctionCreated(String schemaName, String name, boolean deterministic, Method method); + /** + * Callback on table function creation. + * + * @param schemaName Schema name. + * @param name Function name. + * @param colTypes Column types of the table representation. + * @param colNames Column names of the table representation. + * @param method Public method, implementing this function. + */ + public void onTableFunctionCreated( + String schemaName, + String name, + Method method, + Class[] colTypes, + String[] colNames + ); + /** * Callback on system view creation. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/management/SchemaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/management/SchemaManager.java index d7e98885fda4b..eeb35c754f4ff 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/management/SchemaManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/management/SchemaManager.java @@ -42,6 +42,7 @@ import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.cache.QueryIndexType; import org.apache.ignite.cache.query.annotations.QuerySqlFunction; +import org.apache.ignite.cache.query.annotations.QuerySqlTableFunction; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.cache.query.index.IndexName; import org.apache.ignite.internal.cache.query.index.sorted.IndexKeyDefinition; @@ -504,17 +505,28 @@ private void createSqlFunctions(String schema, Class[] clss) throws IgniteChe for (Class cls : clss) { for (Method m : cls.getDeclaredMethods()) { - QuerySqlFunction ann = m.getAnnotation(QuerySqlFunction.class); + QuerySqlFunction fun = m.getAnnotation(QuerySqlFunction.class); + QuerySqlTableFunction tableFun = m.getAnnotation(QuerySqlTableFunction.class); - if (ann != null) { - int modifiers = m.getModifiers(); + if (fun == null && tableFun == null) + continue; - if (!Modifier.isPublic(modifiers)) - throw new IgniteCheckedException("Method " + m.getName() + " must be public."); + if (!Modifier.isPublic(m.getModifiers())) { + throw new IgniteCheckedException("Failed to register method '" + m.getName() + "' as a SQL function: " + + "method must be public."); + } - String alias = ann.alias().isEmpty() ? m.getName() : ann.alias(); + if (fun != null && tableFun != null) { + throw new IgniteCheckedException("Failed to register method '" + m.getName() + "' as a SQL function: " + + "both table and non-table function variants are defined."); + } + + if (fun != null) + lsnr.onFunctionCreated(schema, fun.alias().isEmpty() ? m.getName() : fun.alias(), fun.deterministic(), m); + else { + String alias = tableFun.alias().isEmpty() ? m.getName() : tableFun.alias(); - lsnr.onFunctionCreated(schema, alias, ann.deterministic(), m); + lsnr.onTableFunctionCreated(schema, alias, m, tableFun.columnTypes(), tableFun.columnNames()); } } } @@ -1415,6 +1427,17 @@ private CompoundSchemaChangeListener(GridKernalContext ctx, List executeSafe(() -> lsnr.onFunctionCreated(schemaName, name, deterministic, method))); } + /** {@inheritDoc} */ + @Override public void onTableFunctionCreated( + String schemaName, + String name, + Method method, + Class[] colTypes, + String[] colNames + ) { + lsnrs.forEach(lsnr -> executeSafe(() -> lsnr.onTableFunctionCreated(schemaName, name, method, colTypes, colNames))); + } + /** {@inheritDoc} */ @Override public void onSystemViewCreated(String schemaName, SystemView sysView) { lsnrs.forEach(lsnr -> executeSafe(() -> lsnr.onSystemViewCreated(schemaName, sysView))); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2SchemaManager.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2SchemaManager.java index 73f29291083f8..5031345ab4392 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2SchemaManager.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2SchemaManager.java @@ -237,6 +237,17 @@ private void dropTable(H2TableDescriptor tbl) { } } + /** {@inheritDoc} */ + @Override public void onTableFunctionCreated( + String schemaName, + String name, + Method method, + Class[] colTypes, + String[] colNames + ) { + log.warning("Skipped creation of SQL table function '" + name + "' in H2 engine. Table functions arent't supported yet."); + } + /** * Registers SQL function. * From 45095bfd8a1c3a095ac4040ea5a279014a401c1f Mon Sep 17 00:00:00 2001 From: Vladimir Steshin Date: Fri, 31 Jan 2025 10:05:22 +0300 Subject: [PATCH 07/14] IGNITE-23414 SQL Calcite: Fix cast of dynamic parameters - Fixes #11787. Signed-off-by: Aleksey Plekhanov --- .../calcite/prepare/IgniteSqlValidator.java | 90 ++++++++++--------- .../calcite/prepare/PlanningContext.java | 7 +- .../DynamicParametersIntegrationTest.java | 42 +++++++++ .../JoinRehashIntegrationTest.java | 2 +- 4 files changed, 97 insertions(+), 44 deletions(-) diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java index 0c662acb7206c..91fe45db4b382 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgniteSqlValidator.java @@ -70,6 +70,7 @@ import org.apache.ignite.internal.processors.query.calcite.schema.IgniteTable; import org.apache.ignite.internal.processors.query.calcite.sql.IgniteSqlDecimalLiteral; import org.apache.ignite.internal.processors.query.calcite.type.IgniteTypeFactory; +import org.apache.ignite.internal.processors.query.calcite.type.OtherType; import org.apache.ignite.internal.processors.query.calcite.util.IgniteResource; import org.apache.ignite.internal.util.typedef.F; import org.jetbrains.annotations.Nullable; @@ -102,7 +103,7 @@ public class IgniteSqlValidator extends SqlValidatorImpl { } /** Dynamic parameters. */ - Object[] parameters; + @Nullable private final Object[] parameters; /** * Creates a validator. @@ -110,12 +111,17 @@ public class IgniteSqlValidator extends SqlValidatorImpl { * @param opTab Operator table * @param catalogReader Catalog reader * @param typeFactory Type factory - * @param config Config + * @param cfg Config * @param parameters Dynamic parameters */ - public IgniteSqlValidator(SqlOperatorTable opTab, CalciteCatalogReader catalogReader, - IgniteTypeFactory typeFactory, SqlValidator.Config config, Object[] parameters) { - super(opTab, catalogReader, typeFactory, config); + public IgniteSqlValidator( + SqlOperatorTable opTab, + CalciteCatalogReader catalogReader, + IgniteTypeFactory typeFactory, + SqlValidator.Config cfg, + @Nullable Object[] parameters + ) { + super(opTab, catalogReader, typeFactory, cfg); this.parameters = parameters; } @@ -534,9 +540,47 @@ private boolean isSystemFieldName(String alias) { || QueryUtils.VAL_FIELD_NAME.equalsIgnoreCase(alias); } + /** {@inheritDoc} */ + @Override public RelDataType deriveType(SqlValidatorScope scope, SqlNode expr) { + if (expr instanceof SqlDynamicParam) { + RelDataType type = deriveDynamicParameterType((SqlDynamicParam)expr); + + if (type != null) + return type; + } + + return super.deriveType(scope, expr); + } + + /** @return A derived type or {@code null} if unable to determine. */ + @Nullable private RelDataType deriveDynamicParameterType(SqlDynamicParam node) { + RelDataType type = getValidatedNodeTypeIfKnown(node); + + // Do not clarify the widest type for any value. + if (type instanceof OtherType) + return type; + + if (parameters == null || node.getIndex() >= parameters.length) + return null; + + Object val = parameters[node.getIndex()]; + + if (val == null && type != null) + return type; + + type = val == null + ? typeFactory.createSqlType(SqlTypeName.NULL) + : typeFactory().createTypeWithNullability(typeFactory().toSql(typeFactory().createType(val.getClass())), true); + + setValidatedNodeType(node, type); + + return type; + } + /** {@inheritDoc} */ @Override protected void inferUnknownTypes(RelDataType inferredType, SqlValidatorScope scope, SqlNode node) { - if (inferDynamicParamType(inferredType, node)) + if (node instanceof SqlDynamicParam && unknownType.equals(inferredType) + && deriveDynamicParameterType((SqlDynamicParam)node) != null) return; if (node instanceof SqlCall) { @@ -583,40 +627,6 @@ else if (operandTypeChecker instanceof FamilyOperandTypeChecker) { super.inferUnknownTypes(inferredType, scope, node); } - /** - * Tries to set actual type of dynamic parameter if {@code node} is a {@link SqlDynamicParam} and if its index - * is actual to {@link #parameters}. - * - * @return {@code True} if a new type was set. {@code False} otherwise. - */ - private boolean inferDynamicParamType(RelDataType inferredType, SqlNode node) { - if (parameters == null || !(node instanceof SqlDynamicParam) || ((SqlDynamicParam)node).getIndex() >= parameters.length) - return false; - - Object val = parameters[((SqlDynamicParam)node).getIndex()]; - - if (val == null) { - if (inferredType.equals(unknownType)) { - setValidatedNodeType(node, typeFactory().createSqlType(SqlTypeName.NULL)); - - return true; - } - - return false; - } - - RelDataType valType = typeFactory().toSql(typeFactory().createType(val.getClass())); - - assert !unknownType.equals(valType); - - if (unknownType.equals(inferredType) || valType.getFamily().equals(inferredType.getFamily())) - setValidatedNodeType(node, valType); - else - setValidatedNodeType(node, inferredType); - - return true; - } - /** {@inheritDoc} */ @Override public SqlLiteral resolveLiteral(SqlLiteral literal) { if (literal instanceof SqlNumericLiteral && literal.createSqlType(typeFactory).getSqlTypeName() == SqlTypeName.BIGINT) { diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlanningContext.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlanningContext.java index 2f222083c6a61..d50c2f227604f 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlanningContext.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlanningContext.java @@ -33,6 +33,7 @@ import org.apache.ignite.internal.processors.query.calcite.type.IgniteTypeFactory; import org.apache.ignite.internal.util.typedef.internal.U; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; /** * Planning context. @@ -45,7 +46,7 @@ public final class PlanningContext implements Context { private final String qry; /** */ - private final Object[] parameters; + @Nullable private final Object[] parameters; /** */ private final CancelFlag cancelFlag = new CancelFlag(new AtomicBoolean()); @@ -68,7 +69,7 @@ public final class PlanningContext implements Context { private PlanningContext( Context parentCtx, String qry, - Object[] parameters, + @Nullable Object[] parameters, long plannerTimeout ) { this.qry = qry; @@ -90,7 +91,7 @@ public String query() { * @return Query parameters. */ @SuppressWarnings("AssignmentOrReturnOfFieldWithMutableType") - public Object[] parameters() { + @Nullable public Object[] parameters() { return parameters; } diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/DynamicParametersIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/DynamicParametersIntegrationTest.java index b3eea80c0f32f..cbfc657123ecb 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/DynamicParametersIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/DynamicParametersIntegrationTest.java @@ -26,6 +26,8 @@ import java.time.Period; import java.util.List; import java.util.UUID; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.util.typedef.F; import org.junit.Test; @@ -60,6 +62,46 @@ public void testMetadataTypesForDynamicParameters() { } } + /** */ + @Test + public void testMissedValue() { + assertThrows("SELECT ?", IgniteSQLException.class, "Illegal use of dynamic parameter"); + + assertThrows("SELECT ?, ?", IgniteSQLException.class, "Illegal use of dynamic parameter", "arg0"); + } + + /** */ + @Test + public void testCasts() { + assertQuery("SELECT CAST(? as INTEGER)").withParams('1').returns(1).check(); + assertQuery("SELECT ?::INTEGER").withParams('1').returns(1).check(); + assertQuery("SELECT ?::VARCHAR").withParams(1).returns("1").check(); + assertQuery("SELECT CAST(? as VARCHAR)").withParams(1).returns("1").check(); + + createAndPopulateTable(); + + IgniteCache cache = client.getOrCreateCache(TABLE_NAME); + + cache.put(cache.size(), new Employer("15", 15d)); + + assertQuery("SELECT name FROM Person WHERE id=?::INTEGER").withParams("2").returns("Ilya").check(); + assertQuery("SELECT name FROM Person WHERE id=CAST(? as INTEGER)").withParams("2").returns("Ilya").check(); + + assertQuery("SELECT id FROM Person WHERE name=CAST(? as VARCHAR)").withParams(15).returns(5).check(); + assertQuery("SELECT id FROM Person WHERE name IN (?::VARCHAR)").withParams(15).returns(5).check(); + assertQuery("SELECT name FROM Person WHERE id IN (?::INTEGER)").withParams("2").returns("Ilya").check(); + assertQuery("SELECT name FROM Person WHERE id IN (?::INTEGER, ?::INTEGER)").withParams("2", "3") + .returns("Ilya").returns("Roma").check(); + + assertQuery("SELECT count(*) FROM Person WHERE ? IS NOT NULL").withParams(1).returns(6L).check(); + assertQuery("SELECT count(*) FROM Person WHERE ? IS NOT NULL").withParams("abc").returns(6L).check(); + assertQuery("SELECT count(*) FROM Person WHERE ? IS NOT NULL").withParams(new Object[] { null }).returns(0L).check(); + + assertQuery("SELECT count(*) FROM Person WHERE ? IS NULL").withParams(1).returns(0L).check(); + assertQuery("SELECT count(*) FROM Person WHERE ? IS NULL").withParams("abc").returns(0L).check(); + assertQuery("SELECT count(*) FROM Person WHERE ? IS NULL").withParams(new Object[] {null}).returns(6L).check(); + } + /** */ @Test public void testDynamicParameters() { diff --git a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/JoinRehashIntegrationTest.java b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/JoinRehashIntegrationTest.java index 2370eea664f0c..928ca43049e5f 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/JoinRehashIntegrationTest.java +++ b/modules/calcite/src/test/java/org/apache/ignite/internal/processors/query/calcite/integration/JoinRehashIntegrationTest.java @@ -49,7 +49,7 @@ public void testResourceCleanup() throws Exception { // AbstractBasicIntegrationTest.afterTest method. GridTestUtils.runMultiThreaded(() -> { for (int i = 0; i < 100; i++) - sql(sql, i % 10); + sql(sql, "region" + (i % 10)); }, 10, "query_starter"); } From 509e955d6c2a0afbdb674d23fd6565920550c942 Mon Sep 17 00:00:00 2001 From: Maksim Davydov Date: Fri, 31 Jan 2025 10:13:10 +0300 Subject: [PATCH 08/14] IGNITE-24362 Fix CacheExchangeMessageDuplicatedStateTest - Fixes #11842. Signed-off-by: Aleksey Plekhanov --- .../cache/CacheExchangeMessageDuplicatedStateTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheExchangeMessageDuplicatedStateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheExchangeMessageDuplicatedStateTest.java index 648e2e90f7024..78db7aa4d2ed0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheExchangeMessageDuplicatedStateTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheExchangeMessageDuplicatedStateTest.java @@ -230,7 +230,7 @@ private void checkFullMessage(GridDhtPartitionsFullMessage msg) { assertFalse(dupPartsData.containsKey(CU.cacheId(AFF3_CACHE1))); Map partCntrs = - getFieldValue(getFieldValue(msg, "partCntrs2"), "map"); + getFieldValue(getFieldValue(msg, "partCntrs"), "map"); if (partCntrs != null) { for (CachePartitionFullCountersMap cntrs : partCntrs.values()) { From 1a0534f51e3baba4883213d17797dbd9b51353dd Mon Sep 17 00:00:00 2001 From: Maksim Timonin Date: Fri, 31 Jan 2025 18:56:23 +0300 Subject: [PATCH 09/14] IGNITE-24348 Provide custom FrameworkConfig to SQL views (#11839) --- .../query/calcite/CalciteQueryProcessor.java | 8 ++++---- .../query/calcite/schema/IgniteSchema.java | 6 ++++-- .../query/calcite/schema/SchemaHolderImpl.java | 9 +++++++-- .../query/calcite/schema/ViewTableMacroImpl.java | 9 +++++++-- .../OperatorsExtensionIntegrationTest.java | 12 ++++++++++++ .../query/calcite/planner/AbstractPlannerTest.java | 2 +- 6 files changed, 35 insertions(+), 11 deletions(-) diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessor.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessor.java index 83f1c7e7418f2..f060d32c01197 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessor.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/CalciteQueryProcessor.java @@ -259,8 +259,11 @@ public class CalciteQueryProcessor extends GridProcessorAdapter implements Query public CalciteQueryProcessor(GridKernalContext ctx) { super(ctx); + FrameworkConfig customFrameworkCfg = ctx.plugins().createComponent(FrameworkConfig.class); + frameworkCfg = customFrameworkCfg != null ? customFrameworkCfg : FRAMEWORK_CONFIG; + failureProcessor = ctx.failure(); - schemaHolder = new SchemaHolderImpl(ctx); + schemaHolder = new SchemaHolderImpl(ctx, frameworkCfg); qryPlanCache = new QueryPlanCacheImpl(ctx); parserMetrics = new QueryParserMetricsHolder(ctx.metric()); mailboxRegistry = new MailboxRegistryImpl(ctx); @@ -277,9 +280,6 @@ public CalciteQueryProcessor(GridKernalContext ctx) { qryReg = new QueryRegistryImpl(ctx); injectSvc = new InjectResourcesService(ctx); - FrameworkConfig customFrameworkCfg = ctx.plugins().createComponent(FrameworkConfig.class); - frameworkCfg = customFrameworkCfg != null ? customFrameworkCfg : FRAMEWORK_CONFIG; - QueryEngineConfiguration[] qryEnginesCfg = ctx.config().getSqlConfiguration().getQueryEnginesConfiguration(); if (F.isEmpty(qryEnginesCfg)) diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/IgniteSchema.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/IgniteSchema.java index ef81dc008fd72..2e032f66067ff 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/IgniteSchema.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/IgniteSchema.java @@ -27,6 +27,7 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.calcite.schema.impl.AbstractSchema; +import org.apache.calcite.tools.FrameworkConfig; /** * Ignite schema. @@ -111,12 +112,13 @@ public void removeView(String name) { * Registers current {@code IgniteSchema} in parent {@code SchemaPlus}. * * @param parent Parent schema. + * @param frameworkCfg Framework config. * @return Registered schema. */ - public SchemaPlus register(SchemaPlus parent) { + public SchemaPlus register(SchemaPlus parent, FrameworkConfig frameworkCfg) { SchemaPlus newSchema = parent.add(schemaName, this); - viewMap.forEach((name, sql) -> newSchema.add(name, new ViewTableMacroImpl(sql, newSchema))); + viewMap.forEach((name, sql) -> newSchema.add(name, new ViewTableMacroImpl(sql, newSchema, frameworkCfg))); return newSchema; } diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/SchemaHolderImpl.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/SchemaHolderImpl.java index 80cbee8d300c7..2542228e65453 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/SchemaHolderImpl.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/SchemaHolderImpl.java @@ -27,6 +27,7 @@ import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.Frameworks; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.mapping.Mappings; @@ -66,6 +67,9 @@ public class SchemaHolderImpl extends AbstractService implements SchemaHolder, S /** */ private final GridKernalContext ctx; + /** */ + private final FrameworkConfig frameworkCfg; + /** */ private GridInternalSubscriptionProcessor subscriptionProcessor; @@ -137,10 +141,11 @@ public AffinityIdentity(AffinityFunction aff, int backups, IgnitePredicate schemas) { SchemaPlus dfltSchema = null; for (IgniteSchema igniteSchema : schemas) { - SchemaPlus schema = igniteSchema.register(rootSchema); + SchemaPlus schema = igniteSchema.register(rootSchema, null); if (dfltSchema == null || DEFAULT_SCHEMA.equals(schema.getName())) dfltSchema = schema; From 9617b1c5df9bedf529ac5309c347e6b61b0cdd63 Mon Sep 17 00:00:00 2001 From: Maksim Davydov <70368398+maksaska@users.noreply.github.com> Date: Mon, 3 Feb 2025 13:21:26 +0300 Subject: [PATCH 10/14] IGNITE-24102 Remove VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT (#11821) --- .../tcp/internal/InboundConnectionHandler.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/InboundConnectionHandler.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/InboundConnectionHandler.java index 8107493981772..a288f95dcc761 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/InboundConnectionHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/InboundConnectionHandler.java @@ -46,7 +46,6 @@ import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteInClosure; -import org.apache.ignite.lang.IgniteProductVersion; import org.apache.ignite.lang.IgniteRunnable; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.communication.CommunicationListener; @@ -76,12 +75,6 @@ * This class implement NioListener which process handshake stage, and transmit messages to session. */ public class InboundConnectionHandler extends GridNioServerListenerAdapter { - /** - * Version when client is ready to wait to connect to server (could be needed when client tries to open connection - * before it starts being visible for server) - */ - private static final IgniteProductVersion VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT = IgniteProductVersion.fromString("2.1.4"); - /** Message tracker meta for session. */ private static final int TRACKER_META = GridNioSessionMetaKey.nextUniqueKey(); @@ -505,8 +498,7 @@ private void onFirstMessage(final GridNioSession ses, Message msg) { if (node0 != null) { assert node0.isClient() : node0; - if (node0.version().compareTo(VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT) >= 0) - unknownNode = false; + unknownNode = false; } } else if (discoverySpi instanceof IgniteDiscoverySpi) From d9409b7386ce939d3657f288cefe9691df1101c4 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Mon, 3 Feb 2025 21:26:05 +0300 Subject: [PATCH 11/14] IGNITE-24130 Move binaryMetadata, marshaller directories logic to NodeFileTree (#11840) --- .../FileStoreHeapUtilizationJolBenchmark.java | 11 +- .../JdbcAbstractDmlStatementSelfTest.java | 12 +- .../FoldersReuseCompatibilityTest.java | 7 +- ...ePersistenceCompatibilityAbstractTest.java | 5 +- .../SnapshotCompressionBasicTest.java | 5 +- .../DataStorageConfiguration.java | 6 - .../org/apache/ignite/dump/DumpReader.java | 3 +- .../internal/MarshallerContextImpl.java | 46 +--- .../internal/MarshallerMappingFileStore.java | 3 +- .../apache/ignite/internal/cdc/CdcMain.java | 44 ++-- .../cache/binary/BinaryMetadataFileStore.java | 26 +- .../CacheObjectBinaryProcessorImpl.java | 46 +--- .../persistence/filename/NodeFileTree.java | 229 ++++++++++++++++++ .../filename/PdsConsistentIdProcessor.java | 79 ++++-- .../filename/PdsFolderResolver.java | 5 +- .../filename/PdsFoldersResolver.java | 5 + .../persistence/filename/SharedFileTree.java | 198 +++++++++++++++ .../snapshot/IgniteSnapshotManager.java | 34 +-- .../IncrementalSnapshotFutureTask.java | 15 +- .../snapshot/SnapshotRestoreProcess.java | 10 +- .../cache/persistence/snapshot/dump/Dump.java | 31 ++- .../reader/StandaloneGridKernalContext.java | 9 +- .../apache/ignite/client/ReliabilityTest.java | 6 +- .../ConcurrentMappingFileReadWriteTest.java | 4 +- .../MarshallerContextLockingSelfTest.java | 4 + .../binary/BinaryMarshallerSelfTest.java | 2 + .../OptimizedMarshallerEnumSelfTest.java | 3 + ...rCacheClientRequestsMappingOnMissTest.java | 21 +- ...shallerCacheClientRequestsMappingTest.java | 15 +- .../IgniteMarshallerCacheFSRestoreTest.java | 20 +- .../MarshallerCacheJobRunNodeRestartTest.java | 3 +- .../binary/BinaryMetadataInMemoryTest.java | 8 +- .../BinaryMetadataMoveLegacyFolderTest.java | 17 +- ...ridBinaryCacheEntryMemorySizeSelfTest.java | 2 + ...nitePdsBinaryMetadataAsyncWritingTest.java | 36 +-- ...PdsBinaryMetadataOnClusterRestartTest.java | 13 +- .../IgnitePdsNoSpaceLeftOnDeviceTest.java | 4 +- .../MaintenanceRegistrySimpleTest.java | 5 + .../IgniteUidAsConsistentIdMigrationTest.java | 67 ++--- .../db/wal/reader/IgniteWalReaderTest.java | 12 +- .../db/wal/reader/MockWalIteratorFactory.java | 5 + .../snapshot/PlainSnapshotTest.java | 17 +- .../marshaller/MarshallerContextSelfTest.java | 42 ++-- .../junits/GridAbstractTest.java | 25 +- .../junits/GridTestBinaryMarshaller.java | 2 + .../junits/common/GridCommonAbstractTest.java | 10 +- .../IgniteEncryptedWalConverterTest.java | 14 +- .../utils/IgniteWalConverterTest.java | 18 +- 48 files changed, 794 insertions(+), 410 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java index 35d7d587dcf0e..808cab245cc1b 100644 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java @@ -31,6 +31,7 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.U; @@ -38,8 +39,6 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.openjdk.jol.info.GraphLayout; -import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR; - /** * */ @@ -68,10 +67,12 @@ private void cleanPersistenceDir() throws Exception { if (!F.isEmpty(G.allGrids())) throw new IgniteException("Grids are not stopped"); + SharedFileTree sft = new SharedFileTree(U.defaultWorkDirectory()); + U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "cp", false)); - U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false)); - U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "marshaller", false)); - U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "binary_meta", false)); + U.delete(sft.db()); + U.delete(sft.marshaller()); + U.delete(sft.binaryMetaRoot()); } /** */ diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java index d1ddd2ba0ba97..bee314916f876 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java @@ -29,7 +29,6 @@ import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.U; @@ -144,16 +143,7 @@ protected String getCfgUrl() { assertTrue(conn.isClosed()); } - cleanUpWorkingDir(); - } - - /** - * Clean up working directory. - */ - private void cleanUpWorkingDir() throws Exception { - String workDir = U.defaultWorkDirectory(); - - U.delete(U.resolveWorkDirectory(workDir, DataStorageConfiguration.DFLT_MARSHALLER_PATH, false)); + U.delete(sharedFileTree().marshaller()); } /** diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/FoldersReuseCompatibilityTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/FoldersReuseCompatibilityTest.java index 6883c724df945..7a601eddfe561 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/FoldersReuseCompatibilityTest.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/FoldersReuseCompatibilityTest.java @@ -26,13 +26,13 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.compatibility.testframework.junits.SkipTestIfIsJdkNewer; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.MemoryConfiguration; import org.apache.ignite.configuration.MemoryPolicyConfiguration; import org.apache.ignite.configuration.PersistentStoreConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderResolver; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteInClosure; @@ -222,7 +222,10 @@ private void assertNodeIndexesInFolder(Integer... indexes) throws IgniteCheckedE * @throws IgniteCheckedException if IO error occur */ private void assertPdsDirsDefaultExist(String subDirName) throws IgniteCheckedException { - assertDirectoryExist(DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, subDirName); + NodeFileTree ft = nodeFileTree(subDirName); + + assertTrue(ft.binaryMeta().exists() && ft.binaryMeta().isDirectory()); + assertDirectoryExist(PersistentStoreConfiguration.DFLT_WAL_STORE_PATH, subDirName); assertDirectoryExist(PersistentStoreConfiguration.DFLT_WAL_ARCHIVE_PATH, subDirName); assertDirectoryExist(PdsFolderResolver.DB_DEFAULT_FOLDER, subDirName); diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgnitePersistenceCompatibilityAbstractTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgnitePersistenceCompatibilityAbstractTest.java index 2f33876710a3e..32d4f66a19666 100644 --- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgnitePersistenceCompatibilityAbstractTest.java +++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgnitePersistenceCompatibilityAbstractTest.java @@ -25,13 +25,16 @@ import org.apache.ignite.internal.util.typedef.internal.U; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR; +import static org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree.BINARY_METADATA_DIR; +import static org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree.MARSHALLER_DIR; /** * Super class for all persistence compatibility tests. */ public abstract class IgnitePersistenceCompatibilityAbstractTest extends IgniteCompatibilityAbstractTest { /** Persistence directories. */ - private static final List PERSISTENCE_DIRS = Arrays.asList(DFLT_STORE_DIR, "binary_meta", "cp", "marshaller"); + private static final List PERSISTENCE_DIRS + = Arrays.asList(DFLT_STORE_DIR, BINARY_METADATA_DIR, "cp", MARSHALLER_DIR); /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { diff --git a/modules/compress/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCompressionBasicTest.java b/modules/compress/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCompressionBasicTest.java index 0af75e51d29dd..1c9ef05c6c88b 100644 --- a/modules/compress/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCompressionBasicTest.java +++ b/modules/compress/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotCompressionBasicTest.java @@ -44,7 +44,6 @@ import org.apache.ignite.cache.QueryIndex; import org.apache.ignite.cluster.ClusterState; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.DiskPageCompression; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.GridKernalContextImpl; @@ -292,8 +291,8 @@ public void testRestoreNotCompressed_OnGridWithoutCompression() throws Exception U.delete(U.resolveWorkDirectory(dir.toString(), "cp", false)); U.delete(U.resolveWorkDirectory(dir.toString(), DFLT_STORE_DIR, false)); - U.delete(U.resolveWorkDirectory(dir.toString(), DataStorageConfiguration.DFLT_MARSHALLER_PATH, false)); - U.delete(U.resolveWorkDirectory(dir.toString(), DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false)); + U.delete(nodeFileTree(dir.toString()).marshaller()); + U.delete(nodeFileTree(dir.toString()).binaryMetaRoot()); } } catch (IOException e) { diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java index f5bf5598a692b..819d54c6c4e06 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java @@ -167,12 +167,6 @@ public class DataStorageConfiguration implements Serializable { /** Default change data capture directory maximum size. */ public static final long DFLT_CDC_WAL_DIRECTORY_MAX_SIZE = 0; - /** Default path (relative to working directory) of binary metadata folder */ - public static final String DFLT_BINARY_METADATA_PATH = "db/binary_meta"; - - /** Default path (relative to working directory) of marshaller mappings folder */ - public static final String DFLT_MARSHALLER_PATH = "db/marshaller"; - /** Default write throttling enabled. */ public static final boolean DFLT_WRITE_THROTTLING_ENABLED = false; diff --git a/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java b/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java index b290b319bb192..72499b499f887 100644 --- a/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java +++ b/modules/core/src/main/java/org/apache/ignite/dump/DumpReader.java @@ -46,7 +46,6 @@ import org.apache.ignite.spi.encryption.EncryptionSpi; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_MARSHALLER_PATH; import static org.apache.ignite.internal.IgniteKernal.NL; import static org.apache.ignite.internal.IgniteKernal.SITE; import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR; @@ -87,7 +86,7 @@ public DumpReader(DumpReaderConfiguration cfg, IgniteLogger log) { cnsmr.start(); try { - File[] files = new File(cfg.dumpRoot(), DFLT_MARSHALLER_PATH).listFiles(BinaryUtils::notTmpFile); + File[] files = F.first(dump.fileTrees()).marshaller().listFiles(BinaryUtils::notTmpFile); if (files != null) cnsmr.onMappings(CdcMain.typeMappingIterator(files, tm -> true)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java index 2f09c99f0c020..39e8f557d2ac3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerContextImpl.java @@ -37,10 +37,9 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; -import org.apache.ignite.configuration.DataStorageConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap; +import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree; import org.apache.ignite.internal.processors.closure.GridClosureProcessor; import org.apache.ignite.internal.processors.marshaller.MappedName; import org.apache.ignite.internal.processors.marshaller.MappingExchangeResult; @@ -192,8 +191,10 @@ public void onMappingDataReceived(IgniteLogger log, List> mappings, File dir) throws IgniteCheckedException { - MarshallerMappingFileStore writer = new MarshallerMappingFileStore(ctx, - resolveMappingFileStoreWorkDir(dir.getAbsolutePath())); + MarshallerMappingFileStore writer = new MarshallerMappingFileStore( + ctx, + new SharedFileTree(dir).mkdirMarshaller() + ); addPlatformMappings(ctx.log(MarshallerContextImpl.class), mappings, @@ -583,17 +584,14 @@ private static void putAtIndex( * @param transport Transport. */ public void onMarshallerProcessorStarted( - GridKernalContext ctx, - MarshallerMappingTransport transport + GridKernalContext ctx, + MarshallerMappingTransport transport ) throws IgniteCheckedException { assert ctx != null; - IgniteConfiguration cfg = ctx.config(); - String workDir = U.workDirectory(cfg.getWorkDirectory(), cfg.getIgniteHome()); - - fileStore = marshallerMappingFileStoreDir == null ? - new MarshallerMappingFileStore(ctx, resolveMappingFileStoreWorkDir(workDir)) : - new MarshallerMappingFileStore(ctx, marshallerMappingFileStoreDir); + fileStore = marshallerMappingFileStoreDir == null + ? new MarshallerMappingFileStore(ctx, ctx.pdsFolderResolver().fileTree().mkdirMarshaller()) + : new MarshallerMappingFileStore(ctx, marshallerMappingFileStoreDir); this.transport = transport; closProc = ctx.closure(); clientNode = ctx.clientNode(); @@ -604,30 +602,6 @@ public void onMarshallerProcessorStarted( MarshallerUtils.setNodeName(jdkMarsh, ctx.igniteInstanceName()); } - /** - * @param igniteWorkDir Base ignite working directory. - * @return Resolved directory. - */ - public static File resolveMappingFileStoreWorkDir(String igniteWorkDir) { - File dir = mappingFileStoreWorkDir(igniteWorkDir); - - if (!U.mkdirs(dir)) - throw new IgniteException("Could not create directory for marshaller mappings: " + dir); - - return dir; - } - - /** - * @param igniteWorkDir Base ignite working directory. - * @return Work directory for marshaller mappings. - */ - public static File mappingFileStoreWorkDir(String igniteWorkDir) { - if (F.isEmpty(igniteWorkDir)) - throw new IgniteException("Work directory has not been set: " + igniteWorkDir); - - return new File(igniteWorkDir, DataStorageConfiguration.DFLT_MARSHALLER_PATH); - } - /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerMappingFileStore.java b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerMappingFileStore.java index 26bab219fb12d..3b6c5963c1e66 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerMappingFileStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerMappingFileStore.java @@ -29,6 +29,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.binary.BinaryUtils; +import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree; import org.apache.ignite.internal.util.GridStripedLock; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.typedef.internal.U; @@ -223,7 +224,7 @@ private void fixLegacyFolder() throws IgniteCheckedException { File legacyDir = new File( ctx.config().getWorkDirectory(), - "marshaller" + SharedFileTree.MARSHALLER_DIR ); File legacyTmpDir = new File(legacyDir + TMP_SUFFIX); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java b/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java index bdbff9cf5c094..b5271d3383362 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/cdc/CdcMain.java @@ -47,7 +47,6 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.GridLoggerProxy; import org.apache.ignite.internal.IgniteInterruptedCheckedException; -import org.apache.ignite.internal.MarshallerContextImpl; import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.cdc.WalRecordsConsumer.DataEntryIterator; import org.apache.ignite.internal.pagemem.wal.WALIterator; @@ -55,7 +54,7 @@ import org.apache.ignite.internal.pagemem.wal.record.DataRecord; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.GridLocalConfigManager; -import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderResolver; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; @@ -233,11 +232,8 @@ public class CdcMain implements Runnable { /** Database directory. */ private File dbDir; - /** Binary meta directory. */ - private File binaryMeta; - - /** Marshaller directory. */ - private File marshaller; + /** Ignite folders. */ + private NodeFileTree ft; /** Standalone kernal context. */ private StandaloneGridKernalContext kctx; @@ -329,21 +325,15 @@ public void runX() throws Exception { } try (CdcFileLockHolder lock = lockPds()) { - String consIdDir = cdcDir.getName(cdcDir.getNameCount() - 1).toString(); - Files.createDirectories(cdcDir.resolve(STATE_DIR)); - binaryMeta = CacheObjectBinaryProcessorImpl.binaryWorkDir(igniteCfg.getWorkDirectory(), consIdDir); - - marshaller = MarshallerContextImpl.mappingFileStoreWorkDir(igniteCfg.getWorkDirectory()); - if (log.isInfoEnabled()) { log.info("Change Data Capture [dir=" + cdcDir + ']'); - log.info("Ignite node Binary meta [dir=" + binaryMeta + ']'); - log.info("Ignite node Marshaller [dir=" + marshaller + ']'); + log.info("Ignite node Binary meta [dir=" + ft.binaryMeta() + ']'); + log.info("Ignite node Marshaller [dir=" + ft.marshaller() + ']'); } - kctx = startStandaloneKernal(); + startStandaloneKernal(); initMetrics(); @@ -392,8 +382,8 @@ protected CdcConsumerState createState(Path stateDir) { * @return Kernal instance. * @throws IgniteCheckedException If failed. */ - private StandaloneGridKernalContext startStandaloneKernal() throws IgniteCheckedException { - StandaloneGridKernalContext kctx = new StandaloneGridKernalContext(log, binaryMeta, marshaller) { + private void startStandaloneKernal() throws IgniteCheckedException { + kctx = new StandaloneGridKernalContext(log, ft.binaryMeta(), ft.marshaller()) { @Override protected IgniteConfiguration prepareIgniteConfiguration() { IgniteConfiguration cfg = super.prepareIgniteConfiguration(); @@ -428,14 +418,12 @@ private StandaloneGridKernalContext startStandaloneKernal() throws IgniteChecked } mreg = kctx.metric().registry("cdc"); - - return kctx; } /** Initialize metrics. */ private void initMetrics() { - mreg.objectMetric(BINARY_META_DIR, String.class, "Binary meta directory").value(binaryMeta.getAbsolutePath()); - mreg.objectMetric(MARSHALLER_DIR, String.class, "Marshaller directory").value(marshaller.getAbsolutePath()); + mreg.objectMetric(BINARY_META_DIR, String.class, "Binary meta directory").value(ft.binaryMeta().getAbsolutePath()); + mreg.objectMetric(MARSHALLER_DIR, String.class, "Marshaller directory").value(ft.marshaller().getAbsolutePath()); mreg.objectMetric(CDC_DIR, String.class, "CDC directory").value(cdcDir.toFile().getAbsolutePath()); curSegmentIdx = mreg.longMetric(CUR_SEG_IDX, "Current segment index"); @@ -468,6 +456,8 @@ private CdcFileLockHolder lockPds() throws IgniteCheckedException { "[workDir=" + igniteCfg.getWorkDirectory() + ", consistentId=" + igniteCfg.getConsistentId() + ']'); } + ft = new NodeFileTree(igniteCfg, settings.folderName()); + CdcFileLockHolder lock = settings.getLockedFileLockHolder(); if (lock == null) { @@ -567,8 +557,8 @@ private boolean consumeSegment(Path segment) { IgniteWalIteratorFactory.IteratorParametersBuilder builder = new IgniteWalIteratorFactory.IteratorParametersBuilder() .log(log) - .binaryMetadataFileStoreDir(binaryMeta) - .marshallerMappingFileStoreDir(marshaller) + .binaryMetadataFileStoreDir(ft.binaryMeta()) + .marshallerMappingFileStoreDir(ft.marshaller()) .igniteConfigurationModifier((cfg) -> cfg.setPluginProviders(igniteCfg.getPluginProviders())) .keepBinary(cdcCfg.isKeepBinary()) .filesOrDirs(segment.toFile()); @@ -692,7 +682,7 @@ private void updateMetadata() { /** Search for new or changed {@link BinaryType} and notifies the consumer. */ private void updateTypes() { try { - File[] files = binaryMeta.listFiles(); + File[] files = ft.binaryMeta().listFiles(); if (files == null) return; @@ -710,7 +700,7 @@ private void updateTypes() { typesState.put(typeId, lastModified); try { - kctx.cacheObjects().cacheMetadataLocally(binaryMeta, typeId); + kctx.cacheObjects().cacheMetadataLocally(ft.binaryMeta(), typeId); } catch (IgniteCheckedException e) { throw new IgniteException(e); @@ -739,7 +729,7 @@ private void updateTypes() { /** Search for new or changed {@link TypeMapping} and notifies the consumer. */ private void updateMappings() { try { - File[] files = marshaller.listFiles(BinaryUtils::notTmpFile); + File[] files = ft.marshaller().listFiles(BinaryUtils::notTmpFile); if (files == null) return; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java index 0ffafc4b43280..d2fbc8c804125 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java @@ -30,6 +30,7 @@ import org.apache.ignite.IgniteLogger; import org.apache.ignite.binary.BinaryType; import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.failure.FailureContext; import org.apache.ignite.failure.FailureType; import org.apache.ignite.internal.GridKernalContext; @@ -82,7 +83,7 @@ class BinaryMetadataFileStore { * @param metadataLocCache Metadata locale cache. * @param ctx Context. * @param log Logger. - * @param binaryMetadataFileStoreDir Path to binary metadata store configured by user, should include binary_meta + * @param metadataDir Path to binary metadata store configured by user, should include binary_meta * and consistentId. * @param forceEnabled If {@code true} then will write files even if persistence and CDC disabled. */ @@ -90,13 +91,13 @@ class BinaryMetadataFileStore { final ConcurrentMap metadataLocCache, final GridKernalContext ctx, final IgniteLogger log, - final File binaryMetadataFileStoreDir, + final File metadataDir, final boolean forceEnabled ) throws IgniteCheckedException { this.metadataLocCache = metadataLocCache; this.ctx = ctx; - enabled = forceEnabled || CU.isPersistenceEnabled(ctx.config()) || CU.isCdcEnabled(ctx.config()); + enabled = forceEnabled || enabled(ctx.config()); this.log = log; @@ -107,19 +108,9 @@ class BinaryMetadataFileStore { fileIOFactory = dsCfg == null ? new DataStorageConfiguration().getFileIOFactory() : dsCfg.getFileIOFactory(); - final String nodeFolderName = ctx.pdsFolderResolver().resolveFolders().folderName(); + this.metadataDir = metadataDir; - if (binaryMetadataFileStoreDir != null) - metadataDir = binaryMetadataFileStoreDir; - else { - metadataDir = new File(U.resolveWorkDirectory( - ctx.config().getWorkDirectory(), - DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, - false - ), nodeFolderName); - } - - fixLegacyFolder(nodeFolderName); + fixLegacyFolder(ctx.pdsFolderResolver().resolveFolders().folderName()); } /** @@ -395,6 +386,11 @@ void prepareMetadataRemove(int typeId) { writer.prepareRemoveFuture(typeId); } + /** @return {@code True} if file store enabled. */ + public static boolean enabled(IgniteConfiguration cfg) { + return CU.isPersistenceEnabled(cfg) || CU.isCdcEnabled(cfg); + } + /** * */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java index 1b582d3fcd998..a54bdfd3bb04e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java @@ -21,7 +21,6 @@ import java.io.Serializable; import java.math.BigDecimal; import java.nio.ByteBuffer; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -50,7 +49,6 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.BinaryConfiguration; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; @@ -91,6 +89,7 @@ import org.apache.ignite.internal.processors.cache.IncompleteCacheObject; import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; import org.apache.ignite.internal.processors.cacheobject.UserCacheObjectByteArrayImpl; @@ -222,46 +221,14 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) { metadataLocCache.values(), val -> new BinaryMetadataView(val.metadata())); } - /** - * @param igniteWorkDir Basic ignite working directory. - * @param consId Node consistent id. - * @return Working directory. - */ - public static File resolveBinaryWorkDir(String igniteWorkDir, String consId) { - File workDir = binaryWorkDir(igniteWorkDir, consId); - - if (!U.mkdirs(workDir)) - throw new IgniteException("Could not create directory for binary metadata: " + workDir); - - return workDir; - } - - /** - * @param igniteWorkDir Basic ignite working directory. - * @param consId Node consistent id. - * @return Working directory. - */ - public static File binaryWorkDir(String igniteWorkDir, String consId) { - if (F.isEmpty(igniteWorkDir) || F.isEmpty(consId)) { - throw new IgniteException("Work directory or consistent id has not been set " + - "[igniteWorkDir=" + igniteWorkDir + ", consId=" + consId + ']'); - } - - return Paths.get(igniteWorkDir, DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, consId).toFile(); - } - /** {@inheritDoc} */ @Override public void start() throws IgniteCheckedException { if (marsh instanceof BinaryMarshaller) { if (!ctx.clientNode()) { - metadataFileStore = new BinaryMetadataFileStore(metadataLocCache, - ctx, - log, - CU.isPersistenceEnabled(ctx.config()) && binaryMetadataFileStoreDir == null ? - resolveBinaryWorkDir(ctx.config().getWorkDirectory(), - ctx.pdsFolderResolver().resolveFolders().folderName()) : - binaryMetadataFileStoreDir, - false); + if (BinaryMetadataFileStore.enabled(ctx.config()) && binaryMetadataFileStoreDir == null) + binaryMetadataFileStoreDir = ctx.pdsFolderResolver().fileTree().mkdirBinaryMeta(); + + metadataFileStore = new BinaryMetadataFileStore(metadataLocCache, ctx, log, binaryMetadataFileStoreDir, false); metadataFileStore.start(); } @@ -1016,8 +983,7 @@ public BinaryMetadata binaryMetadata(int typeId) throws BinaryObjectException { BinaryMetadataFileStore writer = new BinaryMetadataFileStore(new ConcurrentHashMap<>(), ctx, log, - resolveBinaryWorkDir(dir.getAbsolutePath(), - ctx.pdsFolderResolver().resolveFolders().folderName()), + new NodeFileTree(dir, ctx.pdsFolderResolver().resolveFolders().folderName()).mkdirBinaryMeta(), true ); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java new file mode 100644 index 0000000000000..2cab984c051ca --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.filename; + +import java.io.File; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.util.typedef.internal.A; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; + +/** + * Provides access to Ignite node file tree. + * Note, that root path can be different for each usage: + *

    + *
  • Ignite node.
    • + *
  • Snapshot files.
    • + *
  • Cache dump files.
    • + *
  • CDC.
    • + *
+ * + * Ignite node file tree structure with the point to currenlty supported dirs. + * Description:
+ *
    + *
  • {@code .} folder is {@code root} constructor parameter.
    • + *
  • {@code node00-e57e62a9-2ccf-4e1b-a11e-c24c21b9ed4c} is the {@link PdsFolderSettings#folderName()} and {@code folderName} + * constructor parameter.
    • + *
+ * + * 
+ * ❯ tree
+ * .                                                                            ← root (work directory, shared between all local nodes).
+ * ├── cp
+ * │  └── sharedfs
+ * │      └── BinaryMarshaller
+ * ├── db                                                                       ← db (shared between all local nodes).
+ * │  ├── binary_meta                                                           ← binaryMetaRoot (shared between all local nodes).
+ * │  │  └── node00-e57e62a9-2ccf-4e1b-a11e-c24c21b9ed4c                        ← binaryMeta for node 0
+ * │  │      └── 1645778359.bin
+ * │  │  └── node01-e57e62a9-2ccf-4e1b-a11e-d35d32c0fe5d                        ← binaryMeta for node 1
+ * │  │      └── 1645778359.bin
+ * │  ├── lock
+ * │  ├── marshaller                                                            ← marshaller (shared between all local nodes)
+ * │  │  └── 1645778359.classname0
+ * │  ├── node00-e57e62a9-2ccf-4e1b-a11e-c24c21b9ed4c                           ← folderName (node 0).
+ * │  │  ├── cache-default
+ * │  │  │  ├── cache_data.dat
+ * │  │  │  ├── index.bin
+ * │  │  │  ├── part-0.bin
+ * │  │  │  ├── part-1.bin
+ * ...
+ * │  │  │  └── part-9.bin
+ * │  │  ├── cache-ignite-sys-cache
+ * │  │  │  ├── cache_data.dat
+ * │  │  │  └── index.bin
+ * │  │  ├── cache-tx-cache
+ * │  │  │  ├── cache_data.dat
+ * │  │  │  ├── index.bin
+ * │  │  │  ├── part-0.bin
+ * │  │  │  ├── part-1.bin
+ * ...
+ * │  │  │  └── part-9.bin
+ * │  │  ├── cp
+ * │  │  │  ├── 1737804007693-96128bb0-5361-495a-b593-53dc4339a56d-END.bin
+ * │  │  │  └── 1737804007693-96128bb0-5361-495a-b593-53dc4339a56d-START.bin
+ * │  │  ├── lock
+ * │  │  ├── maintenance_tasks.mntc
+ * │  │  ├── metastorage
+ * │  │  │  ├── part-0.bin
+ * │  │  │  └── part-1.bin
+ * │  │  └── snp                                                                ← snpTmp (node 0)
+ * │  ├── node01-e57e62a9-2ccf-4e1b-a11e-d35d32c0fe5d                           ← folderName (node 1).
+ * │  │  ├── cache-default
+ * ..
+ * │  │  ├── cache-ignite-sys-cache
+ * ...
+ * │  │  ├── cache-tx-cache
+ * ...
+ * │  │  ├── cp
+ * ...
+ * │  │  ├── lock
+ * │  │  ├── maintenance_tasks.mntc
+ * │  │  ├── metastorage
+ * ...
+ * │  │  └── snp                                                                ← snpTmp (node 1)
+ * ...
+ * ...
+ * │  └── wal
+ * │      ├── archive
+ * │      │  └── ignite_0
+ * │      │      └── node00-e57e62a9-2ccf-4e1b-a11e-c24c21b9ed4c                ← walArchive (node 0)
+ * │      │      └── node01-e57e62a9-2ccf-4e1b-a11e-d35d32c0fe5d                ← walArchive (node 1)
+ * │      ├── cdc
+ * │      │  └── node00-e57e62a9-2ccf-4e1b-a11e-c24c21b9ed4c                    ← walCdc (node 0)
+ * │      │      ├── lock
+ * │      │      └── state
+ * │      │          ├── cdc-caches-state.bin
+ * │      │          ├── cdc-mappings-state.bin
+ * │      │          └── cdc-types-state.bin
+ *
+ * │      │  └── node01-e57e62a9-2ccf-4e1b-a11e-d35d32c0fe5d                    ← walCdc (node 1)
+ * │      └── node00-e57e62a9-2ccf-4e1b-a11e-c24c21b9ed4c                       ← wal (node 0)
+ * │          ├── 0000000000000000.wal
+ * │          ├── 0000000000000001.wal
+ * ...
+ * │          └── 0000000000000009.wal
+ * │      └── node01-e57e62a9-2ccf-4e1b-a11e-d35d32c0fe5d                       ← wal (node 1)
+ * ...
+ * ├── diagnostic
+ * ├── log
+ * │  ├── all.log
+ * │  ├── consistency.log
+ * │  ├── filtered.log
+ * │  ├── ignite-e10fbb91.0.log
+ * │  ├── ignite.log
+ * │  ├── jmx-invoker.0.log
+ * ...
+ * │  └── jmx-invoker.9.log
+ * └── snapshots                                                                ← snapshotRoot (shared between all local nodes).
+ *
+ */ +public class NodeFileTree extends SharedFileTree { + /** Folder name for consistent id. */ + private final String folderName; + + /** Path to the directory containing binary metadata. */ + private final File binaryMeta; + + /** + * Root directory can be Ignite work directory or snapshot root, see {@link U#workDirectory(String, String)} and other methods. + * + * @param root Root directory. + * @param folderName Name of the folder for current node. + * Usually, it a {@link IgniteConfiguration#getConsistentId()} masked to be correct file name. + * + * @see IgniteConfiguration#getWorkDirectory() + * @see IgniteConfiguration#setWorkDirectory(String) + * @see U#workDirectory(String, String) + * @see U#resolveWorkDirectory(String, String, boolean, boolean) + * @see U#IGNITE_WORK_DIR + */ + public NodeFileTree(String root, String folderName) { + this(new File(root), folderName); + } + + /** + * Root directory can be Ignite work directory or snapshot root, see {@link U#workDirectory(String, String)} and other methods. + * + * @param root Root directory. + * @param folderName Name of the folder for current node. + * Usually, it a {@link IgniteConfiguration#getConsistentId()} masked to be correct file name. + * + * @see IgniteConfiguration#getWorkDirectory() + * @see IgniteConfiguration#setWorkDirectory(String) + * @see U#workDirectory(String, String) + * @see U#resolveWorkDirectory(String, String, boolean, boolean) + * @see U#IGNITE_WORK_DIR + */ + public NodeFileTree(File root, String folderName) { + super(root); + + A.notNullOrEmpty(folderName, "Node directory"); + + this.folderName = folderName; + + binaryMeta = new File(binaryMetaRoot.getAbsolutePath(), folderName); + } + + /** + * Creates instance based on config and folder name. + * + * @param cfg Ignite configuration to get parameter from. + * @param folderName Name of the folder for current node. + * Usually, it a {@link IgniteConfiguration#getConsistentId()} masked to be correct file name. + * + * @see IgniteConfiguration#getWorkDirectory() + * @see IgniteConfiguration#setWorkDirectory(String) + * @see U#workDirectory(String, String) + * @see U#resolveWorkDirectory(String, String, boolean, boolean) + * @see U#IGNITE_WORK_DIR + */ + public NodeFileTree(IgniteConfiguration cfg, String folderName) { + super(cfg); + + A.notNullOrEmpty(folderName, "Node directory"); + + this.folderName = folderName; + + binaryMeta = new File(binaryMetaRoot, folderName); + } + + /** @return Folder name. */ + public String folderName() { + return folderName; + } + + /** @return Path to binary metadata directory. */ + public File binaryMeta() { + return binaryMeta; + } + + /** + * Creates {@link #binaryMeta()} directory. + * @return Created directory. + * @see #binaryMeta() + */ + public File mkdirBinaryMeta() { + return mkdir(binaryMeta, "binary metadata"); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(NodeFileTree.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java index f6a9752ffe4ae..142db5f3ddd44 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java @@ -19,6 +19,7 @@ import java.io.File; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.processors.GridProcessorAdapter; @@ -36,7 +37,10 @@ public class PdsConsistentIdProcessor extends GridProcessorAdapter implements Pd private final GridKernalContext ctx; /** Cached folder settings. */ - private PdsFolderSettings settings; + private volatile PdsFolderSettings settings; + + /** Cached Ignite directories. */ + private volatile NodeFileTree ft; /** * Creates folders resolver @@ -52,25 +56,70 @@ public PdsConsistentIdProcessor(final GridKernalContext ctx) { /** {@inheritDoc} */ @Override public PdsFolderSettings resolveFolders() throws IgniteCheckedException { - if (settings == null) { - //here deprecated method is used to get compatible version of consistentId - PdsFolderResolver resolver = - new PdsFolderResolver<>(ctx.config(), log, ctx.discovery().consistentId(), this::tryLock); + if (settings == null) + init(); + + return settings; + } + + /** {@inheritDoc} */ + @Override public NodeFileTree fileTree() { + if (ft == null) { + try { + init(); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } - settings = resolver.resolve(); + return ft; + } - if (settings == null) - settings = resolver.generateNew(); + /** Initialize PDS settings. */ + private synchronized void init() throws IgniteCheckedException { + if (settings != null) + return; - if (!settings.isCompatible()) { - if (log.isInfoEnabled()) - log.info("Consistent ID used for local node is [" + settings.consistentId() + "] " + - "according to persistence data storage folders"); + initSettings(); + initFileTree(); + } - ctx.discovery().consistentId(settings.consistentId()); - } + /** Initialize PDS settings. */ + private void initSettings() throws IgniteCheckedException { + assert settings == null; + assert ft == null; + + //here deprecated method is used to get compatible version of consistentId + PdsFolderResolver resolver = + new PdsFolderResolver<>(ctx.config(), log, ctx.discovery().consistentId(), this::tryLock); + + settings = resolver.resolve(); + + if (settings == null) + settings = resolver.generateNew(); + + if (!settings.isCompatible()) { + if (log.isInfoEnabled()) + log.info("Consistent ID used for local node is [" + settings.consistentId() + "] " + + "according to persistence data storage folders"); + + ctx.discovery().consistentId(settings.consistentId()); } - return settings; + } + + /** Initialize file tree. */ + private void initFileTree() throws IgniteCheckedException { + assert ft == null; + + if (ctx.clientNode()) { + ft = new NodeFileTree( + U.workDirectory(ctx.config().getWorkDirectory(), ctx.config().getIgniteHome()), + resolveFolders().folderName() + ); + } + else + ft = new NodeFileTree(ctx.config(), resolveFolders().folderName()); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsFolderResolver.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsFolderResolver.java index 0d3ec4b624ff3..ed762bdaa17c1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsFolderResolver.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsFolderResolver.java @@ -83,11 +83,10 @@ public class PdsFolderResolver { /** Database subfolders for old style filter. */ private static final FileFilter DB_SUBFOLDERS_OLD_STYLE_FILTER = new FileFilter() { @Override public boolean accept(File pathname) { - String path = pathname.toString(); return pathname.isDirectory() && !"wal".equals(pathname.getName()) - && !path.contains(DataStorageConfiguration.DFLT_BINARY_METADATA_PATH) - && !path.contains(DataStorageConfiguration.DFLT_MARSHALLER_PATH) + && !NodeFileTree.containsBinaryMetaPath(pathname) + && !NodeFileTree.containsMarshaller(pathname) && !pathname.getName().matches(SUBDIR_PATTERN); } }; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsFoldersResolver.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsFoldersResolver.java index 6122e2893646d..bf563a91c32df 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsFoldersResolver.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsFoldersResolver.java @@ -31,4 +31,9 @@ public interface PdsFoldersResolver { * @throws IgniteCheckedException if failed. */ public PdsFolderSettings resolveFolders() throws IgniteCheckedException; + + /** + * @return Ignite node file tree. + */ + public NodeFileTree fileTree(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java new file mode 100644 index 0000000000000..a4ebcf7c4258c --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.filename; + +import java.io.File; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.util.typedef.internal.A; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; + +import static org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderResolver.DB_DEFAULT_FOLDER; + +/** + * Provides access to directories shared between all local nodes. + * 
+ * ❯ tree
+ * .                                                                            ← root (work directory, shared between all local nodes).
+ * ├── db                                                                       ← db (shared between all local nodes).
+ * │  ├── binary_meta                                                           ← binaryMetaRoot (shared between all local nodes).
+ * │  ├── marshaller                                                            ← marshaller (shared between all local nodes).
+ * └── snapshots                                                                ← snapshotRoot (shared between all local nodes).
+ *
+ * + * @see NodeFileTree + */ +public class SharedFileTree { + /** Name of binary metadata folder. */ + public static final String BINARY_METADATA_DIR = "binary_meta"; + + /** Name of marshaller mappings folder. */ + public static final String MARSHALLER_DIR = "marshaller"; + + /** Root(work) directory. */ + protected final File root; + + /** db directory. */ + protected final File db; + + /** Path to the directory containing binary metadata. */ + protected final File binaryMetaRoot; + + /** Path to the directory containing marshaller files. */ + protected final File marshaller; + + /** + * @param root Root directory. + */ + public SharedFileTree(File root) { + A.notNull(root, "Root directory"); + + this.root = root; + db = new File(root, DB_DEFAULT_FOLDER); + marshaller = new File(db, MARSHALLER_DIR); + binaryMetaRoot = new File(db, BINARY_METADATA_DIR); + } + + /** + * @param root Root directory. + */ + public SharedFileTree(String root) { + this(new File(root)); + } + + /** + * @param cfg Config to get {@code root} directory from. + */ + SharedFileTree(IgniteConfiguration cfg) { + this(root(cfg)); + } + + /** + * @return Path to the {@code root} directory. + */ + public File root() { + return root; + } + + /** + * @return Path to the {@code db} directory. + */ + public File db() { + return db; + } + + /** + * @return Path to common binary metadata directory. Note, directory can contain data from several nodes. + * Each node will create own directory inside this root. + */ + public File binaryMetaRoot() { + return binaryMetaRoot; + } + + /** @return Path to marshaller directory. */ + public File marshaller() { + return marshaller; + } + + /** + * Creates {@link #binaryMetaRoot()} directory. + * @return Created directory. + * @see SharedFileTree#binaryMetaRoot() + */ + public File mkdirBinaryMetaRoot() { + return mkdir(binaryMetaRoot, "root binary metadata"); + } + + /** + * Creates {@link #marshaller()} directory. + * @return Created directory. + * @see #marshaller() + */ + public File mkdirMarshaller() { + return mkdir(marshaller, "marshaller mappings"); + } + + /** + * @param f File to check. + * @return {@code True} if argument can be binary meta root directory. + */ + public static boolean binaryMetaRoot(File f) { + return f.getAbsolutePath().endsWith(BINARY_METADATA_DIR); + } + + /** + * @param f File to check. + * @return {@code True} if f ends with binary meta root directory. + */ + public static boolean marshaller(File f) { + return f.getAbsolutePath().endsWith(MARSHALLER_DIR); + } + + /** + * @param file File to check. + * @return {@code True} if {@code f} contains binary meta root directory. + */ + public static boolean containsBinaryMetaPath(File file) { + return file.getPath().contains(BINARY_METADATA_DIR); + } + + /** + * @param f File to check. + * @return {@code True} if {@code f} contains marshaller directory. + */ + public static boolean containsMarshaller(File f) { + return f.getAbsolutePath().contains(MARSHALLER_DIR); + } + + /** + * @param dir Directory to create. + */ + public static File mkdir(File dir, String name) { + if (!U.mkdirs(dir)) + throw new IgniteException("Could not create directory for " + name + ": " + dir); + + if (!dir.canRead()) + throw new IgniteException("Cannot read from directory: " + dir); + + if (!dir.canWrite()) + throw new IgniteException("Cannot write to directory: " + dir); + + return dir; + } + + /** + * @param cfg Ignite config. + * @return Root directory. + */ + private static File root(IgniteConfiguration cfg) { + try { + return new File(U.workDirectory(cfg.getWorkDirectory(), cfg.getIgniteHome())); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(SharedFileTree.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 21890c4df8ca6..a500d35171ce9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -134,6 +134,7 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener; @@ -192,8 +193,6 @@ import static java.nio.file.StandardOpenOption.READ; import static org.apache.ignite.IgniteSystemProperties.IGNITE_SNAPSHOT_SEQUENTIAL_WRITE; -import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_BINARY_METADATA_PATH; -import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_MARSHALLER_PATH; import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_PATH; import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_FAILED; import static org.apache.ignite.events.EventType.EVT_CLUSTER_SNAPSHOT_FINISHED; @@ -202,8 +201,6 @@ import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; import static org.apache.ignite.internal.GridClosureCallMode.BALANCE; import static org.apache.ignite.internal.GridClosureCallMode.BROADCAST; -import static org.apache.ignite.internal.MarshallerContextImpl.mappingFileStoreWorkDir; -import static org.apache.ignite.internal.MarshallerContextImpl.resolveMappingFileStoreWorkDir; import static org.apache.ignite.internal.MarshallerContextImpl.saveMappings; import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT; import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL; @@ -216,8 +213,6 @@ import static org.apache.ignite.internal.pagemem.PageIdUtils.toDetailString; import static org.apache.ignite.internal.processors.cache.GridCacheUtils.baselineNode; import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isPersistenceEnabled; -import static org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.binaryWorkDir; -import static org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.resolveBinaryWorkDir; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_TEMPLATE; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheDirectories; @@ -775,28 +770,20 @@ public void deleteSnapshot(File snpDir, PdsFolderSettings pdsSettings) { String folderName = pdsSettings.folderName(); try { - File binDir = binaryWorkDir(snpDir.getAbsolutePath(), folderName); + NodeFileTree snpFt = new NodeFileTree(snpDir.getAbsolutePath(), folderName); + File nodeDbDir = new File(snpDir.getAbsolutePath(), databaseRelativePath(folderName)); File smf = new File(snpDir, snapshotMetaFileName(U.maskForFileName(pdsSettings.consistentId().toString()))); - U.delete(binDir); + U.delete(snpFt.binaryMeta()); U.delete(nodeDbDir); U.delete(smf); - File marshDir = mappingFileStoreWorkDir(snpDir.getAbsolutePath()); - - deleteDirectory(marshDir); - - File binMetadataDfltDir = new File(snpDir, DFLT_BINARY_METADATA_PATH); - File marshallerDfltDir = new File(snpDir, DFLT_MARSHALLER_PATH); + deleteDirectory(snpFt.binaryMetaRoot()); + deleteDirectory(snpFt.marshaller()); - deleteDirectory(binMetadataDfltDir); - deleteDirectory(marshallerDfltDir); - - File db = new File(snpDir, DB_DEFAULT_FOLDER); - - db.delete(); - snpDir.delete(); + snpFt.db().delete(); + snpFt.root().delete(); } catch (IOException e) { throw new IgniteException(e); @@ -2574,8 +2561,9 @@ public StandaloneGridKernalContext createStandaloneKernalContext( File snpDir, String folderName ) throws IgniteCheckedException { - return new StandaloneGridKernalContext(log, cmpProc, resolveBinaryWorkDir(snpDir.getAbsolutePath(), folderName), - resolveMappingFileStoreWorkDir(snpDir.getAbsolutePath())); + NodeFileTree ft = new NodeFileTree(snpDir, folderName); + + return new StandaloneGridKernalContext(log, cmpProc, ft.binaryMeta(), ft.marshaller()); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotFutureTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotFutureTask.java index 6d877b760d3ee..0c7824af0eb57 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotFutureTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotFutureTask.java @@ -30,19 +30,17 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgniteInterruptedCheckedException; -import org.apache.ignite.internal.MarshallerContextImpl; import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.pagemem.wal.record.IncrementalSnapshotFinishRecord; import org.apache.ignite.internal.pagemem.wal.record.delta.ClusterSnapshotRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer; import org.apache.ignite.internal.util.typedef.internal.CU; import org.jetbrains.annotations.Nullable; -import static org.apache.ignite.internal.MarshallerContextImpl.mappingFileStoreWorkDir; import static org.apache.ignite.internal.binary.BinaryUtils.METADATA_FILE_SUFFIX; -import static org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.binaryWorkDir; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.incrementalSnapshotWalsDir; /** */ @@ -138,15 +136,18 @@ public IncrementalSnapshotFutureTask( copyWal(incrementalSnapshotWalsDir(incSnpDir, folderName), highPtrFut.result()); + NodeFileTree ft = cctx.kernalContext().pdsFolderResolver().fileTree(); + NodeFileTree snpFt = new NodeFileTree(incSnpDir, folderName); + copyFiles( - MarshallerContextImpl.mappingFileStoreWorkDir(cctx.gridConfig().getWorkDirectory()), - mappingFileStoreWorkDir(incSnpDir.getAbsolutePath()), + ft.marshaller(), + snpFt.marshaller(), BinaryUtils::notTmpFile ); copyFiles( - binaryWorkDir(cctx.gridConfig().getWorkDirectory(), folderName), - binaryWorkDir(incSnpDir.getAbsolutePath(), folderName), + ft.binaryMeta(), + snpFt.binaryMeta(), file -> file.getName().endsWith(METADATA_FILE_SUFFIX) ); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 0cff2a5f04462..0132058d31e67 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -79,6 +79,7 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.file.FileVersionCheckingFactory; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.ClusterSnapshotFuture; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; @@ -98,9 +99,7 @@ import org.jetbrains.annotations.Nullable; import static java.util.Optional.ofNullable; -import static org.apache.ignite.internal.MarshallerContextImpl.mappingFileStoreWorkDir; import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION; -import static org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.binaryWorkDir; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_GRP_DIR_PREFIX; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheGroupName; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.partId; @@ -965,12 +964,11 @@ private IgniteInternalFuture preload(UUID reqId) { .incrementalSnapshotLocalDir(opCtx0.snpName, opCtx0.snpPath, opCtx0.incIdx) : snpDir; - File binDir = binaryWorkDir(dir.getAbsolutePath(), meta.folderName()); - File marshallerDir = mappingFileStoreWorkDir(dir.getAbsolutePath()); + NodeFileTree ft = new NodeFileTree(dir, meta.folderName()); - ctx.cacheObjects().updateMetadata(binDir, opCtx0.stopChecker); + ctx.cacheObjects().updateMetadata(ft.binaryMeta(), opCtx0.stopChecker); - restoreMappings(marshallerDir, opCtx0.stopChecker); + restoreMappings(ft.marshaller(), opCtx0.stopChecker); } catch (Throwable t) { log.error("Unable to perform metadata update operation for the cache groups restore process", t); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java index cd5278ae97702..fb715571cc2f9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/dump/Dump.java @@ -46,12 +46,12 @@ import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.StoredCacheData; -import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl; import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator; import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIO; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadata; import org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneGridKernalContext; import org.apache.ignite.internal.util.typedef.F; @@ -63,8 +63,6 @@ import org.jetbrains.annotations.Nullable; import static java.nio.file.StandardOpenOption.READ; -import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_BINARY_METADATA_PATH; -import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_MARSHALLER_PATH; import static org.apache.ignite.internal.processors.cache.GridLocalConfigManager.readCacheData; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_DIR_PREFIX; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_GRP_DIR_PREFIX; @@ -86,6 +84,9 @@ public class Dump implements AutoCloseable { /** Dump directory. */ private final File dumpDir; + /** Dump directories. */ + private final List fts; + /** Specific consistent id. */ private final @Nullable String consistentId; @@ -144,8 +145,11 @@ public Dump( this.dumpDir = dumpDir; this.consistentId = consistentId == null ? null : U.maskForFileName(consistentId); this.metadata = metadata(dumpDir, this.consistentId); + this.fts = metadata.stream() + .map(m -> new NodeFileTree(dumpDir, m.folderName())) + .collect(Collectors.toList()); this.keepBinary = keepBinary; - this.cctx = standaloneKernalContext(dumpDir, log); + this.cctx = standaloneKernalContext(log); this.raw = raw; this.encSpi = encSpi; this.comprParts = metadata.get(0).compressPartitions(); @@ -157,19 +161,15 @@ public Dump( } /** - * @param dumpDir Dump directory. * @param log Logger. * @return Standalone kernal context. */ - private GridKernalContext standaloneKernalContext(File dumpDir, IgniteLogger log) { - File binaryMeta = CacheObjectBinaryProcessorImpl.binaryWorkDir(dumpDir.getAbsolutePath(), F.first(metadata).folderName()); - File marshaller = new File(dumpDir, DFLT_MARSHALLER_PATH); - - A.ensure(binaryMeta.exists(), "binary metadata directory not exists"); - A.ensure(marshaller.exists(), "marshaller directory not exists"); + private GridKernalContext standaloneKernalContext(IgniteLogger log) { + A.ensure(F.first(fts).binaryMeta().exists(), "binary metadata directory not exists"); + A.ensure(F.first(fts).marshaller().exists(), "marshaller directory not exists"); try { - GridKernalContext kctx = new StandaloneGridKernalContext(log, binaryMeta, marshaller); + GridKernalContext kctx = new StandaloneGridKernalContext(log, F.first(fts).binaryMeta(), F.first(fts).marshaller()); startAllComponents(kctx); @@ -188,7 +188,7 @@ public Iterator types() { /** @return List of node directories. */ public List nodesDirectories() { File[] dirs = new File(dumpDir, DFLT_STORE_DIR).listFiles(f -> f.isDirectory() - && !(f.getAbsolutePath().endsWith(DFLT_BINARY_METADATA_PATH) || f.getAbsolutePath().endsWith(DFLT_MARSHALLER_PATH)) + && !(NodeFileTree.binaryMetaRoot(f) || NodeFileTree.marshaller(f)) && (consistentId == null || U.maskForFileName(f.getName()).contains(consistentId))); if (dirs == null) @@ -360,6 +360,11 @@ public File dumpDirectory() { return dumpDir; } + /** @return Dump directories. */ + public List fileTrees() { + return fts; + } + /** */ private File dumpGroupDirectory(String node, int grpId) { File nodeDir = Paths.get(dumpDir.getAbsolutePath(), DFLT_STORE_DIR, node).toFile(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java index a6956db22b37e..c779f4aba7158 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java @@ -58,8 +58,10 @@ import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl; import org.apache.ignite.internal.processors.cache.persistence.defragmentation.IgniteDefragmentation; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver; +import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree; import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; import org.apache.ignite.internal.processors.closure.GridClosureProcessor; import org.apache.ignite.internal.processors.cluster.ClusterProcessor; @@ -210,7 +212,7 @@ public StandaloneGridKernalContext( // Fake folder provided to perform processor startup on empty folder. if (binaryMetadataFileStoreDir == null) - binaryMetadataFileStoreDir = new File(DataStorageConfiguration.DFLT_BINARY_METADATA_PATH).getAbsoluteFile(); + binaryMetadataFileStoreDir = new SharedFileTree(new File(".")).binaryMetaRoot().getAbsoluteFile(); cacheObjProcessor = binaryProcessor(this, binaryMetadataFileStoreDir); @@ -697,6 +699,11 @@ private void setField(IgniteEx kernal, String name, Object val) throws NoSuchFie @Override public PdsFolderSettings resolveFolders() { return new PdsFolderSettings(new File("."), U.maskForFileName("")); } + + /** {@inheritDoc} */ + @Override public NodeFileTree fileTree() { + return new NodeFileTree(new File("."), resolveFolders().folderName()); + } }; } diff --git a/modules/core/src/test/java/org/apache/ignite/client/ReliabilityTest.java b/modules/core/src/test/java/org/apache/ignite/client/ReliabilityTest.java index b019583c0e9d3..506ebd33f64af 100644 --- a/modules/core/src/test/java/org/apache/ignite/client/ReliabilityTest.java +++ b/modules/core/src/test/java/org/apache/ignite/client/ReliabilityTest.java @@ -45,7 +45,6 @@ import org.apache.ignite.cache.query.ScanQuery; import org.apache.ignite.configuration.ClientConfiguration; import org.apache.ignite.configuration.ClientConnectorConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.failure.FailureHandler; import org.apache.ignite.internal.client.thin.AbstractThinClientTest; @@ -620,10 +619,7 @@ public void testServiceMethodInvocationAfterFailover() throws Exception { // Kill the cluster node, clean up the working directory (with cached types) // and drop the client connection. ignite.close(); - U.delete(U.resolveWorkDirectory( - U.defaultWorkDirectory(), - DataStorageConfiguration.DFLT_MARSHALLER_PATH, - false)); + U.delete(sharedFileTree().marshaller()); dropAllThinClientConnections(); // Invoke the service. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ConcurrentMappingFileReadWriteTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ConcurrentMappingFileReadWriteTest.java index 3fe71cfcc33df..98760ea9a1c14 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/ConcurrentMappingFileReadWriteTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/ConcurrentMappingFileReadWriteTest.java @@ -33,7 +33,6 @@ import org.junit.Test; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_MARSHALLER_PATH; /** * Tests concurrent read/write operations for {@code org.apache.ignite.internal.MarshallerMappingFileStore}. @@ -53,8 +52,7 @@ public class ConcurrentMappingFileReadWriteTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { - mappingDir = new File(U.workDirectory(null, null) + DFLT_MARSHALLER_PATH); - mappingDir.mkdirs(); + mappingDir = sharedFileTree().mkdirMarshaller(); mappingFileStore = new MarshallerMappingFileStore( new StandaloneGridKernalContext(log, null, null), diff --git a/modules/core/src/test/java/org/apache/ignite/internal/MarshallerContextLockingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/MarshallerContextLockingSelfTest.java index d5e6ac61bf0d1..337a07200bea0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/MarshallerContextLockingSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/MarshallerContextLockingSelfTest.java @@ -27,11 +27,13 @@ import org.apache.ignite.IgniteInterruptedException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree; import org.apache.ignite.internal.processors.closure.GridClosureProcessor; import org.apache.ignite.internal.processors.marshaller.MarshallerMappingItem; import org.apache.ignite.internal.processors.marshaller.MarshallerMappingTransport; import org.apache.ignite.internal.processors.pool.PoolProcessor; import org.apache.ignite.internal.util.lang.GridPlainRunnable; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestClassLoader; import org.apache.ignite.testframework.junits.GridTestKernalContext; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; @@ -147,6 +149,8 @@ public void executeTest(GridTestLog4jLogger log, GridKernalContext ctx) throws E counter.incrementAndGet(); MarshallerContextImpl marshallerCtx = new MarshallerContextImpl(null, null); + + marshallerCtx.setMarshallerMappingFileStoreDir(new SharedFileTree(U.defaultWorkDirectory()).marshaller()); marshallerCtx.onMarshallerProcessorStarted(ctx, null); MarshallerMappingItem item = new MarshallerMappingItem(JAVA_ID, 1, String.class.getName()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java index 93dda57a99996..b110829015f03 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java @@ -4176,6 +4176,7 @@ protected BinaryMarshaller binaryMarshaller( kernCtx.add(new GridSystemViewManager(kernCtx)); kernCtx.add(new GridDiscoveryManager(kernCtx)); + marshCtx.setMarshallerMappingFileStoreDir(sharedFileTree().marshaller()); marshCtx.onMarshallerProcessorStarted(kernCtx, null); marsh.setContext(marshCtx); @@ -4236,6 +4237,7 @@ protected BinaryObjectBuilder builder( kernCtx.add(new GridSystemViewManager(kernCtx)); kernCtx.add(new GridDiscoveryManager(kernCtx)); + marshCtx.setMarshallerMappingFileStoreDir(sharedFileTree().marshaller()); marshCtx.onMarshallerProcessorStarted(kernCtx, null); marsh.setContext(marshCtx); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerEnumSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerEnumSelfTest.java index 9f620128c3e85..56324ba6c1518 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerEnumSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerEnumSelfTest.java @@ -21,6 +21,8 @@ import org.apache.ignite.IgniteLogger; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.marshaller.MarshallerContextTestImpl; import org.apache.ignite.testframework.junits.GridTestKernalContext; import org.apache.ignite.testframework.junits.logger.GridTestLog4jLogger; @@ -47,6 +49,7 @@ public void testEnumSerialisation() throws Exception { MarshallerContextTestImpl ctx = new MarshallerContextTestImpl(); + ctx.setMarshallerMappingFileStoreDir(new SharedFileTree(U.defaultWorkDirectory()).marshaller()); ctx.onMarshallerProcessorStarted(newContext(), null); marsh.setContext(ctx); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClientRequestsMappingOnMissTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClientRequestsMappingOnMissTest.java index 300fdf9345627..b7735c1a79041 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClientRequestsMappingOnMissTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClientRequestsMappingOnMissTest.java @@ -18,10 +18,7 @@ package org.apache.ignite.internal.processors.cache; import java.io.File; -import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Map; import java.util.UUID; import java.util.concurrent.CountDownLatch; @@ -29,13 +26,13 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.Ignite; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.GridComponent; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.GridTopic; import org.apache.ignite.internal.managers.communication.GridIoManager; import org.apache.ignite.internal.managers.communication.GridMessageListener; +import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.internal.DiscoveryDataPacket; @@ -91,23 +88,11 @@ public class IgniteMarshallerCacheClientRequestsMappingOnMissTest extends GridCo @Override protected void afterTest() throws Exception { stopAllGrids(); - cleanupMarshallerFileStore(); + U.delete(new SharedFileTree(TMP_DIR).marshaller()); mappingReqsCounter.set(0); } - /** - * - */ - private void cleanupMarshallerFileStore() throws IOException { - Path marshCache = Paths.get(TMP_DIR, DataStorageConfiguration.DFLT_MARSHALLER_PATH); - - for (File file : marshCache.toFile().listFiles()) - Files.delete(file.toPath()); - - Files.deleteIfExists(marshCache); - } - /** * @throws Exception If failed. */ @@ -127,7 +112,7 @@ public void testRequestedMappingIsStoredInFS() throws Exception { stopGrid(1); - File[] files = Paths.get(TMP_DIR, DataStorageConfiguration.DFLT_MARSHALLER_PATH).toFile().listFiles(); + File[] files = new SharedFileTree(TMP_DIR).marshaller().listFiles(); assertNotNull(TMP_DIR + "/marshaller directory should contain at least one file", files); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClientRequestsMappingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClientRequestsMappingTest.java index aab0db0aea393..c485ac9441739 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClientRequestsMappingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClientRequestsMappingTest.java @@ -17,12 +17,7 @@ package org.apache.ignite.internal.processors.cache; -import java.io.File; import java.lang.reflect.Constructor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Objects; import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; @@ -32,7 +27,6 @@ import org.apache.ignite.Ignite; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.events.Event; import org.apache.ignite.internal.IgniteEx; @@ -41,6 +35,7 @@ import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper; import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage; import org.apache.ignite.internal.processors.cache.binary.MetadataResponseMessage; +import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree; import org.apache.ignite.internal.processors.marshaller.MappingAcceptedMessage; import org.apache.ignite.internal.processors.marshaller.MappingProposedMessage; import org.apache.ignite.internal.processors.marshaller.MarshallerMappingItem; @@ -57,6 +52,7 @@ import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; + import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.events.EventType.EVTS_CACHE; import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_PUT; @@ -122,12 +118,7 @@ public class IgniteMarshallerCacheClientRequestsMappingTest extends GridCommonAb @Override protected void afterTest() throws Exception { stopAllGrids(); - Path path = Paths.get(clntWorkDir, DataStorageConfiguration.DFLT_MARSHALLER_PATH); - - for (File file : Objects.requireNonNull(path.toFile().listFiles())) - Files.delete(file.toPath()); - - Files.deleteIfExists(path); + U.delete(new SharedFileTree(clntWorkDir).marshaller()); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java index ff7db00e47d87..8dc02770f9d6a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java @@ -21,8 +21,6 @@ import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.nio.file.Paths; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; @@ -32,7 +30,6 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.cluster.ClusterState; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.PersistentStoreConfiguration; import org.apache.ignite.internal.IgniteEx; @@ -105,7 +102,7 @@ private static class SimpleValue { /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { stopAllGrids(); - cleanUpWorkDir(); + U.delete(sharedFileTree().marshaller()); cleanPersistenceDir(); } @@ -114,15 +111,6 @@ private static class SimpleValue { cleanPersistenceDir(); } - /** - * - */ - private void cleanUpWorkDir() throws Exception { - String workDir = U.defaultWorkDirectory(); - - U.delete(U.resolveWorkDirectory(workDir, DataStorageConfiguration.DFLT_MARSHALLER_PATH, false)); - } - /** * Test checks a scenario when in multinode cluster one node may read marshaller mapping * from file storage and add it directly to marshaller context with accepted=true flag, @@ -176,7 +164,7 @@ private void prepareMarshallerFileStore() throws Exception { String fileName = typeId + ".classname0"; - File marshStoreDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_MARSHALLER_PATH, false); + File marshStoreDir = sharedFileTree().mkdirMarshaller(); try (FileOutputStream out = new FileOutputStream(new File(marshStoreDir, fileName))) { try (Writer writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) { @@ -220,9 +208,9 @@ public void testNodeStartFailsOnCorruptedStorage() throws Exception { * Class name for CustomClass class mapping file gets cleaned up from file system. */ private void corruptMarshallerStorage() throws Exception { - Path marshallerDir = Paths.get(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_MARSHALLER_PATH); + File marshallerDir = sharedFileTree().marshaller(); - File[] storedMappingsFiles = marshallerDir.toFile().listFiles(); + File[] storedMappingsFiles = marshallerDir.listFiles(); assert storedMappingsFiles.length == 1; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/MarshallerCacheJobRunNodeRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/MarshallerCacheJobRunNodeRestartTest.java index 3310aa70dd859..9a6beafab288e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/MarshallerCacheJobRunNodeRestartTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/MarshallerCacheJobRunNodeRestartTest.java @@ -20,7 +20,6 @@ import java.io.Serializable; import java.util.concurrent.Callable; import org.apache.ignite.Ignite; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteCallable; @@ -39,7 +38,7 @@ public class MarshallerCacheJobRunNodeRestartTest extends GridCommonAbstractTest @Test public void testJobRun() throws Exception { for (int i = 0; i < 5; i++) { - U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_MARSHALLER_PATH, true); + sharedFileTree().mkdirMarshaller(); log.info("Iteration: " + i); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataInMemoryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataInMemoryTest.java index 635c6b48ac094..939e0f02077c9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataInMemoryTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataInMemoryTest.java @@ -16,11 +16,8 @@ */ package org.apache.ignite.internal.processors.cache.binary; -import java.io.File; -import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; @@ -39,9 +36,6 @@ public void testBinaryMetadataDir() throws Exception { stopGrid(); - File metaDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), - DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false); - - assertTrue(F.isEmpty(metaDir.list())); + assertTrue(F.isEmpty(sharedFileTree().binaryMetaRoot().list())); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataMoveLegacyFolderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataMoveLegacyFolderTest.java index 538b3c681d136..66df272951eeb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataMoveLegacyFolderTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataMoveLegacyFolderTest.java @@ -32,12 +32,15 @@ import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.binary.BinaryMetadata; import org.apache.ignite.internal.binary.BinaryTypeImpl; +import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.After; import org.junit.Before; import org.junit.Test; +import static org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree.MARSHALLER_DIR; + /** * Test for moving binary metadata and marshaller folders to PDS. */ @@ -194,7 +197,7 @@ public void testMarshallerMappingsDirectoryMigration() throws Exception { File legacyDir = U.resolveWorkDirectory( U.defaultWorkDirectory(), - "marshaller", + MARSHALLER_DIR, false ); @@ -211,16 +214,12 @@ public void testMarshallerMappingsDirectoryMigration() throws Exception { // legacy marshaller mappings dir must be deleted at this moment assertFalse(legacyDir.exists()); - File newDir = U.resolveWorkDirectory( - U.defaultWorkDirectory(), - DataStorageConfiguration.DFLT_MARSHALLER_PATH, - false - ); - // assert folder and contents moved to new location - assertTrue(newDir.exists()); + SharedFileTree sft = sharedFileTree(); + + assertTrue(sft.marshaller().exists()); - assertTrue(new File(newDir, typeIdFile).exists()); + assertTrue(new File(sft.marshaller(), typeIdFile).exists()); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridBinaryCacheEntryMemorySizeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridBinaryCacheEntryMemorySizeSelfTest.java index a41b6998ce2d6..6190c755344b6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridBinaryCacheEntryMemorySizeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridBinaryCacheEntryMemorySizeSelfTest.java @@ -61,6 +61,8 @@ public class GridBinaryCacheEntryMemorySizeSelfTest extends GridCacheEntryMemory kernCtx.add(new GridDiscoveryManager(kernCtx)); MarshallerContextTestImpl marshCtx = new MarshallerContextTestImpl(null); + + marshCtx.setMarshallerMappingFileStoreDir(sharedFileTree().marshaller()); marshCtx.onMarshallerProcessorStarted(kernCtx, null); marsh.setContext(marshCtx); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataAsyncWritingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataAsyncWritingTest.java index 1eefddc078163..6ea38391cfb16 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataAsyncWritingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataAsyncWritingTest.java @@ -27,7 +27,6 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; -import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CachePeekMode; @@ -59,6 +58,7 @@ import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC; import static org.apache.ignite.cluster.ClusterState.ACTIVE; +import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.containsBinaryMetaPath; import static org.apache.ignite.testframework.GridTestUtils.suppressException; /** @@ -165,9 +165,14 @@ public void testBinaryMetadataIsRestoredAfterDeletionOnNodeJoin() throws Excepti int key = findAffinityKeyForNode(ig0.affinity(DEFAULT_CACHE_NAME), ig1.localNode()); cache.put(key, new TestAddress(0, "USA", "NYC", "Park Ave")); - String ig1ConsId = ig1.localNode().consistentId().toString(); + File ig1BinaryMeta = ig1.context().pdsFolderResolver().fileTree().binaryMeta(); + + assertTrue(ig1BinaryMeta.exists()); + assertTrue(ig1BinaryMeta.isDirectory()); + stopGrid(1); - cleanBinaryMetaFolderForNode(ig1ConsId); + + U.delete(ig1BinaryMeta); ig1 = startGrid(1); stopGrid(0); @@ -559,22 +564,6 @@ private CountDownLatch initSlowFileIOFactory() { return cdl; } - /** - * Deletes directory with persisted binary metadata for a node with given Consistent ID. - */ - private void cleanBinaryMetaFolderForNode(String consId) throws IgniteCheckedException { - String dfltWorkDir = U.defaultWorkDirectory(); - File metaDir = U.resolveWorkDirectory(dfltWorkDir, DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false); - - for (File subDir : metaDir.listFiles()) { - if (subDir.getName().contains(consId)) { - U.delete(subDir); - - return; - } - } - } - /** Finds a key that target node is neither primary or backup. */ private int findNonAffinityKeyForNode(Affinity aff, ClusterNode targetNode, int startFrom) { int key = startFrom; @@ -690,11 +679,6 @@ static final class TestAccount { } } - /** */ - private static boolean isBinaryMetaFile(File file) { - return file.getPath().contains(DataStorageConfiguration.DFLT_BINARY_METADATA_PATH); - } - /** */ static final class SlowFileIOFactory implements FileIOFactory { /** */ @@ -711,7 +695,7 @@ static final class SlowFileIOFactory implements FileIOFactory { @Override public FileIO create(File file, OpenOption... modes) throws IOException { FileIO delegate = delegateFactory.create(file, modes); - if (isBinaryMetaFile(file)) + if (containsBinaryMetaPath(file)) return new SlowFileIO(delegate, fileWriteLatchRef.get()); return delegate; @@ -761,7 +745,7 @@ static final class FailingFileIOFactory implements FileIOFactory { @Override public FileIO create(File file, OpenOption... modes) throws IOException { FileIO delegate = delegateFactory.create(file, modes); - if (isBinaryMetaFile(file)) + if (containsBinaryMetaPath(file)) return new FailingFileIO(delegate); return delegate; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java index 3c086b64bbe19..111c3bf91e758 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java @@ -18,7 +18,6 @@ import java.io.File; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.Arrays; @@ -41,6 +40,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.binary.BinaryUtils; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; @@ -373,12 +373,13 @@ private void copyIncompatibleBinaryMetadata(String fromWorkDir, ) throws Exception { String workDir = U.defaultWorkDirectory(); - Path fromFile = Paths.get(workDir, fromWorkDir, DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, - fromConsId, fileName); - Path toFile = Paths.get(workDir, toWorkDir, DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, - toConsId, fileName); + NodeFileTree fromFt = new NodeFileTree(new File(workDir, fromWorkDir), fromConsId); + NodeFileTree toFt = new NodeFileTree(new File(workDir, toWorkDir), toConsId); - Files.copy(fromFile, toFile, StandardCopyOption.REPLACE_EXISTING); + File fromFile = new File(fromFt.binaryMeta(), fileName); + File toFile = new File(toFt.binaryMeta(), fileName); + + Files.copy(fromFile.toPath(), toFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsNoSpaceLeftOnDeviceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsNoSpaceLeftOnDeviceTest.java index 91dbade7f4aeb..4f24d363715a8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsNoSpaceLeftOnDeviceTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsNoSpaceLeftOnDeviceTest.java @@ -36,6 +36,8 @@ import org.apache.ignite.transactions.Transaction; import org.junit.Test; +import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.containsBinaryMetaPath; + /** * */ @@ -132,7 +134,7 @@ private static class FailingFileIOFactory implements FileIOFactory { @Override public FileIO create(File file, OpenOption... modes) throws IOException { if (unluckyConsistentId.get() != null && file.getAbsolutePath().contains(unluckyConsistentId.get()) - && file.getAbsolutePath().contains(DataStorageConfiguration.DFLT_BINARY_METADATA_PATH)) + && containsBinaryMetaPath(file)) throw new IOException("No space left on device"); return delegateFactory.create(file, modes); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/MaintenanceRegistrySimpleTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/MaintenanceRegistrySimpleTest.java index 61deef40e1b31..0f10488ccac51 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/MaintenanceRegistrySimpleTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/MaintenanceRegistrySimpleTest.java @@ -29,6 +29,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.maintenance.MaintenanceProcessor; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver; import org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneGridKernalContext; @@ -100,6 +101,10 @@ private GridKernalContext initContext(boolean persistenceEnabled) throws IgniteC @Override public PdsFolderSettings resolveFolders() { return new PdsFolderSettings(new File(dfltWorkDir), U.maskForFileName("")); } + + @Override public NodeFileTree fileTree() { + return new NodeFileTree(dfltWorkDir, U.maskForFileName("")); + } }; } }; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/filename/IgniteUidAsConsistentIdMigrationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/filename/IgniteUidAsConsistentIdMigrationTest.java index bc8677880c45d..84f3008f10dea 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/filename/IgniteUidAsConsistentIdMigrationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/filename/IgniteUidAsConsistentIdMigrationTest.java @@ -23,6 +23,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.UUID; +import java.util.function.Consumer; import java.util.regex.Pattern; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; @@ -32,6 +33,7 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderResolver; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridStringLogger; @@ -118,7 +120,10 @@ private void deleteWorkFiles() throws IgniteCheckedException { if (pstWalStoreCustomPath != null) ok &= U.delete(pstWalStoreCustomPath); - ok &= U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false)); + File binaryMetaRoot = sharedFileTree().binaryMetaRoot(); + + if (binaryMetaRoot.exists()) + ok &= U.delete(binaryMetaRoot); if (failIfDeleteNotCompleted) assertTrue(ok); @@ -168,7 +173,7 @@ public void testNewStyleIdIsGenerated() throws Exception { //test UUID is parsable from consistent ID test UUID.fromString(ignite.cluster().localNode().consistentId().toString()); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(0, ignite)); + assertPdsDirsDefaultExist(ignite, genNewStyleSubfolderName(0, ignite)); stopGrid(0); } @@ -186,7 +191,9 @@ public void testNewStyleIdIsGeneratedInCustomStorePath() throws Exception { UUID.fromString(ignite.cluster().localNode().consistentId().toString()); final String subfolderName = genNewStyleSubfolderName(0, ignite); - assertDirectoryExist(DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, subfolderName); + NodeFileTree ft = nodeFileTree(subfolderName); + + assertTrue(ft.binaryMeta().exists() && ft.binaryMeta().isDirectory()); assertDirectoryExist(pstWalArchCustomPath, subfolderName); assertDirectoryExist(pstWalArchCustomPath, subfolderName); @@ -205,7 +212,7 @@ public void testPreconfiguredConsitentIdIsApplied() throws Exception { this.configuredConsistentId = "someConfiguredConsistentId"; Ignite ignite = startActivateFillDataGrid(0); - assertPdsDirsDefaultExist(configuredConsistentId); + assertPdsDirsDefaultExist(ignite, configuredConsistentId); stopGrid(0); } @@ -227,7 +234,7 @@ public void testRestartOnExistingOldStyleId() throws Exception { igniteEx.getOrCreateCache(CACHE_NAME).put("hi", expVal); - assertPdsDirsDefaultExist(U.maskForFileName(configuredConsistentId)); + assertPdsDirsDefaultExist(igniteEx, U.maskForFileName(configuredConsistentId)); stopGrid(0); this.configuredConsistentId = null; //now set up grid on existing folder @@ -257,7 +264,7 @@ public void testStartWithoutActivate() throws Exception { stopGrid(0); Ignite igniteRestart = startActivateFillDataGrid(0); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(0, igniteRestart)); + assertPdsDirsDefaultExist(igniteRestart, genNewStyleSubfolderName(0, igniteRestart)); stopGrid(0); } @@ -272,7 +279,7 @@ public void testRestartOnSameFolderWillCauseSameUuidGeneration() throws Exceptio { final Ignite ignite = startActivateFillDataGrid(0); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(0, ignite)); + assertPdsDirsDefaultExist(ignite, genNewStyleSubfolderName(0, ignite)); uuid = (UUID)ignite.cluster().localNode().consistentId(); stopGrid(0); @@ -285,7 +292,7 @@ public void testRestartOnSameFolderWillCauseSameUuidGeneration() throws Exceptio final Object consIdRestart = igniteRestart.cluster().localNode().consistentId(); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(0, igniteRestart)); + assertPdsDirsDefaultExist(igniteRestart, genNewStyleSubfolderName(0, igniteRestart)); stopGrid(0); assertEquals(uuid, consIdRestart); @@ -304,7 +311,7 @@ public void testStartNodeAfterDeactivate() throws Exception { { final Ignite ignite = startActivateFillDataGrid(0); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(0, ignite)); + assertPdsDirsDefaultExist(ignite, genNewStyleSubfolderName(0, ignite)); uuid = (UUID)ignite.cluster().localNode().consistentId(); ignite.cluster().state(ClusterState.INACTIVE); @@ -315,7 +322,7 @@ public void testStartNodeAfterDeactivate() throws Exception { grid(0).cluster().state(ClusterState.ACTIVE); final Object consIdRestart = igniteRestart.cluster().localNode().consistentId(); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(1, igniteRestart)); + assertPdsDirsDefaultExist(igniteRestart, genNewStyleSubfolderName(1, igniteRestart)); stopGrid(1); assertFalse(consIdRestart.equals(uuid)); @@ -384,8 +391,8 @@ public void testNodeIndexIncremented() throws Exception { ignite0.getOrCreateCache(CACHE_NAME).put("hi", "there!"); ignite1.getOrCreateCache(CACHE_NAME).put("hi1", "there!"); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(0, ignite0)); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(1, ignite1)); + assertPdsDirsDefaultExist(ignite0, genNewStyleSubfolderName(0, ignite0)); + assertPdsDirsDefaultExist(ignite1, genNewStyleSubfolderName(1, ignite1)); stopGrid(0); stopGrid(1); @@ -409,17 +416,17 @@ public void testNewStyleAlwaysSmallestNodeIndexIsCreated() throws Exception { ignite0.getOrCreateCache(CACHE_NAME).put("hi", "there!"); ignite3.getOrCreateCache(CACHE_NAME).put("hi1", "there!"); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(0, ignite0)); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(1, ignite1)); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(2, ignite2)); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(3, ignite3)); + assertPdsDirsDefaultExist(ignite0, genNewStyleSubfolderName(0, ignite0)); + assertPdsDirsDefaultExist(ignite1, genNewStyleSubfolderName(1, ignite1)); + assertPdsDirsDefaultExist(ignite2, genNewStyleSubfolderName(2, ignite2)); + assertPdsDirsDefaultExist(ignite3, genNewStyleSubfolderName(3, ignite3)); assertNodeIndexesInFolder(0, 1, 2, 3); stopAllGrids(); //this grid should take folder with index 0 as unlocked final Ignite ignite4Restart = startActivateGrid(3); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(0, ignite4Restart)); + assertPdsDirsDefaultExist(ignite4Restart, genNewStyleSubfolderName(0, ignite4Restart)); assertNodeIndexesInFolder(0, 1, 2, 3); stopAllGrids(); @@ -439,14 +446,14 @@ public void testNewStyleAlwaysSmallestNodeIndexIsCreatedMultithreaded() throws E ignite0.getOrCreateCache(CACHE_NAME).put("hi", "there!"); ignite0.getOrCreateCache(CACHE_NAME).put("hi1", "there!"); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(0, ignite0)); + assertPdsDirsDefaultExist(ignite0, genNewStyleSubfolderName(0, ignite0)); assertNodeIndexesInFolder(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10); stopAllGrids(); //this grid should take folder with index 0 as unlocked final Ignite ignite4Restart = startActivateGrid(4); - assertPdsDirsDefaultExist(genNewStyleSubfolderName(0, ignite4Restart)); + assertPdsDirsDefaultExist(ignite4Restart, genNewStyleSubfolderName(0, ignite4Restart)); stopAllGrids(); assertNodeIndexesInFolder(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10); @@ -476,8 +483,8 @@ public void testStartTwoOldStyleNodes() throws Exception { ignite2.getOrCreateCache(CACHE_NAME).put("hi", expVal); - assertPdsDirsDefaultExist(U.maskForFileName(expDfltConsistentId1)); - assertPdsDirsDefaultExist(U.maskForFileName(expDfltConsistentId2)); + assertPdsDirsDefaultExist(ignite, U.maskForFileName(expDfltConsistentId1)); + assertPdsDirsDefaultExist(ignite2, U.maskForFileName(expDfltConsistentId2)); stopAllGrids(); this.configuredConsistentId = null; //now set up grid on existing folder @@ -526,10 +533,10 @@ public void testStartOldStyleNodesByCompatibleProperty() throws Exception { final Object consistentId1 = ignite1.cluster().localNode().consistentId(); - assertPdsDirsDefaultExist(U.maskForFileName(consistentId1.toString())); + assertPdsDirsDefaultExist(ignite1, U.maskForFileName(consistentId1.toString())); final Object consistentId2 = ignite2.cluster().localNode().consistentId(); - assertPdsDirsDefaultExist(U.maskForFileName(consistentId2.toString())); + assertPdsDirsDefaultExist(ignite2, U.maskForFileName(consistentId2.toString())); stopAllGrids(); System.clearProperty(IGNITE_DATA_STORAGE_FOLDER_BY_CONSISTENT_ID); @@ -569,7 +576,7 @@ public void testStartOldStyleNoPortsNodesByCompatibleProperty() throws Exception final Object consistentId1 = ignite1.cluster().localNode().consistentId(); - assertPdsDirsDefaultExist(U.maskForFileName(consistentId1.toString())); + assertPdsDirsDefaultExist(ignite1, U.maskForFileName(consistentId1.toString())); stopAllGrids(); System.clearProperty(IGNITE_DATA_STORAGE_FOLDER_BY_CONSISTENT_ID); @@ -606,7 +613,7 @@ public void testOldStyleNodeWithUnexpectedPort() throws Exception { final String prevVerFolder = U.maskForFileName(ignite.cluster().localNode().consistentId().toString()); final String path = new File(new File(U.defaultWorkDirectory(), "db"), prevVerFolder).getCanonicalPath(); - assertPdsDirsDefaultExist(prevVerFolder); + assertPdsDirsDefaultExist(ignite, prevVerFolder); stopAllGrids(); this.configuredConsistentId = null; @@ -674,10 +681,14 @@ private void assertNodeIndexesInFolder(Integer... indexes) throws IgniteCheckedE * Checks existence of all storage-related directories * * @param subDirName sub directories name expected - * @throws IgniteCheckedException if IO error occur */ - private void assertPdsDirsDefaultExist(String subDirName) throws IgniteCheckedException { - assertDirectoryExist(DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, subDirName); + private void assertPdsDirsDefaultExist(Ignite ign, String subDirName) throws IgniteCheckedException { + NodeFileTree ft = new NodeFileTree(ign.configuration(), subDirName); + + Consumer check = dir -> assertTrue(dir.exists() && dir.isDirectory()); + + check.accept(ft.binaryMeta()); + assertDirectoryExist(DataStorageConfiguration.DFLT_WAL_PATH, subDirName); assertDirectoryExist(DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, subDirName); assertDirectoryExist(PdsFolderResolver.DB_DEFAULT_FOLDER, subDirName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java index 29940f84b484a..76f691ae2b060 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java @@ -69,6 +69,7 @@ import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.GridCacheOperation; import org.apache.ignite.internal.processors.cache.KeyCacheObject; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer; import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder; @@ -1586,15 +1587,12 @@ public void testCheckBoundsIterator() throws Exception { @NotNull private IteratorParametersBuilder createIteratorParametersBuilder( String workDir, String subfolderName - ) throws IgniteCheckedException { - File binaryMeta = U.resolveWorkDirectory(workDir, DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, - false); - File binaryMetaWithConsId = new File(binaryMeta, subfolderName); - File marshallerMapping = U.resolveWorkDirectory(workDir, DataStorageConfiguration.DFLT_MARSHALLER_PATH, false); + ) { + NodeFileTree ft = new NodeFileTree(workDir, subfolderName); return new IteratorParametersBuilder() - .binaryMetadataFileStoreDir(binaryMetaWithConsId) - .marshallerMappingFileStoreDir(marshallerMapping); + .binaryMetadataFileStoreDir(ft.binaryMeta()) + .marshallerMappingFileStoreDir(ft.marshaller()); } /** diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java index 82840151b9409..0dea21c40240f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/MockWalIteratorFactory.java @@ -30,6 +30,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; @@ -111,6 +112,10 @@ public WALIterator iterator(File wal, File walArchive) throws IgniteCheckedExcep @Override public PdsFolderSettings resolveFolders() { return new PdsFolderSettings(new File("."), subfolderName, consistentId, null, false); } + + @Override public NodeFileTree fileTree() { + return new NodeFileTree(new File(".").getAbsolutePath(), subfolderName); + } }); final GridDiscoveryManager disco = Mockito.mock(GridDiscoveryManager.class); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/PlainSnapshotTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/PlainSnapshotTest.java index 0d18214bb2a02..5b8823695a206 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/PlainSnapshotTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/PlainSnapshotTest.java @@ -23,11 +23,11 @@ import java.util.Map; import org.apache.ignite.IgniteException; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.CU; @@ -37,8 +37,6 @@ import org.junit.runners.Parameterized; import static org.apache.ignite.cluster.ClusterState.ACTIVE; -import static org.apache.ignite.internal.MarshallerContextImpl.mappingFileStoreWorkDir; -import static org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.binaryWorkDir; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheDirName; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.databaseRelativePath; import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; @@ -116,13 +114,10 @@ public void testSnapshotLocalPartitions() throws Exception { stopGrid(ig.name()); // Calculate CRCs. - IgniteConfiguration cfg = ig.context().config(); - PdsFolderSettings settings = ig.context().pdsFolderResolver().resolveFolders(); + PdsFolderSettings settings = ig.context().pdsFolderResolver().resolveFolders(); String nodePath = databaseRelativePath(settings.folderName()); - File binWorkDir = binaryWorkDir(cfg.getWorkDirectory(), settings.folderName()); - File marshWorkDir = mappingFileStoreWorkDir(U.workDirectory(cfg.getWorkDirectory(), cfg.getIgniteHome())); - File snpBinWorkDir = binaryWorkDir(mgr.snapshotLocalDir(SNAPSHOT_NAME).getAbsolutePath(), settings.folderName()); - File snpMarshWorkDir = mappingFileStoreWorkDir(mgr.snapshotLocalDir(SNAPSHOT_NAME).getAbsolutePath()); + NodeFileTree ft = ig.context().pdsFolderResolver().fileTree(); + NodeFileTree snpFt = new NodeFileTree(mgr.snapshotLocalDir(SNAPSHOT_NAME).getAbsolutePath(), settings.folderName()); final Map origPartCRCs = calculateCRC32Partitions(cacheWorkDir); final Map snpPartCRCs = calculateCRC32Partitions( @@ -135,9 +130,9 @@ public void testSnapshotLocalPartitions() throws Exception { assertEquals("Partitions must have the same CRC after file copying and merging partition delta files", origPartCRCs, snpPartCRCs); assertEquals("Binary object mappings must be the same for local node and created snapshot", - calculateCRC32Partitions(binWorkDir), calculateCRC32Partitions(snpBinWorkDir)); + calculateCRC32Partitions(ft.binaryMeta()), calculateCRC32Partitions(snpFt.binaryMeta())); assertEquals("Marshaller meta mast be the same for local node and created snapshot", - calculateCRC32Partitions(marshWorkDir), calculateCRC32Partitions(snpMarshWorkDir)); + calculateCRC32Partitions(ft.marshaller()), calculateCRC32Partitions(snpFt.marshaller())); File snpWorkDir = mgr.snapshotTmpDir(); diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextSelfTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextSelfTest.java index 0511b7b64f16f..5e445271aec12 100644 --- a/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/marshaller/MarshallerContextSelfTest.java @@ -20,7 +20,6 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -28,9 +27,10 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.apache.ignite.Ignite; -import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.MarshallerContextImpl; +import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree; import org.apache.ignite.internal.processors.closure.GridClosureProcessor; import org.apache.ignite.internal.processors.marshaller.MappedName; import org.apache.ignite.internal.processors.marshaller.MarshallerMappingItem; @@ -38,6 +38,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.junits.GridTestKernalContext; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jetbrains.annotations.NotNull; import org.junit.Test; import static java.nio.file.Files.readAllBytes; @@ -77,9 +78,7 @@ public class MarshallerContextSelfTest extends GridCommonAbstractTest { */ @Test public void testClassName() throws Exception { - MarshallerContextImpl marshCtx = new MarshallerContextImpl(null, null); - - marshCtx.onMarshallerProcessorStarted(ctx, null); + MarshallerContextImpl marshCtx = marshallerContext(); MarshallerMappingItem item = new MarshallerMappingItem(JAVA_ID, 1, String.class.getName()); @@ -105,9 +104,7 @@ public void testClassName() throws Exception { public void testMultiplatformMappingsCollecting() throws Exception { String nonJavaClsName = "random.platform.Mapping"; - MarshallerContextImpl marshCtx = new MarshallerContextImpl(null, null); - - marshCtx.onMarshallerProcessorStarted(ctx, null); + MarshallerContextImpl marshCtx = marshallerContext(); MarshallerMappingItem item = new MarshallerMappingItem((byte)2, 101, nonJavaClsName); @@ -164,10 +161,7 @@ public void testMultiplatformMappingsDistributing() throws Exception { */ @Test public void testOnUpdated() throws Exception { - File workDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_MARSHALLER_PATH, false); - MarshallerContextImpl ctx = new MarshallerContextImpl(null, null); - - ctx.onMarshallerProcessorStarted(this.ctx, null); + MarshallerContextImpl ctx = marshallerContext(); MarshallerMappingItem item1 = new MarshallerMappingItem(JAVA_ID, 1, String.class.getName()); @@ -176,7 +170,9 @@ public void testOnUpdated() throws Exception { // Wait until marshaller context write class to file. U.sleep(2_000); - checkFileName("java.lang.String", Paths.get(workDir + "/1.classname0")); + SharedFileTree sft = sharedFileTree(); + + checkFileName("java.lang.String", new File(sft.marshaller(), "1.classname0").toPath()); MarshallerMappingItem item2 = new MarshallerMappingItem((byte)2, 2, "Random.Class.Name"); @@ -186,7 +182,7 @@ public void testOnUpdated() throws Exception { execSvc.shutdown(); if (execSvc.awaitTermination(1000, TimeUnit.MILLISECONDS)) - checkFileName("Random.Class.Name", Paths.get(workDir + "/2.classname2")); + checkFileName("Random.Class.Name", new File(sft.marshaller(), "2.classname2").toPath()); else fail("Failed to wait for executor service to shutdown"); } @@ -197,9 +193,7 @@ public void testOnUpdated() throws Exception { */ @Test public void testCacheStructure0() throws Exception { - MarshallerContextImpl ctx = new MarshallerContextImpl(null, null); - - ctx.onMarshallerProcessorStarted(this.ctx, null); + MarshallerContextImpl ctx = marshallerContext(); MarshallerMappingItem item1 = new MarshallerMappingItem(JAVA_ID, 1, String.class.getName()); @@ -231,9 +225,7 @@ public void testCacheStructure0() throws Exception { */ @Test public void testCacheStructure1() throws Exception { - MarshallerContextImpl ctx = new MarshallerContextImpl(null, null); - - ctx.onMarshallerProcessorStarted(this.ctx, null); + MarshallerContextImpl ctx = marshallerContext(); MarshallerMappingItem item1 = new MarshallerMappingItem(JAVA_ID, 1, String.class.getName()); @@ -271,4 +263,14 @@ private void checkFileName(String expected, Path pathToReal) throws IOException assertEquals(expected, new String(fileContent)); } + + /** */ + private @NotNull MarshallerContextImpl marshallerContext() throws IgniteCheckedException { + MarshallerContextImpl mctx = new MarshallerContextImpl(null, null); + + mctx.setMarshallerMappingFileStoreDir(sharedFileTree().marshaller()); + mctx.onMarshallerProcessorStarted(ctx, null); + + return mctx; + } } diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java index d7a1d9072fc55..369748a26f2c6 100755 --- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java @@ -88,6 +88,8 @@ import org.apache.ignite.internal.managers.systemview.JmxSystemViewExporterSpi; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; +import org.apache.ignite.internal.processors.cache.persistence.filename.SharedFileTree; import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree; import org.apache.ignite.internal.processors.resource.DependencyResolver; import org.apache.ignite.internal.processors.resource.GridSpringResourceContext; @@ -674,8 +676,13 @@ protected GridTestKernalContext newContext(IgniteConfiguration cfg) throws Ignit * Will clean and re-create marshaller directory from scratch. */ private void resolveWorkDirectory() throws Exception { - U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_MARSHALLER_PATH, true); - U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, true); + SharedFileTree sft = sharedFileTree(); + + U.delete(sft.marshaller()); + U.delete(sft.binaryMetaRoot()); + + SharedFileTree.mkdir(sft.binaryMetaRoot(), "root binary metadata"); + sft.mkdirMarshaller(); } /** */ @@ -3177,4 +3184,18 @@ public static T getMxBean(String igniteInstanceName, String grp, List list = new LinkedList<>(); final String nodeFolder = createWal(list, null); + final NodeFileTree ft = new NodeFileTree(U.defaultWorkDirectory(), nodeFolder); final ByteArrayOutputStream outByte = new ByteArrayOutputStream(); @@ -174,8 +174,8 @@ public void testIgniteWalConverter() throws Exception { U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_PATH, false), U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_ARCHIVE_PATH, false), DataStorageConfiguration.DFLT_PAGE_SIZE, - new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_BINARY_METADATA_PATH, false), nodeFolder), - U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_MARSHALLER_PATH, false), + ft.binaryMeta(), + ft.marshaller(), false, null, null, null, null, null, true, true, emptyList() @@ -296,6 +296,7 @@ public void testIgniteWalConverterWithBrokenWal() throws Exception { final List list = new LinkedList<>(); final String nodeFolder = createWal(list, null); + final NodeFileTree ft = new NodeFileTree(U.defaultWorkDirectory(), nodeFolder); final File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_PATH, false); @@ -355,8 +356,8 @@ public void testIgniteWalConverterWithBrokenWal() throws Exception { walDir, U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_ARCHIVE_PATH, false), DataStorageConfiguration.DFLT_PAGE_SIZE, - new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_BINARY_METADATA_PATH, false), nodeFolder), - U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_MARSHALLER_PATH, false), + ft.binaryMeta(), + ft.marshaller(), false, null, null, null, null, null, true, true, emptyList() @@ -415,6 +416,7 @@ public void testIgniteWalConverterWithUnreadableWal() throws Exception { final List list = new LinkedList<>(); final String nodeFolder = createWal(list, null); + final NodeFileTree ft = new NodeFileTree(U.defaultWorkDirectory(), nodeFolder); final File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_PATH, false); @@ -460,8 +462,8 @@ public void testIgniteWalConverterWithUnreadableWal() throws Exception { walDir, U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_ARCHIVE_PATH, false), DataStorageConfiguration.DFLT_PAGE_SIZE, - new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_BINARY_METADATA_PATH, false), nodeFolder), - U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_MARSHALLER_PATH, false), + ft.binaryMeta(), + ft.marshaller(), false, null, null, null, null, null, true, true, emptyList() From e762d30130d8a1fe410bc974b3a3172d68012753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=BE=D0=BB=D0=BE=D0=B6=D0=B0=D0=B5=D0=B2=20=D0=94?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=81=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B0?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= <22160647@sigma.sbrf.ru> Date: Wed, 5 Feb 2025 00:20:37 +1000 Subject: [PATCH 12/14] IGNITE-23459 Add console input if password argument presented without value for ./control.sh --- .../internal/commandline/ArgumentParser.java | 39 ++++++++----------- .../internal/commandline/CommandHandler.java | 2 +- .../ConnectionAndSslParameters.java | 18 ++++----- .../argument/parser/CLIArgumentParser.java | 10 ++--- ...GridCommandHandlerSslWithSecurityTest.java | 2 +- 5 files changed, 29 insertions(+), 42 deletions(-) diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ArgumentParser.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ArgumentParser.java index c897892e874c6..8e2d4ed03ccc9 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ArgumentParser.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ArgumentParser.java @@ -143,7 +143,7 @@ public class ArgumentParser { /** */ private final List> common = new ArrayList<>(); - /** Console instance */ + /** Console instance. */ protected final GridConsole console; /** @@ -220,7 +220,7 @@ public ConnectionAndSslParameters parseA parser.parse(args.listIterator()); - String argsToStr = convertCommandToString(raw.listIterator(), parser); + String safeCmd = buildSafeCommandString(raw.listIterator(), parser); A arg = (A)argument( cmdPath.peek().argClass(), @@ -237,7 +237,7 @@ public ConnectionAndSslParameters parseA throw new IllegalArgumentException("Experimental commands disabled"); } - return new ConnectionAndSslParameters<>(cmdPath, arg, parser, argsToStr); + return new ConnectionAndSslParameters<>(cmdPath, arg, parser, safeCmd); } /** @@ -339,39 +339,34 @@ private CLIArgumentParser createArgumentParser() { return new CLIArgumentParser(positionalArgs, namedArgs, console); } - /** - * Create string of command arguments for logging arguments with hidden confidential values - * @param rawIter raw command arguments iterator - * @param parser CLIArgumentParser - * - * @return string of command arguments for logging with hidden confidential values - */ - private String convertCommandToString(ListIterator rawIter, CLIArgumentParser parser) { - SB cmdToStr = new SB(); + /** @return safe command for logging with hidden confidential values. */ + private String buildSafeCommandString(ListIterator rawIter, CLIArgumentParser parser) { + SB res = new SB(); while (rawIter.hasNext()) { String arg = rawIter.next(); - CLIArgument cliArg = parser.getCliArg(arg.toLowerCase()); + CLIArgument argDesc = parser.getArgumentDescriptor(arg.toLowerCase()); - cmdToStr.a(arg).a(' '); + res.a(arg).a(' '); - if (cliArg == null || cliArg.isFlag()) + if (argDesc == null || argDesc.isFlag()) continue; String argVal = readNextValueToken(rawIter); if (argVal != null) { - if (cliArg.isSensitive()) { - cmdToStr.a("***** "); - log.info(String.format("Warning: %s is insecure. Whenever possible, use interactive " + - "prompt for password (just omit the argument value).", cliArg.name())); + if (!argDesc.isSensitive()) + res.a(argVal).a(' '); + else { + res.a("***** "); + + log.info(String.format("Warning: %s is insecure. Whenever possible, use interactive prompt for " + + "password (just omit the argument value).", argDesc.name())); } - else - cmdToStr.a(argVal).a(' '); } } - return cmdToStr.toString(); + return res.toString(); } } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index b30e5c8b88ce1..b3fe15c7b1926 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -271,7 +271,7 @@ public int execute(List rawArgs) { } logger.info("Command [" + cmdName + "] started"); - logger.info("Arguments: " + args.getArgumentsToString()); + logger.info("Arguments: " + args.getSafeCmd()); logger.info(U.DELIM); String deprecationMsg = args.command().deprecationMessage(args.commandArg()); diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ConnectionAndSslParameters.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ConnectionAndSslParameters.java index 180dd49b15fca..b5b8b488bf9dd 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ConnectionAndSslParameters.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ConnectionAndSslParameters.java @@ -65,8 +65,8 @@ public class ConnectionAndSslParameters { /** */ private final CLIArgumentParser parser; - /**String builder for logging arguments with hidden confidential values*/ - private final String argumentsToString; + /** Safe command. */ + private final String safeCmd; /** * @param cmdPath Path to the command in {@link CommandsRegistry} hierarchy. @@ -77,12 +77,12 @@ public ConnectionAndSslParameters( Deque> cmdPath, A arg, CLIArgumentParser parser, - String argumentsToString + String safeCmd ) { this.cmdPath = cmdPath; this.arg = arg; this.parser = parser; - this.argumentsToString = argumentsToString; + this.safeCmd = safeCmd; this.user = parser.get(CMD_USER); this.pwd = parser.get(CMD_PASSWORD); @@ -255,12 +255,8 @@ public boolean verbose() { return parser.get(CMD_VERBOSE); } - /** - * Return sting of arguments for logging with hidden confidential values - * - * @return arguments to string - */ - public String getArgumentsToString() { - return argumentsToString; + /** @return safe command. */ + public String getSafeCmd() { + return safeCmd; } } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java index 758af77771f87..47af60a2cff98 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java @@ -49,7 +49,7 @@ public class CLIArgumentParser { /** */ private final Map parsedArgs = new HashMap<>(); - /** Console instance */ + /** Console instance. */ protected final GridConsole console; /** */ @@ -171,12 +171,10 @@ public T get(int position) { } /** - * Get CLIArgument. - * * @param name Argument name. - * @return CLIArgument. + * @return Command line rgument. */ - public CLIArgument getCliArg(String name) { + public CLIArgument getArgumentDescriptor(String name) { return argConfiguration.get(name); } @@ -252,8 +250,6 @@ public static String readNextValueToken(ListIterator argsIter) { } /** - * Requests password from console with message. - * * @param msg Message. * @return Password. */ diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/processors/security/GridCommandHandlerSslWithSecurityTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/processors/security/GridCommandHandlerSslWithSecurityTest.java index 9275f7227f2e9..28738e99852c5 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/processors/security/GridCommandHandlerSslWithSecurityTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/processors/security/GridCommandHandlerSslWithSecurityTest.java @@ -256,7 +256,7 @@ public void testInputKeyUserPwdOnlyOncePwdArgEnd() throws Exception { } /** - * Perform the test with prepared List arguments + * Perform the test with prepared List arguments. * * @param args List of query arguments. * @throws Exception If failed. From a44606ae1a45015700042b5f0412fd06b21483f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=BE=D0=BB=D0=BE=D0=B6=D0=B0=D0=B5=D0=B2=20=D0=94?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=81=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B0?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= <22160647@sigma.sbrf.ru> Date: Wed, 5 Feb 2025 12:48:30 +1000 Subject: [PATCH 13/14] IGNITE-23459 Add console input if password argument presented without value for ./control.sh --- .../internal/commandline/ArgumentParser.java | 4 +- .../internal/commandline/CommandHandler.java | 2 +- .../ConnectionAndSslParameters.java | 7 +- .../argument/parser/CLIArgumentParser.java | 2 +- ...GridCommandHandlerSslWithSecurityTest.java | 105 +++++++++--------- 5 files changed, 60 insertions(+), 60 deletions(-) diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ArgumentParser.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ArgumentParser.java index 8e2d4ed03ccc9..1f09aeb3b4b4f 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ArgumentParser.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ArgumentParser.java @@ -149,7 +149,7 @@ public class ArgumentParser { /** * @param log Logger. * @param registry Supported commands. - * @param console Supported commands. + * @param console Console. */ public ArgumentParser(IgniteLogger log, IgniteCommandRegistry registry, GridConsole console) { this.log = log; @@ -339,7 +339,7 @@ private CLIArgumentParser createArgumentParser() { return new CLIArgumentParser(positionalArgs, namedArgs, console); } - /** @return safe command for logging with hidden confidential values. */ + /** @return String representation of command with hidden values of sensitive arguments. */ private String buildSafeCommandString(ListIterator rawIter, CLIArgumentParser parser) { SB res = new SB(); diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java index b3fe15c7b1926..2b45198eeb7e7 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java @@ -271,7 +271,7 @@ public int execute(List rawArgs) { } logger.info("Command [" + cmdName + "] started"); - logger.info("Arguments: " + args.getSafeCmd()); + logger.info("Arguments: " + args.safeCommandString()); logger.info(U.DELIM); String deprecationMsg = args.command().deprecationMessage(args.commandArg()); diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ConnectionAndSslParameters.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ConnectionAndSslParameters.java index b5b8b488bf9dd..d1a5bc28248fa 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ConnectionAndSslParameters.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/ConnectionAndSslParameters.java @@ -65,13 +65,14 @@ public class ConnectionAndSslParameters { /** */ private final CLIArgumentParser parser; - /** Safe command. */ + /** String representation of command with hidden values of sensitive arguments. */ private final String safeCmd; /** * @param cmdPath Path to the command in {@link CommandsRegistry} hierarchy. * @param arg Command argument. * @param parser CLI arguments parser. + * @param safeCmd String safe representation of command. */ public ConnectionAndSslParameters( Deque> cmdPath, @@ -255,8 +256,8 @@ public boolean verbose() { return parser.get(CMD_VERBOSE); } - /** @return safe command. */ - public String getSafeCmd() { + /** @return String representation of command with hidden values of sensitive arguments. */ + public String safeCommandString() { return safeCmd; } } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java index 47af60a2cff98..9d7769cac8069 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java @@ -172,7 +172,7 @@ public T get(int position) { /** * @param name Argument name. - * @return Command line rgument. + * @return Argument descriptor. */ public CLIArgument getArgumentDescriptor(String name) { return argConfiguration.get(name); diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/processors/security/GridCommandHandlerSslWithSecurityTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/processors/security/GridCommandHandlerSslWithSecurityTest.java index 28738e99852c5..2a5ac1cd27f6f 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/processors/security/GridCommandHandlerSslWithSecurityTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/processors/security/GridCommandHandlerSslWithSecurityTest.java @@ -209,14 +209,14 @@ public void testConnector() throws Exception { */ @Test public void testInputKeyUserPwdOnlyOncePwdArgStart() throws Exception { - performTest(Arrays.asList( - "--password", - "--state", - "--user", login, - "--keystore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "thinClient" : "connectorServer"), - "--keystore-password", keyStorePassword(), - "--truststore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "trusttwo" : "trustthree"), - "--truststore-password", keyStorePassword())); + doPasswordInteractiveInputTest(Arrays.asList( + "--password", + "--state", + "--user", login, + "--keystore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "thinClient" : "connectorServer"), + "--keystore-password", keyStorePassword(), + "--truststore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "trusttwo" : "trustthree"), + "--truststore-password", keyStorePassword())); } /** @@ -227,14 +227,14 @@ public void testInputKeyUserPwdOnlyOncePwdArgStart() throws Exception { */ @Test public void testInputKeyUserPwdOnlyOncePwdArgMiddle() throws Exception { - performTest(Arrays.asList( - "--state", - "--user", login, - "--password", - "--keystore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "thinClient" : "connectorServer"), - "--keystore-password", keyStorePassword(), - "--truststore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "trusttwo" : "trustthree"), - "--truststore-password", keyStorePassword())); + doPasswordInteractiveInputTest(Arrays.asList( + "--state", + "--user", login, + "--password", + "--keystore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "thinClient" : "connectorServer"), + "--keystore-password", keyStorePassword(), + "--truststore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "trusttwo" : "trustthree"), + "--truststore-password", keyStorePassword())); } /** @@ -245,82 +245,81 @@ public void testInputKeyUserPwdOnlyOncePwdArgMiddle() throws Exception { */ @Test public void testInputKeyUserPwdOnlyOncePwdArgEnd() throws Exception { - performTest(Arrays.asList( - "--state", - "--user", login, - "--keystore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "thinClient" : "connectorServer"), - "--keystore-password", keyStorePassword(), - "--truststore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "trusttwo" : "trustthree"), - "--truststore-password", keyStorePassword(), - "--password")); + doPasswordInteractiveInputTest(Arrays.asList( + "--state", + "--user", login, + "--keystore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "thinClient" : "connectorServer"), + "--keystore-password", keyStorePassword(), + "--truststore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "trusttwo" : "trustthree"), + "--truststore-password", keyStorePassword(), + "--password")); } /** - * Perform the test with prepared List arguments. + * Verify that the command work correctly when request few arguments + * without value that invoke console input. * - * @param args List of query arguments. * @throws Exception If failed. */ - private void performTest(List args) throws Exception { + @Test + public void testInputKeyForFewRequestedArguments() throws Exception { IgniteEx crd = startGrid(); crd.cluster().state(ACTIVE); TestCommandHandler hnd = newCommandHandler(); - AtomicInteger pwdCnt = new AtomicInteger(); + AtomicInteger reqCnt = new AtomicInteger(); ((CommandHandler)GridTestUtils.getFieldValue(hnd, "hnd")).console = new NoopConsole() { @Override public char[] readPassword(String fmt, Object... args) { - pwdCnt.incrementAndGet(); - - return pwd.toCharArray(); + if (reqCnt.incrementAndGet() == 1) + return keyStorePassword().toCharArray(); + else + return pwd.toCharArray(); } }; - int exitCode = hnd.execute(args); + int exitCode = hnd.execute(Arrays.asList( + "--state", + "--user", login, + "--verbose", + "--keystore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "thinClient" : "connectorServer"), + "--keystore-password", + "--truststore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "trusttwo" : "trustthree"), + "--truststore-password", keyStorePassword(), + "--password")); assertEquals(EXIT_CODE_OK, exitCode); - assertEquals(1, pwdCnt.get()); + assertEquals(2, reqCnt.get()); } /** - * Verify that the command work correctly when request few arguments - * without value that invoke console input. + * Perform the test with prepared List arguments. * + * @param args List of query arguments. * @throws Exception If failed. */ - @Test - public void testInputKeyForFewRequestedArguments() throws Exception { + private void doPasswordInteractiveInputTest(List args) throws Exception { IgniteEx crd = startGrid(); crd.cluster().state(ACTIVE); TestCommandHandler hnd = newCommandHandler(); - AtomicInteger reqCnt = new AtomicInteger(); + AtomicInteger pwdCnt = new AtomicInteger(); ((CommandHandler)GridTestUtils.getFieldValue(hnd, "hnd")).console = new NoopConsole() { @Override public char[] readPassword(String fmt, Object... args) { - reqCnt.incrementAndGet(); - if (reqCnt.get() == 1) - return keyStorePassword().toCharArray(); - else - return pwd.toCharArray(); + pwdCnt.incrementAndGet(); + + return pwd.toCharArray(); } }; - int exitCode = hnd.execute(Arrays.asList( - "--state", - "--user", login, - "--verbose", - "--keystore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "thinClient" : "connectorServer"), - "--keystore-password", - "--truststore", keyStorePath(CLI_CMD_HND.equals(commandHandler) ? "trusttwo" : "trustthree"), - "--truststore-password", keyStorePassword(), - "--password")); + int exitCode = hnd.execute(args); assertEquals(EXIT_CODE_OK, exitCode); - assertEquals(2, reqCnt.get()); + assertEquals(1, pwdCnt.get()); } } From ba4b59dd2aaf389d631ff9b509856f065829773d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=BE=D0=BB=D0=BE=D0=B6=D0=B0=D0=B5=D0=B2=20=D0=94?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=81=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B0?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= <22160647@sigma.sbrf.ru> Date: Thu, 6 Feb 2025 13:25:58 +1000 Subject: [PATCH 14/14] IGNITE-23459 Add console input if password argument presented without value for ./control.sh --- .../ignite/util/GridCommandHandlerClusterByClassTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java index 80ce1ccc2078a..efc11f3cd5bdd 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java @@ -1362,7 +1362,7 @@ public void testCacheCreate() { assertContains( log, executeCommand(EXIT_CODE_INVALID_ARGUMENTS, "--cache", CREATE, SPRING_XML_CONFIG), - "Please specify a value for argument: --springXmlConfig" + "Please specify a value for argument: --springxmlconfig" ); autoConfirmation = true;