Skip to content

Commit f240623

Browse files
nizhikovnizhikov
committed
IGNITE-24454 Cache partition files calculation in NodeFileTree
1 parent 2d8891e commit f240623

14 files changed

+91
-86
lines changed

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java

Lines changed: 12 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@
5252
import org.apache.ignite.internal.GridKernalContext;
5353
import org.apache.ignite.internal.client.util.GridConcurrentHashSet;
5454
import org.apache.ignite.internal.managers.encryption.EncryptionCacheKeyProvider;
55-
import org.apache.ignite.internal.pagemem.PageIdAllocator;
5655
import org.apache.ignite.internal.pagemem.PageIdUtils;
5756
import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
5857
import org.apache.ignite.internal.pagemem.store.PageStore;
@@ -82,7 +81,6 @@
8281
import org.jetbrains.annotations.NotNull;
8382
import org.jetbrains.annotations.Nullable;
8483

85-
import static java.lang.String.format;
8684
import static java.nio.file.Files.delete;
8785
import static java.nio.file.Files.newDirectoryStream;
8886
import static java.util.Objects.requireNonNull;
@@ -92,9 +90,9 @@
9290
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.CACHE_DIR_WITH_META_FILTER;
9391
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.FILE_SUFFIX;
9492
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.INDEX_FILE_NAME;
95-
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.PART_FILE_PREFIX;
96-
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.PART_FILE_TEMPLATE;
9793
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.TMP_SUFFIX;
94+
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.partitionFileName;
95+
import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_DIR_NAME;
9896

9997
/**
10098
* File page store manager.
@@ -420,7 +418,7 @@ private List<CacheConfiguration> findCacheGroupsWithDisabledWal() {
420418
PageMetrics pageMetrics = dataRegion.metrics().cacheGrpPageMetrics(grpId);
421419

422420
CacheStoreHolder holder = initDir(
423-
new File(ft.nodeStorage(), MetaStorage.METASTORAGE_DIR_NAME),
421+
METASTORAGE_DIR_NAME,
424422
grpId,
425423
MetaStorage.METASTORAGE_CACHE_NAME,
426424
MetaStorage.METASTORAGE_PARTITIONS.size(),
@@ -510,14 +508,12 @@ private List<CacheConfiguration> findCacheGroupsWithDisabledWal() {
510508
private CacheStoreHolder initForCache(CacheGroupDescriptor grpDesc, CacheConfiguration ccfg) throws IgniteCheckedException {
511509
assert !grpDesc.sharedGroup() || ccfg.getGroupName() != null : ccfg.getName();
512510

513-
File cacheWorkDir = ft.cacheStorage(ccfg);
514-
515511
String dataRegionName = grpDesc.config().getDataRegionName();
516512
DataRegion dataRegion = cctx.database().dataRegion(dataRegionName);
517513
PageMetrics pageMetrics = dataRegion.metrics().cacheGrpPageMetrics(grpDesc.groupId());
518514

519515
return initDir(
520-
cacheWorkDir,
516+
ft.cacheDirName(ccfg),
521517
grpDesc.groupId(),
522518
ccfg.getName(),
523519
grpDesc.config().getAffinity().partitions(),
@@ -590,7 +586,7 @@ public EncryptedFileIOFactory encryptedFileIoFactory(FileIOFactory plainFileIOFa
590586
}
591587

592588
/**
593-
* @param cacheWorkDir Work directory.
589+
* @param cacheDir Cache dir name.
594590
* @param grpId Group ID.
595591
* @param cacheName Cache name.
596592
* @param partitions Number of partitions.
@@ -599,14 +595,17 @@ public EncryptedFileIOFactory encryptedFileIoFactory(FileIOFactory plainFileIOFa
599595
* @return Cache store holder.
600596
* @throws IgniteCheckedException If failed.
601597
*/
602-
private CacheStoreHolder initDir(File cacheWorkDir,
598+
private CacheStoreHolder initDir(
599+
String cacheDir,
603600
int grpId,
604601
String cacheName,
605602
int partitions,
606603
PageMetrics pageMetrics,
607604
boolean encrypted,
608605
Collection<String> grpCaches) throws IgniteCheckedException {
609606
try {
607+
File cacheWorkDir = ft.cacheStorage(cacheDir);
608+
610609
boolean dirExisted = checkAndInitCacheWorkDir(cacheWorkDir, log);
611610

612611
if (dirExisted) {
@@ -656,7 +655,7 @@ private CacheStoreHolder initDir(File cacheWorkDir,
656655
PageStore partStore =
657656
pageStoreFactory.createPageStore(
658657
PageStore.TYPE_DATA,
659-
() -> getPartitionFilePath(cacheWorkDir, p),
658+
() -> NodeFileTree.partitionFile(ft.nodeStorage(), cacheDir, p).toPath(),
660659
pageMetrics.totalPages()::add);
661660

662661
partStores[partId] = partStore;
@@ -677,27 +676,7 @@ private CacheStoreHolder initDir(File cacheWorkDir,
677676
* @param partId Partition id.
678677
*/
679678
@NotNull private Path getPartitionFilePath(File cacheWorkDir, int partId) {
680-
return new File(cacheWorkDir, getPartitionFileName(partId)).toPath();
681-
}
682-
683-
/**
684-
* @param workDir Cache work directory.
685-
* @param cacheDirName Cache directory name.
686-
* @param partId Partition id.
687-
* @return Partition file.
688-
*/
689-
@NotNull public static File getPartitionFile(File workDir, String cacheDirName, int partId) {
690-
return new File(NodeFileTree.cacheStorage(workDir, cacheDirName), getPartitionFileName(partId));
691-
}
692-
693-
/**
694-
* @param partId Partition id.
695-
* @return File name.
696-
*/
697-
public static String getPartitionFileName(int partId) {
698-
assert partId <= MAX_PARTITION_ID || partId == INDEX_PARTITION;
699-
700-
return partId == INDEX_PARTITION ? INDEX_FILE_NAME : format(PART_FILE_TEMPLATE, partId);
679+
return new File(cacheWorkDir, partitionFileName(partId)).toPath();
701680
}
702681

703682
/**
@@ -822,20 +801,6 @@ public static File cacheDirectory(File dir, int grpId) {
822801
.orElse(null);
823802
}
824803

825-
/**
826-
* @param partFileName Partition file name.
827-
* @return Partition id.
828-
*/
829-
public static int partId(String partFileName) {
830-
if (partFileName.equals(INDEX_FILE_NAME))
831-
return PageIdAllocator.INDEX_PARTITION;
832-
833-
if (partFileName.startsWith(PART_FILE_PREFIX))
834-
return Integer.parseInt(partFileName.substring(PART_FILE_PREFIX.length(), partFileName.indexOf('.')));
835-
836-
throw new IllegalStateException("Illegal partition file name: " + partFileName);
837-
}
838-
839804
/**
840805
* @param cacheDir Cache directory to check.
841806
* @return List of cache partitions in given directory.
@@ -896,7 +861,7 @@ public static File[] cacheDataFiles(File root) {
896861
*/
897862
public String cacheDirName(int grpId) throws IgniteCheckedException {
898863
if (grpId == MetaStorage.METASTORAGE_CACHE_ID)
899-
return MetaStorage.METASTORAGE_DIR_NAME;
864+
return METASTORAGE_DIR_NAME;
900865

901866
CacheGroupContext gctx = cctx.cache().cacheGroup(grpId);
902867

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,18 @@
3131
import org.apache.ignite.internal.util.typedef.internal.S;
3232
import org.apache.ignite.internal.util.typedef.internal.U;
3333
import org.apache.ignite.spi.checkpoint.sharedfs.SharedFsCheckpointSpi;
34+
import org.jetbrains.annotations.NotNull;
3435
import org.jetbrains.annotations.Nullable;
3536

37+
import static java.lang.String.format;
3638
import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH;
3739
import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_CDC_PATH;
3840
import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_PATH;
41+
import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
42+
import static org.apache.ignite.internal.pagemem.PageIdAllocator.MAX_PARTITION_ID;
3943
import static org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderResolver.DB_DEFAULT_FOLDER;
44+
import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_NAME;
45+
import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_DIR_NAME;
4046

4147
/**
4248
* Provides access to Ignite node file tree.
@@ -489,6 +495,26 @@ public static String cacheDataFilename(CacheConfiguration<?, ?> ccfg) {
489495
return ccfg.getGroupName() == null ? CACHE_DATA_FILENAME : (ccfg.getName() + CACHE_DATA_FILENAME);
490496
}
491497

498+
/**
499+
* @param workDir Cache work directory.
500+
* @param cacheDirName Cache directory name.
501+
* @param partId Partition id.
502+
* @return Partition file.
503+
*/
504+
@NotNull public static File partitionFile(File workDir, String cacheDirName, int partId) {
505+
return new File(cacheStorage(workDir, cacheDirName), partitionFileName(partId));
506+
}
507+
508+
/**
509+
* @param partId Partition id.
510+
* @return File name.
511+
*/
512+
public static String partitionFileName(int partId) {
513+
assert partId <= MAX_PARTITION_ID || partId == INDEX_PARTITION;
514+
515+
return partId == INDEX_PARTITION ? INDEX_FILE_NAME : format(PART_FILE_TEMPLATE, partId);
516+
}
517+
492518
/**
493519
* @param cacheDirName Cache directory name.
494520
* @return Store directory for given cache.
@@ -552,6 +578,9 @@ public File cacheStorage(String cacheDirName) {
552578
* @return The full cache directory name.
553579
*/
554580
public static String cacheDirName(boolean isSharedGroup, String cacheOrGroupName) {
581+
if (cacheOrGroupName.equals(METASTORAGE_CACHE_NAME))
582+
return METASTORAGE_DIR_NAME;
583+
555584
return isSharedGroup
556585
? CACHE_GRP_DIR_PREFIX + cacheOrGroupName
557586
: CACHE_DIR_PREFIX + cacheOrGroupName;
@@ -569,7 +598,7 @@ public static String cacheName(File dir) {
569598
else if (name.startsWith(CACHE_DIR_PREFIX))
570599
return name.substring(CACHE_DIR_PREFIX.length());
571600
else if (name.equals(MetaStorage.METASTORAGE_DIR_NAME))
572-
return MetaStorage.METASTORAGE_CACHE_NAME;
601+
return METASTORAGE_CACHE_NAME;
573602
else
574603
throw new IgniteException("Directory doesn't match the cache or cache group prefix: " + dir);
575604
}
@@ -584,6 +613,21 @@ public long walSegmentIndex(Path segment) {
584613
return Long.parseLong(fn.substring(0, fn.indexOf('.')));
585614
}
586615

616+
/**
617+
* @param part Partition file name.
618+
* @return Partition id.
619+
*/
620+
public static int partId(File part) {
621+
String name = part.getName();
622+
if (name.equals(INDEX_FILE_NAME))
623+
return INDEX_PARTITION;
624+
625+
if (name.startsWith(PART_FILE_PREFIX))
626+
return Integer.parseInt(name.substring(PART_FILE_PREFIX.length(), name.indexOf('.')));
627+
628+
throw new IllegalStateException("Illegal partition file name: " + name);
629+
}
630+
587631
/**
588632
* Resolves directory specified by the given arguments.
589633
*

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -212,11 +212,11 @@
212212
import static org.apache.ignite.internal.processors.cache.GridCacheUtils.baselineNode;
213213
import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isPersistenceEnabled;
214214
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheDirectories;
215-
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.getPartitionFile;
216-
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.getPartitionFileName;
217215
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.INDEX_FILE_NAME;
218216
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.PART_FILE_TEMPLATE;
219217
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.cacheName;
218+
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.partitionFile;
219+
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.partitionFileName;
220220
import static org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderResolver.DB_DEFAULT_FOLDER;
221221
import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID;
222222
import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_NAME;
@@ -2587,7 +2587,7 @@ public GridCloseableIterator<CacheDataRow> partitionRowIterator(String snpName,
25872587
);
25882588
}
25892589

2590-
File snpPart = getPartitionFile(new File(snapshotLocalDir(snpName, null), databaseRelativePath(folderName)),
2590+
File snpPart = partitionFile(new File(snapshotLocalDir(snpName, null), databaseRelativePath(folderName)),
25912591
grps.get(0).getName(), partId);
25922592

25932593
int grpId = CU.cacheId(grpName);
@@ -3898,7 +3898,7 @@ else if (msg instanceof SnapshotFilesFailureMessage) {
38983898
File tmpCacheDir = U.resolveWorkDirectory(ft.nodeStorage().getAbsolutePath(),
38993899
formatTmpDirName(cacheDir).getName(), false);
39003900

3901-
return Paths.get(tmpCacheDir.getAbsolutePath(), getPartitionFileName(partId)).toString();
3901+
return Paths.get(tmpCacheDir.getAbsolutePath(), partitionFileName(partId)).toString();
39023902
}
39033903
catch (IgniteCheckedException e) {
39043904
throw new IgniteException(e);
@@ -4178,7 +4178,7 @@ public LocalSnapshotSender(String snpName, @Nullable String snpPath) {
41784178

41794179
/** {@inheritDoc} */
41804180
@Override public void sendDelta0(File delta, String cacheDirName, GroupPartitionId pair) {
4181-
File snpPart = getPartitionFile(dbDir, cacheDirName, pair.getPartitionId());
4181+
File snpPart = partitionFile(dbDir, cacheDirName, pair.getPartitionId());
41824182

41834183
if (log.isDebugEnabled()) {
41844184
log.debug("Start partition snapshot recovery with the given delta page file [part=" + snpPart +

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotFutureTask.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,6 @@
7373
import org.apache.ignite.internal.util.typedef.internal.U;
7474
import org.jetbrains.annotations.Nullable;
7575

76-
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.getPartitionFile;
7776
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.cacheStorage;
7877
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.copy;
7978
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.databaseRelativePath;
@@ -412,7 +411,7 @@ public IgniteInternalFuture<?> started() {
412411

413412
return runAsync(() -> {
414413
snpSndr.sendPart(
415-
getPartitionFile(ft.nodeStorage(), cacheDirName, partId),
414+
NodeFileTree.partitionFile(ft.nodeStorage(), cacheDirName, partId),
416415
cacheDirName,
417416
pair,
418417
partLen);

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,10 +78,10 @@
7878
import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.fromOrdinal;
7979
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheDirectories;
8080
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cachePartitionFiles;
81-
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.partId;
8281
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.FILE_SUFFIX;
8382
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.ZIP_SUFFIX;
8483
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.cacheName;
84+
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.partId;
8585
import static org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId.getTypeByPartId;
8686
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.databaseRelativePath;
8787
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.dump.CreateDumpFutureTask.DUMP_FILE_EXT;
@@ -148,7 +148,7 @@ public SnapshotPartitionsVerifyHandler(GridCacheSharedContext<?, ?> cctx) {
148148
for (File part : cachePartitionFiles(dir,
149149
(meta.dump() ? DUMP_FILE_EXT : FILE_SUFFIX) + (meta.compressPartitions() ? ZIP_SUFFIX : "")
150150
)) {
151-
int partId = partId(part.getName());
151+
int partId = partId(part);
152152

153153
if (!parts.remove(partId))
154154
continue;
@@ -212,7 +212,7 @@ private Map<PartitionKey, PartitionHashRecord> checkSnapshotFiles(
212212
part -> {
213213
String grpName = cacheName(part.getParentFile());
214214
int grpId = CU.cacheId(grpName);
215-
int partId = partId(part.getName());
215+
int partId = partId(part);
216216

217217
try (FilePageStore pageStore =
218218
(FilePageStore)storeMgr.getPageStoreFactory(grpId, snpEncrKeyProvider.getActiveKey(grpId) != null ?
@@ -372,7 +372,7 @@ private Map<PartitionKey, PartitionHashRecord> checkDumpFiles(
372372
Collection<PartitionHashRecord> partitionHashRecords = U.doInParallel(
373373
cctx.snapshotMgr().snapshotExecutorService(),
374374
partFiles,
375-
part -> calculateDumpedPartitionHash(dump, cacheName(part.getParentFile()), partId(part.getName()))
375+
part -> calculateDumpedPartitionHash(dump, cacheName(part.getParentFile()), partId(part))
376376
);
377377

378378
return partitionHashRecords.stream().collect(Collectors.toMap(PartitionHashRecord::partitionKey, r -> r));

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotResponseRemoteFutureTask.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
import org.jetbrains.annotations.Nullable;
3434

3535
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheDirectory;
36-
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.getPartitionFile;
36+
import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.partitionFile;
3737
import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.databaseRelativePath;
3838

3939
/** */
@@ -117,7 +117,7 @@ public SnapshotResponseRemoteFutureTask(
117117
", pair=" + gp + ']');
118118
}
119119

120-
File snpPart = getPartitionFile(cacheDir.getParentFile(), cacheDir.getName(), gp.getPartitionId());
120+
File snpPart = partitionFile(cacheDir.getParentFile(), cacheDir.getName(), gp.getPartitionId());
121121

122122
if (!snpPart.exists()) {
123123
throw new IgniteException("Snapshot partition file not found [cacheDir=" + cacheDir +

0 commit comments

Comments
 (0)