diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ac72c34e8..1af9e0930 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/src/main/java/ru/vk/itmo/abramovilya/DaoImpl.java b/src/main/java/ru/vk/itmo/abramovilya/DaoImpl.java index d3ade0446..2648082c5 100644 --- a/src/main/java/ru/vk/itmo/abramovilya/DaoImpl.java +++ b/src/main/java/ru/vk/itmo/abramovilya/DaoImpl.java @@ -1,6 +1,5 @@ package ru.vk.itmo.abramovilya; -import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; @@ -9,66 +8,23 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; public class DaoImpl implements Dao> { private final ConcurrentNavigableMap> map = new ConcurrentSkipListMap<>(DaoImpl::compareMemorySegments); - private final Path storagePath; private final Arena arena = Arena.ofShared(); - private static final String SSTABLE_BASE_NAME = "storage"; - private static final String INDEX_BASE_NAME = "table"; - private final Path metaFilePath; - private final List sstableFileChannels = new ArrayList<>(); - private final List sstableMappedList = new ArrayList<>(); - private final List indexFileChannels = new ArrayList<>(); - private final List indexMappedList = new ArrayList<>(); + private final Storage storage; public DaoImpl(Config config) throws IOException { - storagePath = config.basePath(); - - Files.createDirectories(storagePath); - metaFilePath = storagePath.resolve("meta"); - if (!Files.exists(metaFilePath)) { - Files.createFile(metaFilePath); - Files.writeString(metaFilePath, "0", StandardOpenOption.WRITE); - } - - int totalSSTables = Integer.parseInt(Files.readString(metaFilePath)); - for (int sstableNum = 0; sstableNum < totalSSTables; sstableNum++) { - Path sstablePath = storagePath.resolve(SSTABLE_BASE_NAME + sstableNum); - Path indexPath = storagePath.resolve(INDEX_BASE_NAME + sstableNum); - - FileChannel sstableFileChannel = FileChannel.open(sstablePath, StandardOpenOption.READ); - sstableFileChannels.add(sstableFileChannel); - MemorySegment sstableMapped = - sstableFileChannel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(sstablePath), arena); - sstableMappedList.add(sstableMapped); - - FileChannel indexFileChannel = FileChannel.open(indexPath, StandardOpenOption.READ); - indexFileChannels.add(indexFileChannel); - MemorySegment indexMapped = - indexFileChannel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(indexPath), arena); - indexMappedList.add(indexMapped); - } + this.storage = new Storage(config, arena); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { - return new DaoIterator(getTotalSStables(), from, to, sstableMappedList, indexMappedList, map); - } - - @Override - public void upsert(Entry entry) { - map.put(entry.key(), entry); + return new DaoIterator(storage.getTotalSStables(), from, to, storage, map); } @Override @@ -80,116 +36,29 @@ public Entry get(MemorySegment key) { } return null; } - - int totalSStables = getTotalSStables(); - for (int sstableNum = totalSStables; sstableNum >= 0; sstableNum--) { - var foundEntry = seekForValueInFile(key, sstableNum); - if (foundEntry != null) { - if (foundEntry.value() != null) { - return foundEntry; - } - return null; - } - } - return null; + return storage.get(key); } - private Entry seekForValueInFile(MemorySegment key, int sstableNum) { - if (sstableNum >= sstableFileChannels.size()) { - return null; - } - - MemorySegment storageMapped = sstableMappedList.get(sstableNum); - MemorySegment indexMapped = indexMappedList.get(sstableNum); - - int foundIndex = upperBound(key, storageMapped, indexMapped, indexMapped.byteSize()); - long keyStorageOffset = getKeyStorageOffset(indexMapped, foundIndex); - long foundKeySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, keyStorageOffset); - keyStorageOffset += Long.BYTES; - - if (MemorySegment.mismatch(key, - 0, - key.byteSize(), - storageMapped, - keyStorageOffset, - keyStorageOffset + foundKeySize) == -1) { - return getEntryFromIndexFile(storageMapped, indexMapped, foundIndex); - } - return null; - } - - static int upperBound(MemorySegment key, MemorySegment storageMapped, MemorySegment indexMapped, long indexSize) { - int l = -1; - int r = indexMapped.get(ValueLayout.JAVA_INT_UNALIGNED, indexSize - Long.BYTES - Integer.BYTES); - - while (r - l > 1) { - int m = (r + l) / 2; - long keyStorageOffset = getKeyStorageOffset(indexMapped, m); - long keySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, keyStorageOffset); - keyStorageOffset += Long.BYTES; - - if (compareMemorySegmentsUsingOffset(key, storageMapped, keyStorageOffset, keySize) > 0) { - l = m; - } else { - r = m; - } - } - return r; - } - - static long getKeyStorageOffset(MemorySegment indexMapped, int entryNum) { - return indexMapped.get( - ValueLayout.JAVA_LONG_UNALIGNED, - (long) (Integer.BYTES + Long.BYTES) * entryNum + Integer.BYTES - ); + @Override + public void upsert(Entry entry) { + map.put(entry.key(), entry); } - private Entry getEntryFromIndexFile(MemorySegment storageMapped, - MemorySegment indexMapped, - int entryNum) { - long offsetInStorageFile = indexMapped.get( - ValueLayout.JAVA_LONG_UNALIGNED, - (long) (Integer.BYTES + Long.BYTES) * entryNum + Integer.BYTES - ); - - long keySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, offsetInStorageFile); - offsetInStorageFile += Long.BYTES; - offsetInStorageFile += keySize; - - long valueSize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, offsetInStorageFile); - offsetInStorageFile += Long.BYTES; - MemorySegment key = storageMapped.asSlice(offsetInStorageFile - keySize - Long.BYTES, keySize); - MemorySegment value; - if (valueSize == -1) { - value = null; - } else { - value = storageMapped.asSlice(offsetInStorageFile, valueSize); + @Override + public void compact() throws IOException { + var iterator = get(null, null); + if (!iterator.hasNext()) { + return; } - return new BaseEntry<>(key, value); + storage.compact(iterator, get(null, null)); + map.clear(); } @Override public void flush() throws IOException { - writeMapIntoFile(); - if (!map.isEmpty()) incTotalSStablesAmount(); - } - - private void incTotalSStablesAmount() throws IOException { - int totalSStables = getTotalSStables(); - Files.writeString(metaFilePath, String.valueOf(totalSStables + 1)); - } - - @Override - public void close() throws IOException { - if (arena.scope().isAlive()) { - arena.close(); - } - flush(); - for (FileChannel fc : sstableFileChannels) { - if (fc.isOpen()) fc.close(); - } - for (FileChannel fc : indexFileChannels) { - if (fc.isOpen()) fc.close(); + if (!map.isEmpty()) { + writeMapIntoFile(); + storage.incTotalSStablesAmount(); } } @@ -197,30 +66,17 @@ private void writeMapIntoFile() throws IOException { if (map.isEmpty()) { return; } - - int currSStableNum = getTotalSStables(); - Path sstablePath = storagePath.resolve(SSTABLE_BASE_NAME + currSStableNum); - Path indexPath = storagePath.resolve(INDEX_BASE_NAME + currSStableNum); - - StorageWriter.writeSStableAndIndex(sstablePath, - calcMapByteSizeInFile(), - indexPath, - calcIndexByteSizeInFile(), - map); - } - - private int getTotalSStables() { - return sstableFileChannels.size(); - } - - private long calcIndexByteSizeInFile() { - return (long) map.size() * (Integer.BYTES + Long.BYTES); + storage.writeMapIntoFile( + mapByteSizeInFile(), + indexByteSizeInFile(), + map + ); } - private long calcMapByteSizeInFile() { + private long mapByteSizeInFile() { long size = 0; for (var entry : map.values()) { - size += 2 * Long.BYTES; + size += Storage.BYTES_TO_STORE_ENTRY_SIZE; size += entry.key().byteSize(); if (entry.value() != null) { size += entry.value().byteSize(); @@ -229,17 +85,29 @@ private long calcMapByteSizeInFile() { return size; } + private long indexByteSizeInFile() { + return (long) map.size() * Storage.INDEX_ENTRY_SIZE; + } + + @Override + public void close() throws IOException { + if (arena.scope().isAlive()) { + arena.close(); + } + flush(); + storage.close(); + } + public static int compareMemorySegments(MemorySegment segment1, MemorySegment segment2) { - long mismatch = segment1.mismatch(segment2); - if (mismatch == -1) { + long offset = segment1.mismatch(segment2); + if (offset == -1) { return 0; - } else if (mismatch == segment1.byteSize()) { + } else if (offset == segment1.byteSize()) { return -1; - } else if (mismatch == segment2.byteSize()) { + } else if (offset == segment2.byteSize()) { return 1; } - return Byte.compare(segment1.get(ValueLayout.JAVA_BYTE, mismatch), - segment2.get(ValueLayout.JAVA_BYTE, mismatch)); + return Byte.compare(segment1.get(ValueLayout.JAVA_BYTE, offset), segment2.get(ValueLayout.JAVA_BYTE, offset)); } public static int compareMemorySegmentsUsingOffset(MemorySegment segment1, @@ -261,6 +129,5 @@ public static int compareMemorySegmentsUsingOffset(MemorySegment segment1, } return Byte.compare(segment1.get(ValueLayout.JAVA_BYTE, mismatch), segment2.get(ValueLayout.JAVA_BYTE, segment2Offset + mismatch)); - } } diff --git a/src/main/java/ru/vk/itmo/abramovilya/DaoIterator.java b/src/main/java/ru/vk/itmo/abramovilya/DaoIterator.java index d62fbd2b7..fdc8de6a2 100644 --- a/src/main/java/ru/vk/itmo/abramovilya/DaoIterator.java +++ b/src/main/java/ru/vk/itmo/abramovilya/DaoIterator.java @@ -7,9 +7,7 @@ import ru.vk.itmo.abramovilya.table.TableEntry; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; import java.util.Iterator; -import java.util.List; import java.util.NavigableMap; import java.util.NoSuchElementException; import java.util.PriorityQueue; @@ -18,19 +16,17 @@ class DaoIterator implements Iterator> { private final PriorityQueue priorityQueue = new PriorityQueue<>(); private final MemorySegment from; private final MemorySegment to; - private final List sstableMappedList; - private final List indexMappedList; + private final Storage storage; DaoIterator(int totalSStables, MemorySegment from, MemorySegment to, - List sstableMappedList, - List indexMappedList, + Storage storage, NavigableMap> memTable) { + this.from = from; this.to = to; - this.sstableMappedList = sstableMappedList; - this.indexMappedList = indexMappedList; + this.storage = storage; NavigableMap> subMap = getSubMap(memTable); for (int i = 0; i < totalSStables; i++) { @@ -39,8 +35,8 @@ class DaoIterator implements Iterator> { priorityQueue.add(new SSTable( i, offset, - sstableMappedList.get(i), - indexMappedList.get(i) + storage.mappedSStable(i), + storage.mappedIndex(i) ).currentEntry()); } } @@ -115,33 +111,7 @@ private void cleanUpSStableQueue() { } } - private long findOffsetInIndex(MemorySegment from, MemorySegment to, int i) { - long readOffset = 0; - MemorySegment storageMapped = sstableMappedList.get(i); - MemorySegment indexMapped = indexMappedList.get(i); - - if (from == null && to == null) { - return Integer.BYTES; - } else if (from == null) { - long firstKeySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, readOffset); - readOffset += Long.BYTES; - MemorySegment firstKey = storageMapped.asSlice(readOffset, firstKeySize); - if (DaoImpl.compareMemorySegments(firstKey, to) >= 0) { - return -1; - } - return Integer.BYTES; - } else { - int foundIndex = DaoImpl.upperBound(from, storageMapped, indexMapped, indexMapped.byteSize()); - long keyStorageOffset = DaoImpl.getKeyStorageOffset(indexMapped, foundIndex); - long keySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, keyStorageOffset); - keyStorageOffset += Long.BYTES; - - if (DaoImpl.compareMemorySegmentsUsingOffset(from, storageMapped, keyStorageOffset, keySize) > 0 - || (to != null && DaoImpl.compareMemorySegmentsUsingOffset( - to, storageMapped, keyStorageOffset, keySize) <= 0)) { - return -1; - } - return (long) foundIndex * (Integer.BYTES + Long.BYTES) + Integer.BYTES; - } + private long findOffsetInIndex(MemorySegment from, MemorySegment to, int fileNum) { + return storage.findOffsetInIndex(from, to, fileNum); } } diff --git a/src/main/java/ru/vk/itmo/abramovilya/Storage.java b/src/main/java/ru/vk/itmo/abramovilya/Storage.java new file mode 100644 index 000000000..abcb788dd --- /dev/null +++ b/src/main/java/ru/vk/itmo/abramovilya/Storage.java @@ -0,0 +1,293 @@ +package ru.vk.itmo.abramovilya; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Config; +import ru.vk.itmo.Entry; + +import java.io.Closeable; +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NavigableMap; + +class Storage implements Closeable { + private static final String COMPACTED_SUFFIX = "_compacted"; + private static final String COMPACTING_SUFFIX = "_compacting"; + private static final String SSTABLE_BASE_NAME = "storage"; + private static final String INDEX_BASE_NAME = "index"; + public static final String META_FILE_BASE_NAME = "meta"; + public static final int BYTES_TO_STORE_ENTRY_ELEMENT_SIZE = Long.BYTES; + public static final int BYTES_TO_STORE_ENTRY_SIZE = 2 * BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + public static final int BYTES_TO_STORE_INDEX_KEY = Integer.BYTES; + public static final long INDEX_ENTRY_SIZE = BYTES_TO_STORE_INDEX_KEY + BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + private final Path storagePath; + private final Path metaFilePath; + private final List sstableFileChannels = new ArrayList<>(); + private final List sstableMappedList = new ArrayList<>(); + private final List indexFileChannels = new ArrayList<>(); + private final List indexMappedList = new ArrayList<>(); + + Storage(Config config, Arena arena) throws IOException { + storagePath = config.basePath(); + + Files.createDirectories(storagePath); + metaFilePath = storagePath.resolve(META_FILE_BASE_NAME); + if (!Files.exists(metaFilePath)) { + Files.createFile(metaFilePath); + + int totalSStables = 0; + Files.writeString(metaFilePath, String.valueOf(totalSStables), StandardOpenOption.WRITE); + } + + // Restore consistent state if db was dropped during compaction + if (Files.exists(storagePath.resolve(SSTABLE_BASE_NAME + COMPACTED_SUFFIX)) + || Files.exists(storagePath.resolve(INDEX_BASE_NAME + COMPACTED_SUFFIX))) { + finishCompact(); + } + + // Delete artifacts from unsuccessful compaction + Files.deleteIfExists(storagePath.resolve(SSTABLE_BASE_NAME + COMPACTING_SUFFIX)); + Files.deleteIfExists(storagePath.resolve(INDEX_BASE_NAME + COMPACTING_SUFFIX)); + + int totalSSTables = Integer.parseInt(Files.readString(metaFilePath)); + for (int sstableNum = 0; sstableNum < totalSSTables; sstableNum++) { + Path sstablePath = storagePath.resolve(SSTABLE_BASE_NAME + sstableNum); + Path indexPath = storagePath.resolve(INDEX_BASE_NAME + sstableNum); + + FileChannel sstableFileChannel = FileChannel.open(sstablePath, StandardOpenOption.READ); + sstableFileChannels.add(sstableFileChannel); + MemorySegment sstableMapped = + sstableFileChannel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(sstablePath), arena); + sstableMappedList.add(sstableMapped); + + FileChannel indexFileChannel = FileChannel.open(indexPath, StandardOpenOption.READ); + indexFileChannels.add(indexFileChannel); + MemorySegment indexMapped = + indexFileChannel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(indexPath), arena); + indexMappedList.add(indexMapped); + } + } + + Entry get(MemorySegment key) { + int totalSStables = getTotalSStables(); + for (int sstableNum = totalSStables; sstableNum >= 0; sstableNum--) { + var foundEntry = seekForValueInFile(key, sstableNum); + if (foundEntry != null) { + if (foundEntry.value() != null) { + return foundEntry; + } + return null; + } + } + return null; + } + + final int getTotalSStables() { + return sstableFileChannels.size(); + } + + private Entry seekForValueInFile(MemorySegment key, int sstableNum) { + if (sstableNum >= sstableFileChannels.size()) { + return null; + } + + MemorySegment storageMapped = sstableMappedList.get(sstableNum); + MemorySegment indexMapped = indexMappedList.get(sstableNum); + + int foundIndex = upperBound(key, storageMapped, indexMapped, indexMapped.byteSize()); + long keyStorageOffset = getKeyStorageOffset(indexMapped, foundIndex); + long foundKeySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, keyStorageOffset); + keyStorageOffset += BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + + if (MemorySegment.mismatch(key, + 0, + key.byteSize(), + storageMapped, + keyStorageOffset, + keyStorageOffset + foundKeySize) == -1) { + return getEntryFromIndexFile(storageMapped, indexMapped, foundIndex); + } + return null; + } + + static long getKeyStorageOffset(MemorySegment indexMapped, int entryNum) { + return indexMapped.get( + ValueLayout.JAVA_LONG_UNALIGNED, + INDEX_ENTRY_SIZE * entryNum + BYTES_TO_STORE_INDEX_KEY + ); + } + + private Entry getEntryFromIndexFile(MemorySegment sstableMapped, + MemorySegment indexMapped, + int entryNum) { + long offsetInStorageFile = indexMapped.get( + ValueLayout.JAVA_LONG_UNALIGNED, + INDEX_ENTRY_SIZE * entryNum + BYTES_TO_STORE_INDEX_KEY + ); + + long keySize = sstableMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, offsetInStorageFile); + offsetInStorageFile += BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + offsetInStorageFile += keySize; + + long valueSize = sstableMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, offsetInStorageFile); + offsetInStorageFile += BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + MemorySegment key = sstableMapped.asSlice( + offsetInStorageFile - keySize - BYTES_TO_STORE_ENTRY_ELEMENT_SIZE, keySize); + MemorySegment value; + if (valueSize == -1) { + value = null; + } else { + value = sstableMapped.asSlice(offsetInStorageFile, valueSize); + } + return new BaseEntry<>(key, value); + } + + void writeMapIntoFile(long sstableSize, long indexSize, NavigableMap> map) + throws IOException { + + int totalSStables = getTotalSStables(); + Path sstablePath = storagePath.resolve(Storage.SSTABLE_BASE_NAME + totalSStables); + Path indexPath = storagePath.resolve(Storage.INDEX_BASE_NAME + totalSStables); + StorageFileWriter.writeMapIntoFile(sstableSize, indexSize, map, sstablePath, indexPath); + } + + private Entry calcCompactedSStableIndexSize(Iterator> iterator) { + long storageSize = 0; + long indexSize = 0; + while (iterator.hasNext()) { + Entry entry = iterator.next(); + storageSize += entry.key().byteSize() + entry.value().byteSize() + BYTES_TO_STORE_ENTRY_SIZE; + indexSize += INDEX_ENTRY_SIZE; + } + return new BaseEntry<>(storageSize, indexSize); + } + + void incTotalSStablesAmount() throws IOException { + int totalSStables = getTotalSStables(); + Files.writeString(metaFilePath, String.valueOf(totalSStables + 1)); + } + + @Override + public void close() throws IOException { + for (FileChannel fc : sstableFileChannels) { + if (fc.isOpen()) fc.close(); + } + for (FileChannel fc : indexFileChannels) { + if (fc.isOpen()) fc.close(); + } + } + + public MemorySegment mappedSStable(int i) { + return sstableMappedList.get(i); + } + + public MemorySegment mappedIndex(int i) { + return indexMappedList.get(i); + } + + void compact(Iterator> iterator1, Iterator> iterator2) + throws IOException { + Entry storageIndexSize = calcCompactedSStableIndexSize(iterator1); + Path compactingSStablePath = storagePath.resolve(SSTABLE_BASE_NAME + COMPACTING_SUFFIX); + Path compactingIndexPath = storagePath.resolve(INDEX_BASE_NAME + COMPACTING_SUFFIX); + StorageFileWriter.writeIteratorIntoFile(storageIndexSize.key(), + storageIndexSize.value(), + iterator2, + compactingSStablePath, + compactingIndexPath); + + // Move to ensure that compacting completed successfully + Path compactedSStablePath = storagePath.resolve(SSTABLE_BASE_NAME + COMPACTED_SUFFIX); + Path compactedIndexPath = storagePath.resolve(INDEX_BASE_NAME + COMPACTED_SUFFIX); + Files.move(compactingSStablePath, compactedSStablePath, StandardCopyOption.ATOMIC_MOVE); + Files.move(compactingIndexPath, compactedIndexPath, StandardCopyOption.ATOMIC_MOVE); + + finishCompact(); + } + + private void finishCompact() throws IOException { + int totalSStables = getTotalSStables(); + for (int i = 0; i < totalSStables; i++) { + Files.deleteIfExists(storagePath.resolve(SSTABLE_BASE_NAME + i)); + Files.deleteIfExists(storagePath.resolve(INDEX_BASE_NAME + i)); + } + + Files.writeString(metaFilePath, String.valueOf(1)); + Path compactedSStablePath = storagePath.resolve(SSTABLE_BASE_NAME + COMPACTED_SUFFIX); + if (Files.exists(compactedSStablePath)) { + Files.move(compactedSStablePath, + storagePath.resolve(SSTABLE_BASE_NAME + 0), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + Path compactedIndexPath = storagePath.resolve(INDEX_BASE_NAME + COMPACTED_SUFFIX); + + if (Files.exists(compactedIndexPath)) { + Files.move(compactedIndexPath, + storagePath.resolve(INDEX_BASE_NAME + 0), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + } + + long findOffsetInIndex(MemorySegment from, MemorySegment to, int fileNum) { + long readOffset = 0; + MemorySegment storageMapped = sstableMappedList.get(fileNum); + MemorySegment indexMapped = indexMappedList.get(fileNum); + + if (from == null && to == null) { + return BYTES_TO_STORE_INDEX_KEY; + } else if (from == null) { + long firstKeySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, readOffset); + readOffset += BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + MemorySegment firstKey = storageMapped.asSlice(readOffset, firstKeySize); + if (DaoImpl.compareMemorySegments(firstKey, to) >= 0) { + return -1; + } + return BYTES_TO_STORE_INDEX_KEY; + } else { + int foundIndex = upperBound(from, storageMapped, indexMapped, indexMapped.byteSize()); + long keyStorageOffset = getKeyStorageOffset(indexMapped, foundIndex); + long keySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, keyStorageOffset); + keyStorageOffset += BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + + if (DaoImpl.compareMemorySegmentsUsingOffset(from, storageMapped, keyStorageOffset, keySize) > 0 + || (to != null && DaoImpl.compareMemorySegmentsUsingOffset( + to, storageMapped, keyStorageOffset, keySize) <= 0)) { + return -1; + } + return (long) foundIndex * INDEX_ENTRY_SIZE + BYTES_TO_STORE_INDEX_KEY; + } + } + + private static int upperBound(MemorySegment key, + MemorySegment storageMapped, + MemorySegment indexMapped, + long indexSize) { + int l = -1; + int r = indexMapped.get(ValueLayout.JAVA_INT_UNALIGNED, indexSize - INDEX_ENTRY_SIZE); + + while (r - l > 1) { + int m = (r + l) / 2; + long keyStorageOffset = getKeyStorageOffset(indexMapped, m); + long keySize = storageMapped.get(ValueLayout.JAVA_LONG_UNALIGNED, keyStorageOffset); + keyStorageOffset += BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + + if (DaoImpl.compareMemorySegmentsUsingOffset(key, storageMapped, keyStorageOffset, keySize) > 0) { + l = m; + } else { + r = m; + } + } + return r; + } +} diff --git a/src/main/java/ru/vk/itmo/abramovilya/StorageFileWriter.java b/src/main/java/ru/vk/itmo/abramovilya/StorageFileWriter.java new file mode 100644 index 000000000..67dceed60 --- /dev/null +++ b/src/main/java/ru/vk/itmo/abramovilya/StorageFileWriter.java @@ -0,0 +1,149 @@ +package ru.vk.itmo.abramovilya; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Iterator; +import java.util.NavigableMap; + +final class StorageFileWriter { + + public static final ValueLayout.OfInt ENTRY_NUMBER_LAYOUT = ValueLayout.JAVA_INT_UNALIGNED; + public static final ValueLayout.OfLong MEMORY_SEGMENT_SIZE_LAYOUT = ValueLayout.JAVA_LONG_UNALIGNED; + + private StorageFileWriter() { + } + + static void writeIteratorIntoFile(long storageSize, + long indexSize, + Iterator> iterator, + Path sstablePath, + Path indexPath) throws IOException { + long storageWriteOffset = 0; + long indexWriteOffset = 0; + try (var storageChannel = FileChannel.open(sstablePath, + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE); + + var indexChannel = FileChannel.open(indexPath, + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE); + + var writeArena = Arena.ofConfined()) { + + MemorySegment mappedStorage = + storageChannel.map(FileChannel.MapMode.READ_WRITE, 0, storageSize, writeArena); + MemorySegment mappedIndex = + indexChannel.map(FileChannel.MapMode.READ_WRITE, 0, indexSize, writeArena); + + int entryNum = 0; + while (iterator.hasNext()) { + var entry = iterator.next(); + indexWriteOffset = + writeEntryNumAndStorageOffset(mappedIndex, indexWriteOffset, entryNum, storageWriteOffset); + entryNum++; + + storageWriteOffset = writeMemorySegment(entry.key(), mappedStorage, storageWriteOffset); + storageWriteOffset = writeMemorySegment(entry.value(), mappedStorage, storageWriteOffset); + } + mappedStorage.force(); + mappedIndex.force(); + } + } + + // writeMapIntoFile and writeIteratorInto file are pretty much the same, + // but I can't use writeIteratorIntoFile here due to optimization purposes: + // I have to write sstable and index separately + // I can't use writeMapIntoFile's code in the method above either, + // because it will slow down the execution due to the need of creating iterator twice + // And it also won't give any speed boost, + // because I would still be in need to find iterator.next() entry in another file + static void writeMapIntoFile(long sstableSize, + long indexSize, + NavigableMap> map, + Path sstablePath, + Path indexPath) throws IOException { + long storageWriteOffset = 0; + long indexWriteOffset = 0; + try (var storageChannel = FileChannel.open(sstablePath, + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE); + + var indexChannel = FileChannel.open(indexPath, + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE); + + var writeArena = Arena.ofConfined()) { + MemorySegment mappedIndex = + indexChannel.map(FileChannel.MapMode.READ_WRITE, 0, indexSize, writeArena); + + int entryNum = 0; + for (var entry : map.values()) { + indexWriteOffset = writeEntryNumAndStorageOffset( + mappedIndex, + indexWriteOffset, + entryNum, + storageWriteOffset + ); + entryNum++; + + storageWriteOffset += Storage.BYTES_TO_STORE_ENTRY_SIZE; + storageWriteOffset += entry.key().byteSize(); + if (entry.value() != null) { + storageWriteOffset += entry.value().byteSize(); + } + } + mappedIndex.force(); + + MemorySegment mappedStorage = + storageChannel.map(FileChannel.MapMode.READ_WRITE, 0, sstableSize, writeArena); + storageWriteOffset = 0; + for (var entry : map.values()) { + storageWriteOffset = writeMemorySegment(entry.key(), mappedStorage, storageWriteOffset); + storageWriteOffset = writeMemorySegment(entry.value(), mappedStorage, storageWriteOffset); + } + mappedStorage.force(); + } + } + + static long writeEntryNumAndStorageOffset(MemorySegment mappedIndex, + long indexWriteOffset, + int entryNum, + long storageWriteOffset) { + long offset = indexWriteOffset; + mappedIndex.set(ENTRY_NUMBER_LAYOUT, offset, entryNum); + offset += Storage.BYTES_TO_STORE_INDEX_KEY; + mappedIndex.set(MEMORY_SEGMENT_SIZE_LAYOUT, offset, storageWriteOffset); + offset += Storage.BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + return offset; + } + + // Every memorySegment in file has the following structure: + // 8 bytes - size, bytes - value + // If memorySegment has the size of -1 byte, then it means its value is DELETED + static long writeMemorySegment(MemorySegment memorySegment, MemorySegment mapped, long writeOffset) { + long offset = writeOffset; + if (memorySegment == null) { + mapped.set(MEMORY_SEGMENT_SIZE_LAYOUT, offset, -1); + offset += Storage.BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + } else { + long msSize = memorySegment.byteSize(); + mapped.set(MEMORY_SEGMENT_SIZE_LAYOUT, offset, msSize); + offset += Storage.BYTES_TO_STORE_ENTRY_ELEMENT_SIZE; + MemorySegment.copy(memorySegment, 0, mapped, offset, msSize); + offset += msSize; + } + return offset; + } + +} diff --git a/src/main/java/ru/vk/itmo/abramovilya/StorageWriter.java b/src/main/java/ru/vk/itmo/abramovilya/StorageWriter.java deleted file mode 100644 index 9f34bbdfa..000000000 --- a/src/main/java/ru/vk/itmo/abramovilya/StorageWriter.java +++ /dev/null @@ -1,96 +0,0 @@ -package ru.vk.itmo.abramovilya; - -import ru.vk.itmo.Entry; - -import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.NavigableMap; - -final class StorageWriter { - private StorageWriter() { - } - - static long writeEntryNumAndStorageOffset(MemorySegment mappedIndex, - long indexWriteOffset, - int entryNum, - long storageWriteOffset) { - long offset = indexWriteOffset; - mappedIndex.set(ValueLayout.JAVA_INT_UNALIGNED, offset, entryNum); - offset += Integer.BYTES; - mappedIndex.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, storageWriteOffset); - offset += Long.BYTES; - return offset; - } - - static long writeMemorySegment(MemorySegment memorySegment, MemorySegment mapped, long writeOffset) { - long offset = writeOffset; - if (memorySegment == null) { - mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, -1); - offset += Long.BYTES; - } else { - long msSize = memorySegment.byteSize(); - mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, msSize); - offset += Long.BYTES; - MemorySegment.copy(memorySegment, 0, mapped, offset, msSize); - offset += msSize; - } - return offset; - } - - // SSTable: |keySize: 8 bytes|key|valueSize: 8 bytes (size == -1 means value is deleted)|value| - // Index: |entryNum: 4 bytes|storageKeyOffset: 8 bytes| - static void writeSStableAndIndex(Path sstablePath, - long sstableSize, - Path indexPath, - long indexSize, - NavigableMap> map) throws IOException { - long storageWriteOffset = 0; - long indexWriteOffset = 0; - try (var storageChannel = FileChannel.open(sstablePath, - StandardOpenOption.READ, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE); - - var indexChannel = FileChannel.open(indexPath, - StandardOpenOption.READ, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE); - - var writeArena = Arena.ofConfined()) { - MemorySegment mappedIndex = - indexChannel.map(FileChannel.MapMode.READ_WRITE, 0, indexSize, writeArena); - - int entryNum = 0; - for (var entry : map.values()) { - indexWriteOffset = StorageWriter.writeEntryNumAndStorageOffset( - mappedIndex, - indexWriteOffset, - entryNum, - storageWriteOffset - ); - entryNum++; - - storageWriteOffset += 2 * Long.BYTES; - storageWriteOffset += entry.key().byteSize(); - if (entry.value() != null) { - storageWriteOffset += entry.value().byteSize(); - } - } - mappedIndex.force(); - - MemorySegment mappedStorage = - storageChannel.map(FileChannel.MapMode.READ_WRITE, 0, sstableSize, writeArena); - storageWriteOffset = 0; - for (var entry : map.values()) { - storageWriteOffset = StorageWriter.writeMemorySegment(entry.key(), mappedStorage, storageWriteOffset); - storageWriteOffset = StorageWriter.writeMemorySegment(entry.value(), mappedStorage, storageWriteOffset); - } - mappedStorage.force(); - } - } -} diff --git a/src/main/java/ru/vk/itmo/bandurinvladislav/InMemoryDao.java b/src/main/java/ru/vk/itmo/bandurinvladislav/InMemoryDao.java deleted file mode 100644 index c06bbfa9f..000000000 --- a/src/main/java/ru/vk/itmo/bandurinvladislav/InMemoryDao.java +++ /dev/null @@ -1,59 +0,0 @@ -package ru.vk.itmo.bandurinvladislav; - -import ru.vk.itmo.Dao; -import ru.vk.itmo.Entry; - -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.util.Comparator; -import java.util.Iterator; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; - -public class InMemoryDao implements Dao> { - private static final Comparator MEMORY_SEGMENT_COMPARATOR = (m1, m2) -> { - long mismatch = m1.mismatch(m2); - if (mismatch == m2.byteSize()) { - return 1; - } else if (mismatch == m1.byteSize()) { - return -1; - } else if (mismatch == -1) { - return 0; - } else { - return m1.get(ValueLayout.JAVA_BYTE, mismatch) - m2.get(ValueLayout.JAVA_BYTE, mismatch); - } - }; - - private final ConcurrentNavigableMap> inMemoryStorage; - - public InMemoryDao() { - inMemoryStorage = new ConcurrentSkipListMap<>(MEMORY_SEGMENT_COMPARATOR); - } - - @Override - public Iterator> get(MemorySegment from, MemorySegment to) { - return getEntryIterator(from, to); - } - - @Override - public Entry get(MemorySegment key) { - return inMemoryStorage.get(key); - } - - @Override - public void upsert(Entry entry) { - inMemoryStorage.put(entry.key(), entry); - } - - private Iterator> getEntryIterator(MemorySegment from, MemorySegment to) { - if (from == null && to == null) { - return inMemoryStorage.values().iterator(); - } else if (from == null) { - return inMemoryStorage.headMap(to, false).values().iterator(); - } else if (to == null) { - return inMemoryStorage.tailMap(from, true).values().iterator(); - } else { - return inMemoryStorage.subMap(from, true, to, false).values().iterator(); - } - } -} diff --git a/src/main/java/ru/vk/itmo/bazhenovkirill/MemorySegmentUtils.java b/src/main/java/ru/vk/itmo/bazhenovkirill/MemorySegmentUtils.java new file mode 100644 index 000000000..487da9450 --- /dev/null +++ b/src/main/java/ru/vk/itmo/bazhenovkirill/MemorySegmentUtils.java @@ -0,0 +1,72 @@ +package ru.vk.itmo.bazhenovkirill; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +public final class MemorySegmentUtils { + + private MemorySegmentUtils() { + + } + + public static MemorySegment getSlice(MemorySegment segment, long start, long end) { + return segment.asSlice(start, end - start); + } + + public static long startOfKey(MemorySegment segment, long inx) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, inx * 2 * Long.BYTES); + } + + public static long endOfKey(MemorySegment segment, long inx) { + return normalize(startOfValue(segment, inx)); + } + + public static MemorySegment getKey(MemorySegment segment, long inx) { + return getSlice(segment, startOfKey(segment, inx), endOfKey(segment, inx)); + } + + public static long startOfValue(MemorySegment segment, long inx) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, Long.BYTES + inx * 2 * Long.BYTES); + } + + public static long endOfValue(MemorySegment segment, long inx) { + if (inx < recordsCount(segment) - 1) { + return startOfKey(segment, inx + 1); + } + return segment.byteSize(); + } + + public static Entry getEntry(MemorySegment segment, long inx) { + MemorySegment key = getKey(segment, inx); + MemorySegment value = getValue(segment, inx); + return new BaseEntry<>(key, value); + } + + public static MemorySegment getValue(MemorySegment segment, long inx) { + long start = startOfValue(segment, inx); + if (start < 0) { + return null; + } + return getSlice(segment, start, endOfValue(segment, inx)); + } + + public static long tombstone(long offset) { + return 1L << 63 | offset; + } + + public static long normalize(long offset) { + return offset & ~(1L << 63); + } + + public static long recordsCount(MemorySegment segment) { + return indexSize(segment) / (2 * Long.BYTES); + } + + public static long indexSize(MemorySegment segment) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + +} diff --git a/src/main/java/ru/vk/itmo/bazhenovkirill/MergeIterator.java b/src/main/java/ru/vk/itmo/bazhenovkirill/MergeIterator.java new file mode 100644 index 000000000..4d047a82c --- /dev/null +++ b/src/main/java/ru/vk/itmo/bazhenovkirill/MergeIterator.java @@ -0,0 +1,123 @@ +package ru.vk.itmo.bazhenovkirill; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +public class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + + private final Comparator comparator; + + private PeekIterator peek; + + private static class PeekIterator implements Iterator { + + private final Iterator iterator; + private T next; + private final int id; + + public PeekIterator(int id, Iterator iterator) { + this.id = id; + this.iterator = iterator; + if (iterator.hasNext()) { + next = iterator.next(); + } + } + + private T peek() { + return next; + } + + @Override + public boolean hasNext() { + return next != null || iterator.hasNext(); + } + + @Override + public T next() { + T curr = next; + next = iterator.hasNext() ? iterator.next() : null; + return curr; + } + } + + public MergeIterator(Collection> iterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + protected boolean skip(T t) { + return t == null; + } + + private PeekIterator peek() { + while (peek == null) { + peek = priorityQueue.poll(); + if (peek == null) { + return null; + } + + PeekIterator next = priorityQueue.peek(); + while (next != null && comparator.compare(peek.peek(), next.peek()) == 0) { + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + skipElement(poll); + } + next = priorityQueue.peek(); + } + + if (!peek.hasNext()) { + peek = null; + continue; + } + + if (skip(peek.peek())) { + skipElement(peek); + peek = null; + } + } + + return peek; + } + + private void skipElement(PeekIterator iterator) { + iterator.next(); + if (iterator.hasNext()) { + priorityQueue.add(iterator); + } + } + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator peekIterator = peek(); + if (peekIterator == null) { + throw new NoSuchElementException(); + } + T next = peek.next(); + this.peek = null; + if (peekIterator.hasNext()) { + priorityQueue.add(peekIterator); + } + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/bazhenovkirill/Offset.java b/src/main/java/ru/vk/itmo/bazhenovkirill/Offset.java new file mode 100644 index 000000000..7f65af2b6 --- /dev/null +++ b/src/main/java/ru/vk/itmo/bazhenovkirill/Offset.java @@ -0,0 +1,11 @@ +package ru.vk.itmo.bazhenovkirill; + +public class Offset { + long data; + long index; + + Offset(long data, long index) { + this.data = data; + this.index = index; + } +} diff --git a/src/main/java/ru/vk/itmo/bazhenovkirill/PersistentDaoImpl.java b/src/main/java/ru/vk/itmo/bazhenovkirill/PersistentDaoImpl.java index f0e4b770e..a7e24a1e1 100644 --- a/src/main/java/ru/vk/itmo/bazhenovkirill/PersistentDaoImpl.java +++ b/src/main/java/ru/vk/itmo/bazhenovkirill/PersistentDaoImpl.java @@ -1,94 +1,48 @@ package ru.vk.itmo.bazhenovkirill; -import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import java.io.FileNotFoundException; import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.channels.FileChannel.MapMode; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.Iterator; -import java.util.Set; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; public class PersistentDaoImpl implements Dao> { - - private static final String DATA_FILE = "sstable.db"; - - private static final Set WRITE_OPTIONS = Set.of( - StandardOpenOption.CREATE, - StandardOpenOption.READ, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.WRITE - ); - private final ConcurrentNavigableMap> memTable = new ConcurrentSkipListMap<>(new MemorySegmentComparator()); - - private final Path dataPath; - private final Arena arena; - private MemorySegment mappedMS; + private final Path dataPath; + private final Storage storage; public PersistentDaoImpl(Config config) throws IOException { - dataPath = config.basePath().resolve(DATA_FILE); - arena = Arena.ofShared(); - + dataPath = config.basePath().resolve("data"); if (!Files.exists(dataPath)) { - mappedMS = MemorySegment.NULL; - if (!arena.scope().isAlive()) { - arena.close(); - } - return; - } - - boolean segmentMapped = false; - try (FileChannel channel = FileChannel.open(dataPath, StandardOpenOption.READ)) { - mappedMS = channel.map(MapMode.READ_ONLY, - 0, channel.size(), arena).asReadOnly(); - segmentMapped = true; - } catch (FileNotFoundException e) { - mappedMS = MemorySegment.NULL; - } finally { - if (!segmentMapped) { - arena.close(); - } + Files.createDirectories(dataPath); } + arena = Arena.ofShared(); + storage = new Storage(Storage.loadData(dataPath, arena)); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { - if (from == null) { - if (to != null) { - return memTable.headMap(to).values().iterator(); - } - return memTable.values().iterator(); - } else { - if (to == null) { - return memTable.tailMap(from).values().iterator(); - } - return memTable.subMap(from, true, to, false).values().iterator(); - } + return storage.range(getInMemTable(from, to), from, to); } @Override public Entry get(MemorySegment key) { - Entry value = memTable.get(key); - if (value == null) { - return getDataFromSSTable(key); + Entry entry = memTable.get(key); + if (entry == null) { + return storage.get(key); } - return value; + return entry.value() == null ? null : entry; } @Override @@ -98,77 +52,38 @@ public void upsert(Entry entry) { @Override public void flush() throws IOException { - try (FileChannel channel = FileChannel.open(dataPath, WRITE_OPTIONS)) { - try (Arena confinedArena = Arena.ofConfined()) { - MemorySegment dataMemorySegment = channel.map(MapMode.READ_WRITE, 0, - getMemTableSizeInBytes(), confinedArena); - long offset = 0; - for (var entry : memTable.values()) { - offset = writeEntry(entry, dataMemorySegment, offset); - } - } + if (!memTable.isEmpty()) { + Storage.save(dataPath, memTable.values()); + } + } + + @Override + public void compact() throws IOException { + if (storage.compact(dataPath, this::all)) { + memTable.clear(); } } @Override public void close() throws IOException { + flush(); if (!arena.scope().isAlive()) { return; } arena.close(); - - flush(); } - private Entry getDataFromSSTable(MemorySegment key) { - long offset = 0; - long valueSize = 0; - while (offset < mappedMS.byteSize()) { - long keySize = mappedMS.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; - - if (keySize == key.byteSize()) { - MemorySegment possibleKey = mappedMS.asSlice(offset, keySize); - - valueSize = mappedMS.get(ValueLayout.JAVA_LONG_UNALIGNED, offset + keySize); - offset += (keySize + Long.BYTES); - if (key.mismatch(possibleKey) == -1) { - MemorySegment value = mappedMS.asSlice(offset, valueSize); - return new BaseEntry<>(possibleKey, value); - } - } else { - offset += keySize; - valueSize = mappedMS.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; + private Iterator> getInMemTable(MemorySegment from, MemorySegment to) { + if (from == null) { + if (to != null) { + return memTable.headMap(to).values().iterator(); } - - offset += valueSize; - } - return null; - } - - private long getMemTableSizeInBytes() { - long size = memTable.size() * Long.BYTES * 2L; - for (var entry : memTable.values()) { - size += entry.key().byteSize() + entry.value().byteSize(); + return memTable.values().iterator(); + } else { + if (to == null) { + return memTable.tailMap(from).values().iterator(); + } + return memTable.subMap(from, true, to, false).values().iterator(); } - return size; - } - - private long writeEntry(Entry entry, MemorySegment destination, long offset) { - long updatedOffset = writeDataToMemorySegment(entry.key(), destination, offset); - return writeDataToMemorySegment(entry.value(), destination, updatedOffset); - } - - private long writeDataToMemorySegment(MemorySegment entryPart, MemorySegment destination, long offset) { - long currentOffset = offset; - - destination.set(ValueLayout.JAVA_LONG_UNALIGNED, currentOffset, entryPart.byteSize()); - currentOffset += Long.BYTES; - - MemorySegment.copy(entryPart, 0, destination, currentOffset, entryPart.byteSize()); - currentOffset += entryPart.byteSize(); - - return currentOffset; } } diff --git a/src/main/java/ru/vk/itmo/bazhenovkirill/Storage.java b/src/main/java/ru/vk/itmo/bazhenovkirill/Storage.java new file mode 100644 index 000000000..636ab732e --- /dev/null +++ b/src/main/java/ru/vk/itmo/bazhenovkirill/Storage.java @@ -0,0 +1,267 @@ +package ru.vk.itmo.bazhenovkirill; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +public class Storage { + + private static final AtomicInteger SSTABLE_ID = new AtomicInteger(); + private static final MemorySegmentComparator comparator = new MemorySegmentComparator(); + private static final String INDEX_FILE_NAME = "index.db"; + private static final Set WRITE_OPTIONS = Set.of( + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.READ + ); + private final List segments; + + public Storage(List segments) { + this.segments = segments; + } + + /* + dataFile: + index -> |key0_offset|value0_offset|...|keyn_offset|valuen_offset| + data -> |ke0|value0|...|keyn|valuen| + index, data in one file + */ + public static void save(Path dataPath, Iterable> values) throws IOException { + Path indexFile = createOrMapIndexFile(dataPath); + List existedFiles = Files.readAllLines(indexFile); + + String fileName = String.valueOf(SSTABLE_ID.incrementAndGet()); + writeDataToSSTable(dataPath.resolve(fileName), values); + + existedFiles.add(fileName); + Files.write(indexFile, + existedFiles, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + } + + public boolean compact(Path dataPath, Iterable> values) throws IOException { + Path indexFile = createOrMapIndexFile(dataPath); + List existedFiles = Files.readAllLines(indexFile); + if (existedFiles.isEmpty()) { + return false; + } + + Path compactionTmpFile = getCompactionPath(dataPath); + + compactData(compactionTmpFile, values); + finalizeCompaction(dataPath, existedFiles); + return true; + } + + private void finalizeCompaction(Path dataPath, List existedFiles) throws IOException { + Path compactionTmpFile = getCompactionPath(dataPath); + if (Files.size(compactionTmpFile) == 0) { + return; + } + + for (String name : existedFiles) { + Files.deleteIfExists(dataPath.resolve(name)); + } + + Path indexFile = createOrMapIndexFile(dataPath); + String newSSTableName = String.valueOf(SSTABLE_ID.incrementAndGet()); + Files.write( + indexFile, + Collections.singleton(newSSTableName), + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.move(compactionTmpFile, dataPath.resolve(newSSTableName), StandardCopyOption.ATOMIC_MOVE); + } + + private Path getCompactionPath(Path dataPath) { + return dataPath.resolve("compaction.tmp"); + } + + private void compactData(Path ssTablePath, Iterable> values) throws IOException { + long dataSize = 0; + long entriesCount = 0; + + for (Entry entry : values) { + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + dataSize += (value == null) ? 0 : value.byteSize(); + entriesCount++; + } + long indexSize = 2 * Long.BYTES * entriesCount; + + try (FileChannel channel = FileChannel.open(ssTablePath, WRITE_OPTIONS); + Arena arena = Arena.ofConfined()) { + MemorySegment segment = channel.map(FileChannel.MapMode.READ_WRITE, + 0, + dataSize + indexSize, + arena); + + Offset offset = new Offset(indexSize, 0); + for (Entry entry : values) { + writeEntry(entry, segment, offset); + + } + } + } + + private static void writeDataToSSTable(Path ssTablePath, Iterable> values) throws IOException { + long dataSize = 0; + long entriesCount = 0; + for (Entry entry : values) { + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + dataSize += (value == null) ? 0 : value.byteSize(); + entriesCount++; + } + long indexSize = 2 * Long.BYTES * entriesCount; + + try (FileChannel channel = FileChannel.open(ssTablePath, WRITE_OPTIONS); + Arena arena = Arena.ofConfined()) { + MemorySegment segment = channel.map(FileChannel.MapMode.READ_WRITE, + 0, + dataSize + indexSize, + arena); + + Offset offset = new Offset(indexSize, 0); + for (Entry entry : values) { + writeEntry(entry, segment, offset); + } + } + } + + private static void writeEntry(Entry entry, MemorySegment segment, Offset offset) { + MemorySegment key = entry.key(); + segment.set(ValueLayout.JAVA_LONG_UNALIGNED, offset.index, offset.data); + MemorySegment.copy(key, 0, segment, offset.data, key.byteSize()); + offset.data += key.byteSize(); + offset.index += Long.BYTES; + + MemorySegment value = entry.value(); + if (value == null) { + segment.set(ValueLayout.JAVA_LONG_UNALIGNED, offset.index, MemorySegmentUtils.tombstone(offset.data)); + } else { + segment.set(ValueLayout.JAVA_LONG_UNALIGNED, offset.index, offset.data); + MemorySegment.copy(value, 0, segment, offset.data, value.byteSize()); + offset.data += value.byteSize(); + } + offset.index += Long.BYTES; + } + + public static List loadData(Path dataPath, Arena arena) throws IOException { + Path indexFile = createOrMapIndexFile(dataPath); + + List existedFiles = Files.readAllLines(indexFile); + List segments = new ArrayList<>(existedFiles.size()); + for (String name : existedFiles) { + Path dataFile = dataPath.resolve(name); + try (FileChannel channel = FileChannel.open(dataFile, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + MemorySegment segment = channel.map(FileChannel.MapMode.READ_WRITE, 0, channel.size(), arena); + segments.add(segment); + } + } + + return segments; + } + + public Entry get(MemorySegment key) { + for (int i = segments.size() - 1; i >= 0; --i) { + long index = indexOf(segments.get(i), key); + if (index >= 0) { + Entry entry = MemorySegmentUtils.getEntry(segments.get(i), index); + return entry.value() == null ? null : entry; + } + } + return null; + } + + public Iterator> range(Iterator> inMemoryIterator, + MemorySegment from, + MemorySegment to) { + List>> iterators = new ArrayList<>(segments.size() + 1); + for (MemorySegment segment : segments) { + iterators.add(iterator(segment, from, to)); + } + iterators.add(inMemoryIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, comparator)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + private static Iterator> iterator(MemorySegment segment, + MemorySegment from, + MemorySegment to) { + long size = MemorySegmentUtils.recordsCount(segment); + long start = (from == null) ? 0 : MemorySegmentUtils.normalize(indexOf(segment, from)); + long end = (to == null) ? size : MemorySegmentUtils.normalize(indexOf(segment, to)); + + return new Iterator<>() { + long inx = start; + + @Override + public boolean hasNext() { + return inx < end; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return MemorySegmentUtils.getEntry(segment, inx++); + } + }; + } + + private static long indexOf(MemorySegment segment, MemorySegment key) { + long l = 0; + long r = MemorySegmentUtils.recordsCount(segment) - 1; + while (l <= r) { + long mid = (l + r) >>> 1; + int resultOfComparing = comparator.compare(key, MemorySegmentUtils.getKey(segment, mid)); + if (resultOfComparing == 0) { + return mid; + } else if (resultOfComparing < 0) { + r = mid - 1; + } else { + l = mid + 1; + } + } + return MemorySegmentUtils.tombstone(l); + } + + private static Path createOrMapIndexFile(Path dataPath) throws IOException { + Path indexFile = dataPath.resolve(INDEX_FILE_NAME); + + if (!Files.exists(indexFile)) { + Files.createFile(indexFile); + } + + return indexFile; + } +} diff --git a/src/main/java/ru/vk/itmo/bandurinvladislav/DiskStorage.java b/src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorage.java similarity index 69% rename from src/main/java/ru/vk/itmo/bandurinvladislav/DiskStorage.java rename to src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorage.java index 8c7aa561a..e03c8fcfc 100644 --- a/src/main/java/ru/vk/itmo/bandurinvladislav/DiskStorage.java +++ b/src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorage.java @@ -1,4 +1,4 @@ -package ru.vk.itmo.bandurinvladislav; +package ru.vk.itmo.belonogovnikolay; import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Entry; @@ -15,16 +15,16 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; public class DiskStorage { - private static final String INDEX_FILE_NAME = "index.idx"; - private static final String INDEX_TMP_FILE_NAME = "index.tmp"; - private final List segmentList; + private static final String INDEX_FILE_NAME = "index.idx"; + private static final String INDEX_TEMP_FILE_NAME = "index.tmp"; public DiskStorage(List segmentList) { this.segmentList = segmentList; @@ -40,7 +40,7 @@ public Iterator> range( } iterators.add(firstIterator); - return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, PersistentDao::compare)) { + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, InMemoryTreeDao::compare)) { @Override protected boolean skip(Entry memorySegmentEntry) { return memorySegmentEntry.value() == null; @@ -48,9 +48,12 @@ protected boolean skip(Entry memorySegmentEntry) { }; } - public static void save(Path storagePath, Iterable> iterable) + public static boolean save(Path storagePath, Iterable> iterable) throws IOException { - final Path indexTmp = storagePath.resolve(INDEX_TMP_FILE_NAME); + if (!iterable.iterator().hasNext()) { + return false; + } + final Path indexTmp = storagePath.resolve(INDEX_TEMP_FILE_NAME); final Path indexFile = storagePath.resolve(INDEX_FILE_NAME); try { @@ -62,62 +65,6 @@ public static void save(Path storagePath, Iterable> iterabl String newFileName = String.valueOf(existedFiles.size()); - fillSSTable(storagePath.resolve(newFileName), iterable); - - Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - - List list = new ArrayList<>(existedFiles.size() + 1); - list.addAll(existedFiles); - list.add(newFileName); - Files.write( - indexFile, - list, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING - ); - - Files.delete(indexTmp); - } - - public void compact(Path storagePath, Iterator> mergeIterator) throws IOException { - Path indexFile = storagePath.resolve(INDEX_FILE_NAME); - - if (!Files.exists(indexFile)) { - throw new IllegalStateException("Unexpected missing file index.idx"); - } - - Path newFilePath = storagePath.resolve("compactedTable"); - - ArrayList> compactedValues = new ArrayList<>(); - while (mergeIterator.hasNext()) { - compactedValues.add(mergeIterator.next()); - } - - List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); - fillSSTable(newFilePath, compactedValues); - - for (String existedFile : existedFiles) { - Files.deleteIfExists(storagePath.resolve(existedFile)); - } - - Files.move(newFilePath, - storagePath.resolve("0"), - StandardCopyOption.ATOMIC_MOVE, - StandardCopyOption.REPLACE_EXISTING); - Files.deleteIfExists(newFilePath); - - Files.writeString( - indexFile, - "0", - StandardOpenOption.WRITE, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING - ); - } - - private static void fillSSTable(Path newFilePath, Iterable> iterable) - throws IOException { long dataSize = 0; long count = 0; for (Entry entry : iterable) { @@ -132,7 +79,7 @@ private static void fillSSTable(Path newFilePath, Iterable> try ( FileChannel fileChannel = FileChannel.open( - newFilePath, + storagePath.resolve(newFileName), StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE @@ -142,13 +89,10 @@ private static void fillSSTable(Path newFilePath, Iterable> MemorySegment fileSegment = fileChannel.map( FileChannel.MapMode.READ_WRITE, 0, - indexSize == 0 ? Long.BYTES : indexSize + dataSize, + indexSize + dataSize, writeArena ); - // index: - // |key0_Start|value0_Start|key1_Start|value1_Start|key2_Start|value2_Start|... - // key0_Start = data start = end of index long dataOffset = indexSize; int indexOffset = 0; for (Entry entry : iterable) { @@ -158,7 +102,8 @@ private static void fillSSTable(Path newFilePath, Iterable> MemorySegment value = entry.value(); if (value == null) { - fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, StorageUtil.tombstone(dataOffset)); + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, + DiskStorageHelper.tombstone(dataOffset)); } else { fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); dataOffset += value.byteSize(); @@ -166,8 +111,6 @@ private static void fillSSTable(Path newFilePath, Iterable> indexOffset += Long.BYTES; } - // data: - // |key0|value0|key1|value1|... dataOffset = indexSize; for (Entry entry : iterable) { MemorySegment key = entry.key(); @@ -181,10 +124,26 @@ private static void fillSSTable(Path newFilePath, Iterable> } } } + + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + + List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + list.add(newFileName); + Files.write( + indexFile, + list, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.delete(indexTmp); + return true; } public static List loadOrRecover(Path storagePath, Arena arena) throws IOException { - Path indexTmp = storagePath.resolve(INDEX_TMP_FILE_NAME); + Path indexTmp = storagePath.resolve(INDEX_TEMP_FILE_NAME); Path indexFile = storagePath.resolve(INDEX_FILE_NAME); if (Files.exists(indexTmp)) { @@ -216,14 +175,13 @@ public static List loadOrRecover(Path storagePath, Arena arena) t } private static Iterator> iterator(MemorySegment page, MemorySegment from, MemorySegment to) { - long recordIndexFrom = from == null ? 0 : StorageUtil.normalize(StorageUtil.indexOf(page, from)); - long recordIndexTo = to == null - ? StorageUtil.recordsCount(page) - : StorageUtil.normalize(StorageUtil.indexOf(page, to)); - long recordsCount = StorageUtil.recordsCount(page); + long recordIndexFrom = from == null ? 0 : DiskStorageHelper.normalize(DiskStorageHelper.indexOf(page, from)); + long recordIndexTo = to == null ? DiskStorageHelper.recordsCount(page) + : DiskStorageHelper.normalize(DiskStorageHelper.indexOf(page, to)); + long recordsCount = DiskStorageHelper.recordsCount(page); return new Iterator<>() { - long index = recordIndexFrom; + private long index = recordIndexFrom; @Override public boolean hasNext() { @@ -235,17 +193,58 @@ public Entry next() { if (!hasNext()) { throw new NoSuchElementException(); } - MemorySegment key = StorageUtil - .slice(page, StorageUtil.startOfKey(page, index), StorageUtil.endOfKey(page, index)); - long startOfValue = StorageUtil.startOfValue(page, index); + MemorySegment key = DiskStorageHelper.slice(page, DiskStorageHelper.startOfKey(page, index), + DiskStorageHelper.endOfKey(page, index)); + long startOfValue = DiskStorageHelper.startOfValue(page, index); MemorySegment value = startOfValue < 0 - ? null - : StorageUtil.slice(page, startOfValue, StorageUtil.endOfValue(page, index, recordsCount)); + ? null + : DiskStorageHelper.slice(page, startOfValue, + DiskStorageHelper.endOfValue(page, index, recordsCount)); index++; return new BaseEntry<>(key, value); } }; } + public static void compact(Path storagePath, Iterable> iterable) throws IOException { + if (!save(storagePath, iterable)) { + return; + } + + Path indexTmp = storagePath.resolve(INDEX_TEMP_FILE_NAME); + Path indexFile = storagePath.resolve(INDEX_FILE_NAME); + + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + } + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + int filesCount = existedFiles.size(); + + if (filesCount >= 2) { + for (int i = 0; i < filesCount - 1; i++) { + Path file = storagePath.resolve(existedFiles.get(i)); + if (!Files.deleteIfExists(file)) { + break; + } + } + + Files.write( + indexFile, + Collections.singletonList("0"), + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + Files.move(storagePath.resolve(existedFiles.get(filesCount - 1)), storagePath.resolve("0"), + StandardCopyOption.ATOMIC_MOVE); + } + } } diff --git a/src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorageHelper.java b/src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorageHelper.java new file mode 100644 index 000000000..f3abf747c --- /dev/null +++ b/src/main/java/ru/vk/itmo/belonogovnikolay/DiskStorageHelper.java @@ -0,0 +1,92 @@ +package ru.vk.itmo.belonogovnikolay; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +public final class DiskStorageHelper { + + private DiskStorageHelper() { + + } + + static long endOfKey(MemorySegment segment, long recordIndex) { + return normalizedStartOfValue(segment, recordIndex); + } + + static long indexSize(MemorySegment segment) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + + static long recordsCount(MemorySegment segment) { + long indexSize = indexSize(segment); + return indexSize / Long.BYTES / 2; + } + + static long indexOf(MemorySegment segment, MemorySegment key) { + long recordsCount = recordsCount(segment); + + long left = 0; + long right = recordsCount - 1; + while (left <= right) { + long mid = (left + right) >>> 1; + + long startOfKey = DiskStorageHelper.startOfKey(segment, mid); + long endOfKey = DiskStorageHelper.endOfKey(segment, mid); + long mismatch = MemorySegment.mismatch(segment, startOfKey, endOfKey, key, 0, key.byteSize()); + if (mismatch == -1) { + return mid; + } + + if (mismatch == key.byteSize()) { + right = mid - 1; + continue; + } + + if (mismatch == endOfKey - startOfKey) { + left = mid + 1; + continue; + } + + int b1 = Byte.toUnsignedInt(segment.get(ValueLayout.JAVA_BYTE, startOfKey + mismatch)); + int b2 = Byte.toUnsignedInt(key.get(ValueLayout.JAVA_BYTE, mismatch)); + if (b1 > b2) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + return DiskStorageHelper.tombstone(left); + } + + static long normalizedStartOfValue(MemorySegment segment, long recordIndex) { + return normalize(startOfValue(segment, recordIndex)); + } + + static long startOfValue(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES + Long.BYTES); + } + + static long startOfKey(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES); + } + + static long endOfValue(MemorySegment segment, long recordIndex, long recordsCount) { + if (recordIndex < recordsCount - 1) { + return startOfKey(segment, recordIndex + 1); + } + return segment.byteSize(); + } + + static long tombstone(long offset) { + return 1L << 63 | offset; + } + + static long normalize(long value) { + return value & ~(1L << 63); + } + + static MemorySegment slice(MemorySegment page, long start, long end) { + return page.asSlice(start, end - start); + } +} diff --git a/src/main/java/ru/vk/itmo/belonogovnikolay/FileType.java b/src/main/java/ru/vk/itmo/belonogovnikolay/FileType.java deleted file mode 100644 index 7709ee335..000000000 --- a/src/main/java/ru/vk/itmo/belonogovnikolay/FileType.java +++ /dev/null @@ -1,25 +0,0 @@ -package ru.vk.itmo.belonogovnikolay; - -/** - * The class represents an enumeration of two file types. `data-file` stores data that is written when reconnecting, - * disconnecting, etc. `offset-file` contains data about data offsets in `data-file`. - * @author Belonogov Nikolay - */ - -public enum FileType { - - DATA("data-file"), - - OFFSET("offset-file"); - - private final String fileName; - - FileType(String name) { - this.fileName = name; - } - - @Override - public String toString() { - return this.fileName; - } -} diff --git a/src/main/java/ru/vk/itmo/belonogovnikolay/InMemoryTreeDao.java b/src/main/java/ru/vk/itmo/belonogovnikolay/InMemoryTreeDao.java index 6985eb368..c842aa40f 100644 --- a/src/main/java/ru/vk/itmo/belonogovnikolay/InMemoryTreeDao.java +++ b/src/main/java/ru/vk/itmo/belonogovnikolay/InMemoryTreeDao.java @@ -5,84 +5,117 @@ import ru.vk.itmo.Entry; import java.io.IOException; +import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Comparator; import java.util.Iterator; import java.util.NavigableMap; import java.util.concurrent.ConcurrentSkipListMap; -/** - * The class is an implementation of in memory persistent DAO. - * - * @author Belonogov Nikolay - */ -public final class InMemoryTreeDao implements Dao> { +public class InMemoryTreeDao implements Dao> { - private final NavigableMap> memTable; - private PersistenceHelper persistenceHelper; + private final Comparator comparator = InMemoryTreeDao::compare; + private NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); + private final Arena arena; + private final DiskStorage diskStorage; + private final Path path; - private InMemoryTreeDao() { - this.memTable = new ConcurrentSkipListMap<>(new MemorySegmentComparator()); - } - - private InMemoryTreeDao(Config config) { - this(); - this.persistenceHelper = PersistenceHelper.newInstance(config.basePath()); - } + public InMemoryTreeDao(Config config) throws IOException { + this.path = config.basePath().resolve("data"); + Files.createDirectories(path); - public static Dao> newInstance() { - return new InMemoryTreeDao(); - } + arena = Arena.ofShared(); - public static Dao> newInstance(Config config) { - return new InMemoryTreeDao(config); + this.diskStorage = new DiskStorage(DiskStorage.loadOrRecover(path, arena)); } @Override - public Iterator> allFrom(MemorySegment from) { - return this.memTable.tailMap(from).values().iterator(); + public void compact() throws IOException { + DiskStorage.compact(path, () -> diskStorage.range(getInMemory(null, null), null, null)); + + if (this.arena.scope().isAlive()) { + arena.close(); + } + storage = new ConcurrentSkipListMap<>(comparator); } - @Override - public Iterator> allTo(MemorySegment to) { - return this.memTable.headMap(to).values().iterator(); + static int compare(MemorySegment memorySegment1, MemorySegment memorySegment2) { + long mismatch = memorySegment1.mismatch(memorySegment2); + if (mismatch == -1) { + return 0; + } + + if (mismatch == memorySegment1.byteSize()) { + return -1; + } + + if (mismatch == memorySegment2.byteSize()) { + return 1; + } + byte b1 = memorySegment1.get(ValueLayout.JAVA_BYTE, mismatch); + byte b2 = memorySegment2.get(ValueLayout.JAVA_BYTE, mismatch); + return Byte.compare(b1, b2); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { + return diskStorage.range(getInMemory(from, to), from, to); + } + + private Iterator> getInMemory(MemorySegment from, MemorySegment to) { if (from == null && to == null) { - return this.memTable.values().iterator(); - } else if (from == null) { - return allTo(to); - } else if (to == null) { - return allFrom(from); + return storage.values().iterator(); } + if (from == null) { + return storage.headMap(to).values().iterator(); + } + if (to == null) { + return storage.tailMap(from).values().iterator(); + } + return storage.subMap(from, to).values().iterator(); + } - return this.memTable.subMap(from, to).values().iterator(); + @Override + public void upsert(Entry entry) { + storage.put(entry.key(), entry); } @Override public Entry get(MemorySegment key) { - Entry entry = this.memTable.get(key); - if (entry == null) { - try { - entry = persistenceHelper.readEntry(key); - } catch (IOException e) { + Entry entry = storage.get(key); + if (entry != null) { + if (entry.value() == null) { return null; } + return entry; + } + + Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + + if (!iterator.hasNext()) { + return null; + } + Entry next = iterator.next(); + if (compare(next.key(), key) == 0) { + return next; } - return entry; + return null; } @Override - public void upsert(Entry entry) { - if (entry == null) { + public void close() throws IOException { + if (!arena.scope().isAlive()) { return; } - this.memTable.put(entry.key(), entry); - } - @Override - public void flush() throws IOException { - persistenceHelper.writeEntries(this.memTable); + arena.close(); + + if (!storage.isEmpty()) { + DiskStorage.save(path, storage.values()); + } } } diff --git a/src/main/java/ru/vk/itmo/belonogovnikolay/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/belonogovnikolay/MemorySegmentComparator.java deleted file mode 100644 index 7b6b2c672..000000000 --- a/src/main/java/ru/vk/itmo/belonogovnikolay/MemorySegmentComparator.java +++ /dev/null @@ -1,25 +0,0 @@ -package ru.vk.itmo.belonogovnikolay; - -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.util.Comparator; - -public class MemorySegmentComparator implements Comparator { - - @Override - public int compare(MemorySegment prevSegment, MemorySegment nextSegment) { - - long offset = prevSegment.mismatch(nextSegment); - - if (offset == nextSegment.byteSize()) { - return 1; - } else if (offset == prevSegment.byteSize()) { - return -1; - } else if (offset == -1) { - return 0; - } - - return Byte.compare(prevSegment.get(ValueLayout.JAVA_BYTE, offset), - nextSegment.get(ValueLayout.JAVA_BYTE, offset)); - } -} diff --git a/src/main/java/ru/vk/itmo/belonogovnikolay/MergeIterator.java b/src/main/java/ru/vk/itmo/belonogovnikolay/MergeIterator.java new file mode 100644 index 000000000..749f8de4a --- /dev/null +++ b/src/main/java/ru/vk/itmo/belonogovnikolay/MergeIterator.java @@ -0,0 +1,142 @@ +package ru.vk.itmo.belonogovnikolay; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +public class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + private final Comparator comparator; + private PeekIterator peekIterator; + + public MergeIterator(Collection> iterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + private PeekIterator peek() { + while (peekIterator == null) { + peekIterator = priorityQueue.poll(); + if (peekIterator == null) { + return null; + } + + whileProcess(); + + if (peekIterator.peek() == null) { + peekIterator = null; + continue; + } + + if (skip(peekIterator.peek())) { + peekIterator.next(); + if (peekIterator.hasNext()) { + priorityQueue.add(peekIterator); + } + peekIterator = null; + } + } + + return peekIterator; + } + + private void whileProcess() { + while (true) { + PeekIterator next = priorityQueue.peek(); + if (next == null) { + break; + } + + int compare = comparator.compare(peekIterator.peek(), next.peek()); + if (compare == 0) { + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + } + } else { + break; + } + } + } + + protected boolean skip(T t) { + return t == null; + } + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator peek = peek(); + if (peek == null) { + throw new NoSuchElementException(); + } + T next = peek.next(); + this.peekIterator = null; + if (peek.hasNext()) { + priorityQueue.add(peek); + } + return next; + } + + private static class PeekIterator implements Iterator { + + public final int id; + private final Iterator delegate; + private T peekEntry; + + private PeekIterator(int id, Iterator delegate) { + this.id = id; + this.delegate = delegate; + } + + @Override + public boolean hasNext() { + if (peekEntry == null) { + return delegate.hasNext(); + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T peek = peek(); + this.peekEntry = null; + return peek; + } + + private T peek() { + if (peekEntry == null) { + if (!delegate.hasNext()) { + return null; + } + peekEntry = delegate.next(); + } + return peekEntry; + } + + } +} diff --git a/src/main/java/ru/vk/itmo/belonogovnikolay/PersistenceHelper.java b/src/main/java/ru/vk/itmo/belonogovnikolay/PersistenceHelper.java deleted file mode 100644 index 42dd1d0f0..000000000 --- a/src/main/java/ru/vk/itmo/belonogovnikolay/PersistenceHelper.java +++ /dev/null @@ -1,214 +0,0 @@ -package ru.vk.itmo.belonogovnikolay; - -import ru.vk.itmo.BaseEntry; -import ru.vk.itmo.Entry; - -import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Map; -import java.util.NavigableMap; - -/** - * Util class for write, read, and persistence recovery operations when the {@link InMemoryTreeDao DAO} is restarted. - * - * @author Belonogov Nikolay - */ -public final class PersistenceHelper { - - private final Path basePath; - private final MemorySegmentComparator segmentComparator; - - private MemorySegment dataMappedSegment; - private MemorySegment offsetMappedSegment; - private long[] positionOffsets; - private int position; - private Path pathToDataFile; - private Path pathToOffsetFile; - private long offsetFileSize; - private boolean isReadingPrepared; - private Arena readingDataArena; - private Arena readingOffsetArena; - - private PersistenceHelper(Path basePath) { - this.basePath = basePath; - this.segmentComparator = new MemorySegmentComparator(); - resolvePaths(); - } - - /** - * Returns instance of PersistenceHelper class. - * - * @param basePath directory for storing snapshots. - * @return PersistentHelper instance. - * @throws NullPointerException is thrown when the path to the directory with snapshot files is not specified. - */ - public static PersistenceHelper newInstance(Path basePath) { - return new PersistenceHelper(basePath); - } - - /** - * The function writes to the file specified in the config. If the config is not specified, an exception is thrown. - * - * @param entries data to be written to disk. - * @throws IOException is thrown when exceptions occur while working with a file. - */ - public void writeEntries(NavigableMap> entries) throws IOException { - - int size = entries.size(); - - positionOffsets = new long[size * 2 + 1]; - - long fileSize = 0; - for (Map.Entry> entry : entries.entrySet()) { - fileSize += entry.getKey().byteSize() + entry.getValue().value().byteSize(); - } - - Files.deleteIfExists(pathToDataFile); - Files.deleteIfExists(pathToOffsetFile); - - Files.createFile(pathToDataFile); - Files.createFile(pathToOffsetFile); - - try (Arena dataArena = Arena.ofConfined()) { - try (Arena offsetArena = Arena.ofConfined()) { - - this.dataMappedSegment = mapFilesWriteRead(pathToDataFile, fileSize, dataArena); - this.offsetMappedSegment = mapFilesWriteRead(pathToOffsetFile, - (long) Long.BYTES * positionOffsets.length, offsetArena); - - entries.values().forEach(entry -> { - long keySize = entry.key().byteSize(); - long valueSize = entry.value().byteSize(); - positionOffsets[position + 1] = positionOffsets[position] + keySize; - positionOffsets[position + 2] = positionOffsets[position + 1] + valueSize; - this.dataMappedSegment.asSlice(positionOffsets[position], keySize).copyFrom(entry.key()); - this.dataMappedSegment.asSlice(positionOffsets[position + 1], valueSize).copyFrom(entry.value()); - position = position + 2; - }); - - offsetMappedSegment - .asSlice(0L, (long) Long.BYTES * positionOffsets.length) - .copyFrom(MemorySegment.ofArray(positionOffsets)); - } - } finally { - if (this.readingDataArena != null) { - this.readingDataArena.close(); - this.readingDataArena = null; - } - - if (this.readingOffsetArena != null) { - this.readingOffsetArena.close(); - this.readingOffsetArena = null; - } - } - } - - /** - * Returns entry of data which is read from file. - * - * @param key is search key of data entry which is read from file. - * @return entry of data. - * @throws IOException is thrown when exceptions occur while working with a file. - */ - public Entry readEntry(MemorySegment key) throws IOException { - - if (Files.notExists(pathToDataFile) || Files.notExists(pathToOffsetFile)) { - return null; - } - - if (!isReadingPrepared) { - readingPreparation(); - this.isReadingPrepared = true; - } - - long index = 0; - long beginLong; - long endLong; - long keyValueSize; - long offsetFileOffsetCount = (this.offsetFileSize - Long.BYTES) / 8 - 1; - - while (index < offsetFileOffsetCount) { - beginLong = offsetMappedSegment.getAtIndex(ValueLayout.JAVA_LONG, index); - endLong = offsetMappedSegment.getAtIndex(ValueLayout.JAVA_LONG, index + 1); - keyValueSize = endLong - beginLong; - MemorySegment keySegment = dataMappedSegment.asSlice(beginLong, keyValueSize); - if (this.segmentComparator.compare(keySegment, key) == 0) { - keyValueSize = offsetMappedSegment.getAtIndex(ValueLayout.JAVA_LONG, index + 2) - endLong; - return new BaseEntry<>(keySegment, dataMappedSegment.asSlice(endLong, keyValueSize)); - } - index++; - } - return null; - } - - /** - * Function of mapping MemorySegment and data file in READ-ONLY mode. - * - * @param filePath file path. - * @param byteSize file size (offset). - * @return {@link MemorySegment} which map with file. - * @throws IOException is thrown when exceptions occur while working with a file. - */ - private MemorySegment mapDataFileReadOnly(Path filePath, long byteSize) throws IOException { - try (FileChannel channel = FileChannel.open(filePath, StandardOpenOption.READ)) { - this.readingDataArena = Arena.ofConfined(); - return channel.map(FileChannel.MapMode.READ_ONLY, 0, byteSize, this.readingDataArena); - } - } - - /** - * Function of mapping MemorySegment and offset file in READ-ONLY mode. - * - * @param filePath file path. - * @param byteSize file size (offset). - * @return {@link MemorySegment} which map with file. - * @throws IOException is thrown when exceptions occur while working with a file. - */ - private MemorySegment mapOffsetFileReadOnly(Path filePath, long byteSize) throws IOException { - try (FileChannel channel = FileChannel.open(filePath, StandardOpenOption.READ)) { - this.readingOffsetArena = Arena.ofConfined(); - return channel.map(FileChannel.MapMode.READ_ONLY, 0, byteSize, this.readingOffsetArena); - } - } - - /** - * Function of mapping MemorySegment and file in READ-WRITE mode. - * - * @param filePath file path. - * @param byteSize file size (offset). - * @return {@link MemorySegment} which map with file. - * @throws IOException is thrown when exceptions occur while working with a file. - */ - private MemorySegment mapFilesWriteRead(Path filePath, long byteSize, Arena arena) throws IOException { - try (FileChannel channel = FileChannel.open(filePath, StandardOpenOption.READ, StandardOpenOption.WRITE)) { - return channel.map(FileChannel.MapMode.READ_WRITE, 0, byteSize, arena); - } - } - - /** - * Function resolves {@link #basePath} to file {@link #pathToDataFile}, {@link #pathToOffsetFile} paths. - */ - private void resolvePaths() { - this.pathToDataFile = this.basePath.resolve(FileType.DATA.toString()); - this.pathToOffsetFile = this.basePath.resolve(FileType.OFFSET.toString()); - } - - /** - * Until function to prepare for reading operations. - * - * @throws IOException is thrown when exceptions occur while working with a file. - */ - private void readingPreparation() throws IOException { - this.offsetFileSize = Files.size(pathToOffsetFile); - long dataFileSize = Files.size(pathToDataFile); - - this.dataMappedSegment = mapDataFileReadOnly(pathToDataFile, dataFileSize); - this.offsetMappedSegment = mapOffsetFileReadOnly(pathToOffsetFile, this.offsetFileSize); - } -} diff --git a/src/main/java/ru/vk/itmo/chernyshevyaroslav/ChernyshevDao.java b/src/main/java/ru/vk/itmo/chernyshevyaroslav/ChernyshevDao.java new file mode 100644 index 000000000..48da8b030 --- /dev/null +++ b/src/main/java/ru/vk/itmo/chernyshevyaroslav/ChernyshevDao.java @@ -0,0 +1,124 @@ +package ru.vk.itmo.chernyshevyaroslav; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +public class ChernyshevDao implements Dao> { + + private static final String DATA_PATH = "data"; + private final Comparator comparator = ChernyshevDao::compare; + private final NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); + private final Arena arena; + private final DiskStorage diskStorage; + private final Path path; + + public ChernyshevDao(Config config) throws IOException { + this.path = config.basePath().resolve(DATA_PATH); + Files.createDirectories(path); + + arena = Arena.ofShared(); + + this.diskStorage = new DiskStorage(FileUtils.loadOrRecover(path, arena)); + } + + static int compare(MemorySegment memorySegment1, MemorySegment memorySegment2) { + long mismatch = memorySegment1.mismatch(memorySegment2); + if (mismatch == -1) { + return 0; + } + + if (mismatch == memorySegment1.byteSize()) { + return -1; + } + + if (mismatch == memorySegment2.byteSize()) { + return 1; + } + byte b1 = memorySegment1.get(ValueLayout.JAVA_BYTE, mismatch); + byte b2 = memorySegment2.get(ValueLayout.JAVA_BYTE, mismatch); + return Byte.compareUnsigned(b1, b2); + } + + @Override + public Iterator> get(MemorySegment from, MemorySegment to) { + return diskStorage.range(getInMemory(from, to), from, to); + } + + private Iterator> getInMemory(MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return storage.values().iterator(); + } + if (from == null) { + return storage.headMap(to).values().iterator(); + } + if (to == null) { + return storage.tailMap(from).values().iterator(); + } + return storage.subMap(from, to).values().iterator(); + } + + @Override + public Entry get(MemorySegment key) { + Entry entry = storage.get(key); + if (entry != null) { + if (entry.value() == null) { + return null; + } + return entry; + } + + Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + + if (!iterator.hasNext()) { + return null; + } + Entry next = iterator.next(); + if (compare(next.key(), key) == 0) { + return next; + } + return null; + } + + @Override + public void close() throws IOException { + if (!arena.scope().isAlive()) { + return; + } + + arena.close(); + + flush(); + } + + @Override + public void flush() throws IOException { + if (!storage.isEmpty()) { + FileUtils.save(path, storage.values(), false); + } + } + + @Override + public void compact() throws IOException { + flush(); + FileUtils.compact(path, this::all); + storage.clear(); + } + + @Override + public void upsert(Entry entry) { + storage.put(entry.key(), entry); + } +} diff --git a/src/main/java/ru/vk/itmo/chernyshevyaroslav/DiskStorage.java b/src/main/java/ru/vk/itmo/chernyshevyaroslav/DiskStorage.java new file mode 100644 index 000000000..031fc8076 --- /dev/null +++ b/src/main/java/ru/vk/itmo/chernyshevyaroslav/DiskStorage.java @@ -0,0 +1,151 @@ +package ru.vk.itmo.chernyshevyaroslav; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +public class DiskStorage { + + private final List segmentList; + + public DiskStorage(List segmentList) { + this.segmentList = segmentList; + } + + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to) { + List>> iterators = new ArrayList<>(segmentList.size() + 1); + for (MemorySegment memorySegment : segmentList) { + iterators.add(iterator(memorySegment, from, to)); + } + iterators.add(firstIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, ChernyshevDao::compare)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + private static long indexOf(MemorySegment segment, MemorySegment key) { + long recordsCount = recordsCount(segment); + + long left = 0; + long right = recordsCount - 1; + while (left <= right) { + long mid = (left + right) >>> 1; + + long startOfKey = startOfKey(segment, mid); + long endOfKey = endOfKey(segment, mid); + long mismatch = MemorySegment.mismatch(segment, startOfKey, endOfKey, key, 0, key.byteSize()); + if (mismatch == -1) { + return mid; + } + + if (mismatch == key.byteSize()) { + right = mid - 1; + continue; + } + + if (mismatch == endOfKey - startOfKey) { + left = mid + 1; + continue; + } + + int b1 = Byte.toUnsignedInt(segment.get(ValueLayout.JAVA_BYTE, startOfKey + mismatch)); + int b2 = Byte.toUnsignedInt(key.get(ValueLayout.JAVA_BYTE, mismatch)); + if (b1 > b2) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + return tombstone(left); + } + + private static long recordsCount(MemorySegment segment) { + long indexSize = indexSize(segment); + return indexSize / Long.BYTES / 2; + } + + private static long indexSize(MemorySegment segment) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + + static Iterator> iterator(MemorySegment page, MemorySegment from, MemorySegment to) { + long recordIndexFrom = from == null ? 0 : normalize(indexOf(page, from)); + long recordIndexTo = to == null ? recordsCount(page) : normalize(indexOf(page, to)); + long recordsCount = recordsCount(page); + + return new Iterator<>() { + long index = recordIndexFrom; + + @Override + public boolean hasNext() { + return index < recordIndexTo; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + MemorySegment key = slice(page, startOfKey(page, index), endOfKey(page, index)); + long startOfValue = startOfValue(page, index); + MemorySegment value = + startOfValue < 0 + ? null + : slice(page, startOfValue, endOfValue(page, index, recordsCount)); + index++; + return new BaseEntry<>(key, value); + } + }; + } + + private static MemorySegment slice(MemorySegment page, long start, long end) { + return page.asSlice(start, end - start); + } + + private static long startOfKey(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES); + } + + private static long endOfKey(MemorySegment segment, long recordIndex) { + return normalizedStartOfValue(segment, recordIndex); + } + + private static long normalizedStartOfValue(MemorySegment segment, long recordIndex) { + return normalize(startOfValue(segment, recordIndex)); + } + + private static long startOfValue(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES + Long.BYTES); + } + + private static long endOfValue(MemorySegment segment, long recordIndex, long recordsCount) { + if (recordIndex < recordsCount - 1) { + return startOfKey(segment, recordIndex + 1); + } + return segment.byteSize(); + } + + static long tombstone(long offset) { + return 1L << 63 | offset; + } + + private static long normalize(long value) { + return value & ~(1L << 63); + } + +} diff --git a/src/main/java/ru/vk/itmo/chernyshevyaroslav/FileUtils.java b/src/main/java/ru/vk/itmo/chernyshevyaroslav/FileUtils.java new file mode 100644 index 000000000..e19b33c7d --- /dev/null +++ b/src/main/java/ru/vk/itmo/chernyshevyaroslav/FileUtils.java @@ -0,0 +1,205 @@ +package ru.vk.itmo.chernyshevyaroslav; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; + +public final class FileUtils { + + private static final String INDEX_IDX = "index.idx"; + private static final String INDEX_TMP = "index.tmp"; + private static final String COMPACTION_TMP = "Compaction.tmp"; + + private FileUtils() { + throw new IllegalStateException("Utility class"); + } + + public static void save(Path storagePath, Iterable> iterable, boolean isCompaction) + throws IOException { + final Path indexTmp = storagePath.resolve(INDEX_TMP); + final Path indexFile = storagePath.resolve(INDEX_IDX); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + + String newFileName = isCompaction ? COMPACTION_TMP : String.valueOf(existedFiles.size()); + + long dataSize = 0; + long count = 0; + for (Entry entry : iterable) { + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + if (value != null) { + dataSize += value.byteSize(); + } + count++; + } + long indexSize = count * 2 * Long.BYTES; + + try ( + FileChannel fileChannel = FileChannel.open( + storagePath.resolve(newFileName), + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE + ); + Arena writeArena = Arena.ofConfined() + ) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + indexSize + dataSize, + writeArena + ); + + // index: + // |key0_Start|value0_Start|key1_Start|value1_Start|key2_Start|value2_Start|... + // key0_Start = data start = end of index + long dataOffset = indexSize; + int indexOffset = 0; + for (Entry entry : iterable) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += entry.key().byteSize(); + indexOffset += Long.BYTES; + + MemorySegment value = entry.value(); + if (value == null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, DiskStorage.tombstone(dataOffset)); + } else { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += value.byteSize(); + } + indexOffset += Long.BYTES; + } + + // data: + // |key0|value0|key1|value1|... + dataOffset = indexSize; + for (Entry entry : iterable) { + MemorySegment key = entry.key(); + MemorySegment.copy(key, 0, fileSegment, dataOffset, key.byteSize()); + dataOffset += key.byteSize(); + + MemorySegment value = entry.value(); + if (value != null) { + MemorySegment.copy(value, 0, fileSegment, dataOffset, value.byteSize()); + dataOffset += value.byteSize(); + } + } + } + + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + + List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + if (!isCompaction) { + list.add(newFileName); + } + Files.write( + indexFile, + list, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.delete(indexTmp); + } + + public static List loadOrRecover(Path storagePath, Arena arena) throws IOException { + if (Files.exists(storagePath.resolve(COMPACTION_TMP))) { + finalizeCompaction(storagePath); + } + Path indexTmp = storagePath.resolve(INDEX_TMP); + Path indexFile = storagePath.resolve(INDEX_IDX); + + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + } + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + List result = new ArrayList<>(existedFiles.size()); + for (String fileName : existedFiles) { + Path file = storagePath.resolve(fileName); + try (FileChannel fileChannel = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(file), + arena + ); + result.add(fileSegment); + } + } + + return result; + } + + public static void compact(Path storagePath, Iterable> iterable) throws IOException { + Path indexFile = storagePath.resolve(INDEX_IDX); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + List existingFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + + if (existingFiles.isEmpty()) { + Files.delete(indexFile); + return; + } + save(storagePath, iterable, true); + finalizeCompaction(storagePath); + } + + private static void finalizeCompaction(Path storagePath) { + Path indexFile = storagePath.resolve(INDEX_IDX); + + try { + List existingFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + + for (String file : existingFiles) { + Files.deleteIfExists(storagePath.resolve(file)); + } + + Files.writeString( + indexFile, + "0", + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.move(storagePath.resolve(COMPACTION_TMP), + storagePath.resolve("0"), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/src/main/java/ru/vk/itmo/chernyshevyaroslav/InMemoryDao.java b/src/main/java/ru/vk/itmo/chernyshevyaroslav/InMemoryDao.java deleted file mode 100644 index 894abe181..000000000 --- a/src/main/java/ru/vk/itmo/chernyshevyaroslav/InMemoryDao.java +++ /dev/null @@ -1,35 +0,0 @@ -package ru.vk.itmo.chernyshevyaroslav; - -import ru.vk.itmo.Dao; -import ru.vk.itmo.Entry; -import java.lang.foreign.MemorySegment; -import java.util.Iterator; -import java.util.concurrent.ConcurrentSkipListMap; - -public class InMemoryDao implements Dao> { - private final ConcurrentSkipListMap> data = - new ConcurrentSkipListMap<>(new MemorySegmentComparator()); - - @Override - public Iterator> get(MemorySegment from, MemorySegment to) { - if (from == null && to == null) { - return data.values().iterator(); - } else if (from == null) { - return data.headMap(to).values().iterator(); - } else if (to == null) { - return data.tailMap(from).values().iterator(); - } else { - return data.subMap(from, to).values().iterator(); - } - } - - @Override - public Entry get(MemorySegment key) { - return data.get(key); - } - - @Override - public void upsert(Entry entry) { - data.put(entry.key(), entry); - } -} diff --git a/src/main/java/ru/vk/itmo/chernyshevyaroslav/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/chernyshevyaroslav/MemorySegmentComparator.java deleted file mode 100644 index b95b4ddc5..000000000 --- a/src/main/java/ru/vk/itmo/chernyshevyaroslav/MemorySegmentComparator.java +++ /dev/null @@ -1,21 +0,0 @@ -package ru.vk.itmo.chernyshevyaroslav; - -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.util.Comparator; - -public final class MemorySegmentComparator implements Comparator { - @Override - public int compare(MemorySegment o1, MemorySegment o2) { - long offset = o1.mismatch(o2); - if (offset == -1) { - return 0; - } else if (o1.byteSize() == offset) { - return -1; - } else if (o2.byteSize() == offset) { - return 1; - } else { - return Byte.compareUnsigned(o1.get(ValueLayout.JAVA_BYTE, offset), o2.get(ValueLayout.JAVA_BYTE, offset)); - } - } -} diff --git a/src/main/java/ru/vk/itmo/chernyshevyaroslav/MergeIterator.java b/src/main/java/ru/vk/itmo/chernyshevyaroslav/MergeIterator.java new file mode 100644 index 000000000..2c85a0cc6 --- /dev/null +++ b/src/main/java/ru/vk/itmo/chernyshevyaroslav/MergeIterator.java @@ -0,0 +1,136 @@ +package ru.vk.itmo.chernyshevyaroslav; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +public abstract class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + private final Comparator comparator; + private PeekIterator peek; + + private static class PeekIterator implements Iterator { + + public final int id; + private final Iterator delegate; + private T peek; + + private PeekIterator(int id, Iterator delegate) { + this.id = id; + this.delegate = delegate; + } + + @Override + public boolean hasNext() { + if (peek == null) { + return delegate.hasNext(); + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T newNext = peek(); + this.peek = null; + return newNext; + } + + private T peek() { + if (peek == null) { + if (!delegate.hasNext()) { + return null; + } + peek = delegate.next(); + } + return peek; + } + } + + protected MergeIterator(Collection> iterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + private PeekIterator peek() { + while (peek == null) { + peek = priorityQueue.poll(); + if (peek == null) { + return null; + } + + PeekIterator next = priorityQueue.peek(); + skipIdentical(next); + + if (peek.peek() == null) { + peek = null; + continue; + } + + if (skip(peek.peek())) { + peek.next(); + if (peek.hasNext()) { + priorityQueue.add(peek); + } + peek = null; + } + } + + return peek; + } + + private void skipIdentical(PeekIterator next) { + PeekIterator result = next; + while (result != null) { + + int compare = comparator.compare(peek.peek(), result.peek()); + PeekIterator poll = priorityQueue.peek(); + if ((compare != 0) || (poll == null)) { + break; + } + priorityQueue.remove(); + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + result = priorityQueue.peek(); + } + } + + protected abstract boolean skip(T t); + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator peekIterator = peek(); + if (peekIterator == null) { + throw new NoSuchElementException(); + } + T next = peekIterator.next(); + this.peek = null; + if (peekIterator.hasNext()) { + priorityQueue.add(peekIterator); + } + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/DiskStorage.java b/src/main/java/ru/vk/itmo/cheshevandrey/DiskStorage.java new file mode 100644 index 000000000..8905edf00 --- /dev/null +++ b/src/main/java/ru/vk/itmo/cheshevandrey/DiskStorage.java @@ -0,0 +1,297 @@ +package ru.vk.itmo.cheshevandrey; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +public class DiskStorage { + + private final List segmentList; + + private static final String INDEX_TMP_FILE_NAME = "index.tmp"; + private static final String INDEX_FILE_NAME = "index.idx"; + private static final String NEW_SSTABLE_FILE_NAME = "newSsTable.tmp"; + private static final String COMPACT_FILE_NAME = "compact.cmpct"; + + public DiskStorage(Arena arena, Path storagePath) throws IOException { + + Path indexFile = storagePath.resolve(INDEX_FILE_NAME); + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it's ok. + } + + // Если существует скомпакченный файл, то ранее произошла ошибка в методе bringUpToDateState(). + // Следовательно, необходимо доделать компакт, приведя хранилище в актуальное состояние. + if (Files.exists(storagePath.resolve(COMPACT_FILE_NAME))) { + bringUpToDateState(storagePath); + } + + List fileNames = Files.readAllLines(indexFile); + int filesCount = fileNames.size(); + this.segmentList = new ArrayList<>(filesCount); + for (int i = 0; i < filesCount; i++) { + Path file = storagePath.resolve(String.valueOf(i)); + try (FileChannel fileChannel = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(file), + arena + ); + segmentList.add(fileSegment); + } + } + } + + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to) { + List>> iterators = new ArrayList<>(segmentList.size() + 1); + for (MemorySegment memorySegment : segmentList) { + iterators.add(iterator(memorySegment, from, to)); + } + iterators.add(firstIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, PersistentDao::compare)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + public static void compact( + Path storagePath, + DiskStorage diskStorage, + Iterable> iterableMemTable + ) throws IOException { + IterableStorage storage = new IterableStorage(iterableMemTable, diskStorage); + if (diskStorage.segmentList.size() <= 1 && !iterableMemTable.iterator().hasNext()) { + return; + } + + // После успешного выполнения ожидаем увидеть скомпакченый файл COMPACT_FILE_NAME. + save(storagePath, storage, true); + + bringUpToDateState(storagePath); + } + + public static void save(Path storagePath, Iterable> iterable, boolean isForCompact) + throws IOException { + Path newSsTablePath = storagePath.resolve(NEW_SSTABLE_FILE_NAME); + + long dataSize = 0; + long count = 0; + for (Entry entry : iterable) { + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + if (value != null) { + dataSize += value.byteSize(); + } + count++; + } + long indexSize = count * 2 * Long.BYTES; + + try ( + FileChannel fileChannel = FileChannel.open( + newSsTablePath, + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE + ); + Arena writeArena = Arena.ofConfined() + ) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + indexSize + dataSize, + writeArena + ); + + // index: + // |key0_Start|value0_Start|key1_Start|value1_Start|key2_Start|value2_Start|... + // key0_Start = data start = end of index + long dataOffset = indexSize; + int indexOffset = 0; + for (Entry entry : iterable) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += entry.key().byteSize(); + indexOffset += Long.BYTES; + + MemorySegment value = entry.value(); + if (value == null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, Tools.tombstone(dataOffset)); + } else { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += value.byteSize(); + } + indexOffset += Long.BYTES; + } + + // data: + // |key0|value0|key1|value1|... + dataOffset = indexSize; + for (Entry entry : iterable) { + MemorySegment key = entry.key(); + MemorySegment.copy(key, 0, fileSegment, dataOffset, key.byteSize()); + dataOffset += key.byteSize(); + + MemorySegment value = entry.value(); + if (value != null) { + MemorySegment.copy(value, 0, fileSegment, dataOffset, value.byteSize()); + dataOffset += value.byteSize(); + } + } + } + + if (isForCompact) { + Files.move( + newSsTablePath, + storagePath.resolve(COMPACT_FILE_NAME), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + return; + } + + Path indexFile = storagePath.resolve(INDEX_FILE_NAME); + List fileNames = Files.readAllLines(indexFile); + String newFileName = String.valueOf(fileNames.size()); + fileNames.add(newFileName); + + updateIndex(storagePath, fileNames); + + Path newFilePath = storagePath.resolve(newFileName); + Files.move( + newSsTablePath, + newFilePath, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + } + + private static void bringUpToDateState(Path storagePath) throws IOException { + List fileNames = Files.readAllLines(storagePath.resolve(INDEX_FILE_NAME)); + + for (int i = 1; i < fileNames.size(); i++) { + Files.delete(storagePath.resolve(String.valueOf(i))); + } + + String newFileName = "0"; + + updateIndex(storagePath, Collections.singletonList(newFileName)); + + Files.move( + storagePath.resolve(COMPACT_FILE_NAME), + storagePath.resolve(newFileName), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + } + + private static void updateIndex(Path storagePath, List files) throws IOException { + Path indexTmpPath = storagePath.resolve(INDEX_TMP_FILE_NAME); + Path indexPath = storagePath.resolve(INDEX_FILE_NAME); + + Files.deleteIfExists(indexTmpPath); + Files.write( + indexTmpPath, + files, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.move( + indexTmpPath, + indexPath, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + } + + private static long indexOf(MemorySegment segment, MemorySegment key) { + long recordsCount = Tools.recordsCount(segment); + + long left = 0; + long right = recordsCount - 1; + while (left <= right) { + long mid = (left + right) >>> 1; + + long startOfKey = Tools.startOfKey(segment, mid); + long endOfKey = Tools.endOfKey(segment, mid); + long mismatch = MemorySegment.mismatch(segment, startOfKey, endOfKey, key, 0, key.byteSize()); + if (mismatch == -1) { + return mid; + } + + if (mismatch == key.byteSize()) { + right = mid - 1; + continue; + } + + if (mismatch == endOfKey - startOfKey) { + left = mid + 1; + continue; + } + + int b1 = Byte.toUnsignedInt(segment.get(ValueLayout.JAVA_BYTE, startOfKey + mismatch)); + int b2 = Byte.toUnsignedInt(key.get(ValueLayout.JAVA_BYTE, mismatch)); + if (b1 > b2) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + return Tools.tombstone(left); + } + + private static Iterator> iterator(MemorySegment page, MemorySegment from, MemorySegment to) { + long recordIndexFrom = from == null ? 0 : Tools.normalize(indexOf(page, from)); + long recordIndexTo = to == null ? Tools.recordsCount(page) : Tools.normalize(indexOf(page, to)); + long recordsCount = Tools.recordsCount(page); + + return new Iterator<>() { + long index = recordIndexFrom; + + @Override + public boolean hasNext() { + return index < recordIndexTo; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + MemorySegment key = Tools.slice(page, Tools.startOfKey(page, index), Tools.endOfKey(page, index)); + long startOfValue = Tools.startOfValue(page, index); + MemorySegment value = + startOfValue < 0 + ? null + : Tools.slice(page, startOfValue, Tools.endOfValue(page, index, recordsCount)); + index++; + return new BaseEntry<>(key, value); + } + }; + } +} diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/InMemoryDao.java b/src/main/java/ru/vk/itmo/cheshevandrey/InMemoryDao.java deleted file mode 100644 index 90f5842b0..000000000 --- a/src/main/java/ru/vk/itmo/cheshevandrey/InMemoryDao.java +++ /dev/null @@ -1,185 +0,0 @@ -package ru.vk.itmo.cheshevandrey; - -import ru.vk.itmo.BaseEntry; -import ru.vk.itmo.Config; -import ru.vk.itmo.Dao; -import ru.vk.itmo.Entry; - -import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Iterator; -import java.util.NavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.logging.Logger; - -public class InMemoryDao implements Dao> { - - private final Arena offHeapArena; - private final Path storagePath; - private static final String STORAGE_FILE_NAME = "output.sst"; - private static final Logger logger = Logger.getLogger(InMemoryDao.class.getName()); - private final NavigableMap> memTable = new ConcurrentSkipListMap<>( - this::compare - ); - - public InMemoryDao(Config config) throws IOException { - this.storagePath = config.basePath().resolve(STORAGE_FILE_NAME); - - if (!Files.exists(storagePath)) { - Files.createDirectories(storagePath.getParent()); - Files.createFile(storagePath); - } - - offHeapArena = Arena.ofConfined(); - } - - @Override - public Iterator> get(MemorySegment from, MemorySegment to) { - - if (from == null && to == null) { - return memTable.values().iterator(); - } - if (from == null) { - return memTable.headMap(to).values().iterator(); - } - if (to == null) { - return memTable.tailMap(from).values().iterator(); - } - return memTable.subMap(from, to).values().iterator(); - } - - @Override - public Entry get(MemorySegment key) { - - Entry entry = memTable.get(key); - if (entry != null) { - return entry; - } - - try (FileChannel channel = FileChannel.open( - storagePath, - StandardOpenOption.CREATE, - StandardOpenOption.READ - )) { - MemorySegment ssTable = channel.map( - FileChannel.MapMode.READ_ONLY, - 0, - channel.size(), - offHeapArena - ); - - if (ssTable.byteSize() == 0) { - return null; - } - - long offset = Long.BYTES; - long mismatch; - long operandSize; - long tableSize = readSizeFromSsTable(ssTable, 0); - while (offset < tableSize) { - - operandSize = readSizeFromSsTable(ssTable, offset); - offset += Long.BYTES; - - mismatch = MemorySegment.mismatch(ssTable, offset, offset + operandSize, key, 0, key.byteSize()); - offset += operandSize; - - operandSize = readSizeFromSsTable(ssTable, offset); - offset += Long.BYTES; - - if (mismatch == -1) { - return new BaseEntry<>( - key, - ssTable.asSlice(offset, operandSize) - ); - } - offset += operandSize; - } - } catch (IOException e) { - logger.severe("Ошибка при создании FileChannel: " + e.getMessage()); - } - - return null; - } - - private long readSizeFromSsTable(MemorySegment ssTable, long offset) { - return ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - } - - @Override - public void close() throws IOException { - - try ( - FileChannel channel = FileChannel.open( - storagePath, - StandardOpenOption.CREATE, - StandardOpenOption.READ, - StandardOpenOption.WRITE - ); - Arena closeArena = Arena.ofConfined() - ) { - long ssTableSize = 2L * Long.BYTES * memTable.size() + Long.BYTES; - for (Entry entry : memTable.values()) { - ssTableSize += entry.key().byteSize(); - ssTableSize += entry.value().byteSize(); - } - - MemorySegment ssTable = channel.map( - FileChannel.MapMode.READ_WRITE, - 0, - ssTableSize, - closeArena - ); - - ssTable.set(ValueLayout.JAVA_LONG_UNALIGNED, 0, ssTableSize); - - long offset = Long.BYTES; - for (Entry entry : memTable.values()) { - offset = storeAndGetOffset(ssTable, entry.key(), offset); - offset = storeAndGetOffset(ssTable, entry.value(), offset); - } - } finally { - offHeapArena.close(); - } - } - - private long storeAndGetOffset(MemorySegment ssTable, MemorySegment value, long offset) { - long newOffset = offset; - long valueSize = value.byteSize(); - - ssTable.set(ValueLayout.JAVA_LONG_UNALIGNED, newOffset, valueSize); - newOffset += Long.BYTES; - - MemorySegment.copy(value, 0, ssTable, newOffset, valueSize); - newOffset += valueSize; - - return newOffset; - } - - @Override - public void upsert(Entry entry) { - memTable.put(entry.key(), entry); - } - - private int compare(MemorySegment seg1, MemorySegment seg2) { - long mismatch = seg1.mismatch(seg2); - if (mismatch == -1) { - return 0; - } - if (mismatch == seg1.byteSize()) { - return -1; - } - if (mismatch == seg2.byteSize()) { - return 1; - } - byte b1 = seg1.get(ValueLayout.JAVA_BYTE, mismatch); - byte b2 = seg2.get(ValueLayout.JAVA_BYTE, mismatch); - return Byte.compare(b1, b2); - } -} diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/IterableStorage.java b/src/main/java/ru/vk/itmo/cheshevandrey/IterableStorage.java new file mode 100644 index 000000000..d29e2a3f3 --- /dev/null +++ b/src/main/java/ru/vk/itmo/cheshevandrey/IterableStorage.java @@ -0,0 +1,22 @@ +package ru.vk.itmo.cheshevandrey; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; + +public class IterableStorage implements Iterable> { + + Iterable> iterableMemTable; + DiskStorage diskStorage; + + IterableStorage(Iterable> iterableMemTable, DiskStorage diskStorage) { + this.iterableMemTable = iterableMemTable; + this.diskStorage = diskStorage; + } + + @Override + public Iterator> iterator() { + return diskStorage.range(iterableMemTable.iterator(), null, null); + } +} diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/MergeIterator.java b/src/main/java/ru/vk/itmo/cheshevandrey/MergeIterator.java new file mode 100644 index 000000000..24cadec32 --- /dev/null +++ b/src/main/java/ru/vk/itmo/cheshevandrey/MergeIterator.java @@ -0,0 +1,106 @@ +package ru.vk.itmo.cheshevandrey; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +public class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + private final Comparator comparator; + + PeekIterator peek; + + public MergeIterator(Collection> iterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + private PeekIterator peek() { + while (peek == null) { + peek = priorityQueue.poll(); + if (peek == null) { + return null; + } + + updateQueueState(); + + if (peek.peek() == null) { + peek = null; + continue; + } + + if (skip(peek.peek())) { + peek.next(); + if (peek.hasNext()) { + priorityQueue.add(peek); + } + peek = null; + } + } + + return peek; + } + + private void updateQueueState() { + while (true) { + PeekIterator next = priorityQueue.peek(); + if (next == null) { + break; + } + + int compare = comparator.compare(peek.peek(), next.peek()); + if (compare == 0) { + pollNext(); + } else { + break; + } + } + } + + private void pollNext() { + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + } + } + + protected boolean skip(T t) { + return t == null; + } + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator currIterator = peek(); + if (currIterator == null) { + throw new NoSuchElementException(); + } + T next = currIterator.next(); + this.peek = null; + if (currIterator.hasNext()) { + priorityQueue.add(currIterator); + } + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/PeekIterator.java b/src/main/java/ru/vk/itmo/cheshevandrey/PeekIterator.java new file mode 100644 index 000000000..bd9ab6526 --- /dev/null +++ b/src/main/java/ru/vk/itmo/cheshevandrey/PeekIterator.java @@ -0,0 +1,44 @@ +package ru.vk.itmo.cheshevandrey; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +class PeekIterator implements Iterator { + + public final int id; + private final Iterator delegate; + private T currIterator; + + PeekIterator(int id, Iterator delegate) { + this.id = id; + this.delegate = delegate; + } + + @Override + public boolean hasNext() { + if (currIterator == null) { + return delegate.hasNext(); + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T peek = peek(); + this.currIterator = null; + return peek; + } + + T peek() { + if (currIterator == null) { + if (!delegate.hasNext()) { + return null; + } + currIterator = delegate.next(); + } + return currIterator; + } +} diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/PersistentDao.java b/src/main/java/ru/vk/itmo/cheshevandrey/PersistentDao.java new file mode 100644 index 000000000..7df4a85e6 --- /dev/null +++ b/src/main/java/ru/vk/itmo/cheshevandrey/PersistentDao.java @@ -0,0 +1,123 @@ +package ru.vk.itmo.cheshevandrey; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +public class PersistentDao implements Dao> { + + private final Comparator comparator = PersistentDao::compare; + private final Arena arena; + private final DiskStorage diskStorage; + private final Path path; + + private NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); + + public PersistentDao(Config config) throws IOException { + this.path = config.basePath().resolve("data"); + Files.createDirectories(path); + + arena = Arena.ofShared(); + + this.diskStorage = new DiskStorage(arena, path); + } + + static int compare(MemorySegment memorySegment1, MemorySegment memorySegment2) { + long mismatch = memorySegment1.mismatch(memorySegment2); + if (mismatch == -1) { + return 0; + } + + if (mismatch == memorySegment1.byteSize()) { + return -1; + } + + if (mismatch == memorySegment2.byteSize()) { + return 1; + } + byte b1 = memorySegment1.get(ValueLayout.JAVA_BYTE, mismatch); + byte b2 = memorySegment2.get(ValueLayout.JAVA_BYTE, mismatch); + return Byte.compare(b1, b2); + } + + @Override + public Iterator> get(MemorySegment from, MemorySegment to) { + return diskStorage.range(getInMemory(from, to).iterator(), from, to); + } + + private Iterable> getInMemory(MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return storage.values(); + } + if (from == null) { + return storage.headMap(to).values(); + } + if (to == null) { + return storage.tailMap(from).values(); + } + return storage.subMap(from, to).values(); + } + + @Override + public void upsert(Entry entry) { + storage.put(entry.key(), entry); + } + + @Override + public Entry get(MemorySegment key) { + Entry entry = storage.get(key); + if (entry != null) { + if (entry.value() == null) { + return null; + } + return entry; + } + + Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + + if (!iterator.hasNext()) { + return null; + } + Entry next = iterator.next(); + if (compare(next.key(), key) == 0) { + return next; + } + return null; + } + + @Override + public void compact() throws IOException { + DiskStorage.compact(path, diskStorage, getInMemory(null, null)); + storage = new ConcurrentSkipListMap<>(comparator); + } + + @Override + public void flush() throws IOException { + DiskStorage.save(path, storage.values(), false); + } + + @Override + public void close() throws IOException { + if (!arena.scope().isAlive()) { + return; + } + + arena.close(); + + if (!storage.isEmpty()) { + flush(); + } + } +} diff --git a/src/main/java/ru/vk/itmo/cheshevandrey/Tools.java b/src/main/java/ru/vk/itmo/cheshevandrey/Tools.java new file mode 100644 index 000000000..3c77549c7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/cheshevandrey/Tools.java @@ -0,0 +1,56 @@ +package ru.vk.itmo.cheshevandrey; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +public final class Tools { + + // to hide the implicit public constructor + private Tools() { + } + + static long recordsCount(MemorySegment segment) { + long indexSize = indexSize(segment); + return indexSize / Long.BYTES / 2; + } + + static long indexSize(MemorySegment segment) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + + static MemorySegment slice(MemorySegment page, long start, long end) { + return page.asSlice(start, end - start); + } + + static long startOfKey(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES); + } + + static long endOfKey(MemorySegment segment, long recordIndex) { + return normalizedStartOfValue(segment, recordIndex); + } + + static long normalizedStartOfValue(MemorySegment segment, long recordIndex) { + return normalize(startOfValue(segment, recordIndex)); + } + + static long startOfValue(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES + Long.BYTES); + } + + static long endOfValue(MemorySegment segment, long recordIndex, long recordsCount) { + if (recordIndex < recordsCount - 1) { + return startOfKey(segment, recordIndex + 1); + } + return segment.byteSize(); + } + + static long tombstone(long offset) { + return 1L << 63 | offset; + } + + static long normalize(long value) { + return value & ~(1L << 63); + } + +} diff --git a/src/main/java/ru/vk/itmo/shemetovalexey/StorageUtils.java b/src/main/java/ru/vk/itmo/danilinandrew/DiskStorage.java similarity index 79% rename from src/main/java/ru/vk/itmo/shemetovalexey/StorageUtils.java rename to src/main/java/ru/vk/itmo/danilinandrew/DiskStorage.java index 4e96b5824..9eb03f4b6 100644 --- a/src/main/java/ru/vk/itmo/shemetovalexey/StorageUtils.java +++ b/src/main/java/ru/vk/itmo/danilinandrew/DiskStorage.java @@ -1,4 +1,4 @@ -package ru.vk.itmo.shemetovalexey; +package ru.vk.itmo.danilinandrew; import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Entry; @@ -15,44 +15,51 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.ArrayList; +import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; -final class StorageUtils { - private static final String TMP_DIRECTORY_NAME = "tmp"; - private static final String INDEX_FILE_NAME = "index"; - private static final String IDX_EXTENSION = ".idx"; - private static final String TMP_EXTENSION = ".tmp"; +public class DiskStorage { - private StorageUtils() { + private final List segmentList; + private static final String INDEX_FILE = "index.idx"; + private static final String TMP_FILE = "index.tmp"; + + public DiskStorage(List segmentList) { + this.segmentList = segmentList; } - public static void compact(Path path, Iterable> iterable) throws IOException { - try { - Files.createDirectory(path.resolve(TMP_DIRECTORY_NAME)); - } catch (IOException ignored) { - // it is mean, that directory already exist + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to) { + List>> iterators = new ArrayList<>(segmentList.size() + 1); + for (MemorySegment memorySegment : segmentList) { + iterators.add(iterator(memorySegment, from, to)); } - final Path tmpDirectory = path.resolve(TMP_DIRECTORY_NAME); - save(tmpDirectory, iterable); - final Path indexFile = path.resolve(INDEX_FILE_NAME + IDX_EXTENSION); - List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); - for (String existedFile : existedFiles) { - Files.delete(path.resolve(existedFile)); + iterators.add(firstIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, StorageDao::compare)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + public void clearStorage(Path storagePath) throws IOException { + final Path indexFile = storagePath.resolve(INDEX_FILE); + final List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + for (String fileName : existedFiles) { + Files.deleteIfExists(storagePath.resolve(fileName)); } - final Path indexTmp = tmpDirectory.resolve(INDEX_FILE_NAME + IDX_EXTENSION); - Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); - final Path dataTmp = tmpDirectory.resolve(existedFiles.get(0)); - final Path dataFile = path.resolve(existedFiles.get(0)); - Files.move(dataTmp, dataFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } public static void save(Path storagePath, Iterable> iterable) throws IOException { - final Path indexTmp = storagePath.resolve(INDEX_FILE_NAME + TMP_EXTENSION); - final Path indexFile = storagePath.resolve(INDEX_FILE_NAME + IDX_EXTENSION); + final Path indexTmp = storagePath.resolve(TMP_FILE); + final Path indexFile = storagePath.resolve(INDEX_FILE); try { Files.createFile(indexFile); @@ -127,25 +134,33 @@ public static void save(Path storagePath, Iterable> iterabl } } - Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - List list = new ArrayList<>(existedFiles.size() + 1); list.addAll(existedFiles); list.add(newFileName); Files.write( - indexFile, + indexTmp, list, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING ); - Files.delete(indexTmp); + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } public static List loadOrRecover(Path storagePath, Arena arena) throws IOException { - Path indexTmp = storagePath.resolve(INDEX_FILE_NAME + TMP_EXTENSION); - Path indexFile = storagePath.resolve(INDEX_FILE_NAME + IDX_EXTENSION); + Path indexTmp = storagePath.resolve(TMP_FILE); + Path indexFile = storagePath.resolve(INDEX_FILE); + + // Не смог вынести названия в константы, потому что codeclimate фейлится по количеству строк этого файла + if (Files.exists(storagePath.resolve("0tmp"))) { + Files.move( + storagePath.resolve("0tmp"), + storagePath.resolve("0"), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + } if (Files.exists(indexTmp)) { Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); @@ -221,7 +236,7 @@ private static long indexSize(MemorySegment segment) { return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); } - static Iterator> iterator(MemorySegment page, MemorySegment from, MemorySegment to) { + private static Iterator> iterator(MemorySegment page, MemorySegment from, MemorySegment to) { long recordIndexFrom = from == null ? 0 : normalize(indexOf(page, from)); long recordIndexTo = to == null ? recordsCount(page) : normalize(indexOf(page, to)); long recordsCount = recordsCount(page); diff --git a/src/main/java/ru/vk/itmo/danilinandrew/DiskStorageWithCompact.java b/src/main/java/ru/vk/itmo/danilinandrew/DiskStorageWithCompact.java new file mode 100644 index 000000000..8dff5a2b1 --- /dev/null +++ b/src/main/java/ru/vk/itmo/danilinandrew/DiskStorageWithCompact.java @@ -0,0 +1,126 @@ +package ru.vk.itmo.danilinandrew; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.Iterator; + +public class DiskStorageWithCompact { + private final DiskStorage diskStorage; + private static final String INDEX_FILE = "index.idx"; + private static final String TMP_FILE = "index.tmp"; + + private static final String TMP_COMPACTED_FILE = "0tmp"; + private static final String COMPACTED_FILE = "0"; + + public DiskStorageWithCompact(DiskStorage diskStorage) { + this.diskStorage = diskStorage; + } + + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to + ) { + return diskStorage.range(firstIterator, from, to); + } + + public void compact( + Path storagePath, + Iterator> it1, + Iterator> it2 + ) throws IOException { + long sizeData = 0; + long sizeIndexes = 0; + while (it1.hasNext()) { + Entry entry = it1.next(); + if (entry.value() != null) { + sizeData += entry.key().byteSize(); + sizeData += entry.value().byteSize(); + sizeIndexes++; + } + } + + if (sizeIndexes == 0) { + return; + } + sizeIndexes *= 2 * Long.BYTES; + + final Path indexTmp = storagePath.resolve(TMP_FILE); + final Path indexFile = storagePath.resolve(INDEX_FILE); + + try ( + FileChannel fileChannel = FileChannel.open( + storagePath.resolve(TMP_COMPACTED_FILE), + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + Arena writeArena = Arena.ofConfined() + ) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + sizeIndexes + sizeData, + writeArena + ); + long offsetIndexes = 0; + long currentIndex = sizeIndexes; + while (it2.hasNext()) { + Entry currentEntry = it2.next(); + if (currentEntry.value() != null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, offsetIndexes, currentIndex); + offsetIndexes += Long.BYTES; + + MemorySegment.copy( + currentEntry.key(), + 0, + fileSegment, + currentIndex, + currentEntry.key().byteSize() + ); + currentIndex += currentEntry.key().byteSize(); + + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, offsetIndexes, currentIndex); + offsetIndexes += Long.BYTES; + + MemorySegment.copy( + currentEntry.value(), + 0, + fileSegment, + currentIndex, + currentEntry.value().byteSize() + ); + currentIndex += currentEntry.value().byteSize(); + } + } + + Files.writeString( + indexTmp, + COMPACTED_FILE, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + diskStorage.clearStorage(storagePath); + + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + Files.move( + storagePath.resolve(TMP_COMPACTED_FILE), + storagePath.resolve(COMPACTED_FILE), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + } + } +} diff --git a/src/main/java/ru/vk/itmo/danilinandrew/InMemoryDao.java b/src/main/java/ru/vk/itmo/danilinandrew/InMemoryDao.java deleted file mode 100644 index 71756157e..000000000 --- a/src/main/java/ru/vk/itmo/danilinandrew/InMemoryDao.java +++ /dev/null @@ -1,146 +0,0 @@ -package ru.vk.itmo.danilinandrew; - -import ru.vk.itmo.BaseEntry; -import ru.vk.itmo.Config; -import ru.vk.itmo.Dao; -import ru.vk.itmo.Entry; - -import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Iterator; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; - -public class InMemoryDao implements Dao> { - private final ConcurrentNavigableMap> data = - new ConcurrentSkipListMap<>(new MemorySegmentComparator()); - private final Path ssTablePath; - private final Arena readArena = Arena.ofConfined(); - private final MemorySegment mappedMemorySegment; - - public InMemoryDao(Config config) { - ssTablePath = config.basePath().resolve("data.txt"); - - MemorySegment tempMemorySegment; - try (FileChannel fileChannel = FileChannel.open(ssTablePath)) { - long size = Files.size(ssTablePath); - tempMemorySegment = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, size, readArena); - } catch (IOException e) { - tempMemorySegment = null; - } - - mappedMemorySegment = tempMemorySegment; - } - - @Override - public Entry get(MemorySegment key) { - Entry value = data.get(key); - - if (value != null) { - return value; - } - - if (mappedMemorySegment == null) { - return null; - } - - long offset = 0; - while (offset < mappedMemorySegment.byteSize()) { - long keySize = mappedMemorySegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; - - if (keySize != key.byteSize()) { - offset += keySize; - long valueSize = mappedMemorySegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES + valueSize; - continue; - } - - MemorySegment readKey = mappedMemorySegment.asSlice(offset, keySize); - offset += keySize; - if (key.mismatch(readKey) == -1) { - long valueSize = mappedMemorySegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; - MemorySegment readValue = mappedMemorySegment.asSlice(offset, valueSize); - - return new BaseEntry<>(key, readValue); - } - - } - - return null; - } - - @Override - public Iterator> get(MemorySegment from, MemorySegment to) { - if (from == null && to == null) { - return data.values().iterator(); - } - - if (from == null) { - return data.headMap(to).values().iterator(); - } - - if (to == null) { - return data.tailMap(from).values().iterator(); - } - - return data.subMap(from, to).values().iterator(); - } - - @Override - public void upsert(Entry entry) { - if (entry == null) { - return; - } - - data.put(entry.key(), entry); - } - - @Override - public void close() throws IOException { - readArena.close(); - - try (Arena writeArena = Arena.ofConfined()) { - long size = 0; - - for (Entry value : data.values()) { - size += value.key().byteSize() + value.value().byteSize(); - } - - size += 2L * Long.BYTES * size; - - try (FileChannel fileChannel = FileChannel.open( - ssTablePath, - StandardOpenOption.WRITE, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.CREATE, - StandardOpenOption.READ - )) { - MemorySegment page = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, size, writeArena); - - long offset = 0; - - for (Entry value : data.values()) { - page.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, value.key().byteSize()); - offset += Long.BYTES; - - page.asSlice(offset).copyFrom(value.key()); - offset += value.key().byteSize(); - - page.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, value.value().byteSize()); - offset += Long.BYTES; - - page.asSlice(offset).copyFrom(value.value()); - offset += value.value().byteSize(); - } - } - } - } -} diff --git a/src/main/java/ru/vk/itmo/danilinandrew/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/danilinandrew/MemorySegmentComparator.java deleted file mode 100644 index ed9a37ecf..000000000 --- a/src/main/java/ru/vk/itmo/danilinandrew/MemorySegmentComparator.java +++ /dev/null @@ -1,30 +0,0 @@ -package ru.vk.itmo.danilinandrew; - -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.util.Comparator; - -public class MemorySegmentComparator implements Comparator { - - @Override - public int compare(MemorySegment o1, MemorySegment o2) { - long mismatch = o1.mismatch(o2); - - if (mismatch == -1) { - return 0; - } - - if (mismatch == o1.byteSize()) { - return -1; - } - - if (mismatch == o2.byteSize()) { - return 1; - } - - return Byte.compare( - o1.get(ValueLayout.JAVA_BYTE, mismatch), - o2.get(ValueLayout.JAVA_BYTE, mismatch) - ); - } -} diff --git a/src/main/java/ru/vk/itmo/danilinandrew/MergeIterator.java b/src/main/java/ru/vk/itmo/danilinandrew/MergeIterator.java new file mode 100644 index 000000000..b6b3b37cb --- /dev/null +++ b/src/main/java/ru/vk/itmo/danilinandrew/MergeIterator.java @@ -0,0 +1,142 @@ +package ru.vk.itmo.danilinandrew; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +public class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + private final Comparator comparator; + + private PeekIterator peek; + + private static class PeekIterator implements Iterator { + + public final int id; + private final Iterator delegate; + private T peek; + + private PeekIterator(int id, Iterator delegate) { + this.id = id; + this.delegate = delegate; + } + + @Override + public boolean hasNext() { + if (peek == null) { + return delegate.hasNext(); + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T curPeek = peek(); + this.peek = null; + return curPeek; + } + + private T peek() { + if (peek == null) { + if (!delegate.hasNext()) { + return null; + } + peek = delegate.next(); + } + return peek; + } + } + + public MergeIterator(Collection> iterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + private PeekIterator peek() { + while (peek == null) { + peek = priorityQueue.poll(); + if (peek == null) { + return null; + } + + findElement(); + + if (peek.peek() == null) { + peek = null; + continue; + } + + if (skip(peek.peek())) { + peek.next(); + if (peek.hasNext()) { + priorityQueue.add(peek); + } + peek = null; + } + } + + return peek; + } + + private void findElement() { + while (true) { + PeekIterator next = priorityQueue.peek(); + if (next == null) { + break; + } + + int compare = comparator.compare(peek.peek(), next.peek()); + if (compare == 0) { + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + } + } else { + break; + } + } + } + + protected boolean skip(T t) { + return t == null; + } + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator curPeek = peek(); + if (curPeek == null) { + throw new NoSuchElementException(); + } + T next = curPeek.next(); + this.peek = null; + if (curPeek.hasNext()) { + priorityQueue.add(curPeek); + } + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/bandurinvladislav/PersistentDao.java b/src/main/java/ru/vk/itmo/danilinandrew/StorageDao.java similarity index 81% rename from src/main/java/ru/vk/itmo/bandurinvladislav/PersistentDao.java rename to src/main/java/ru/vk/itmo/danilinandrew/StorageDao.java index f27737204..59baef5a4 100644 --- a/src/main/java/ru/vk/itmo/bandurinvladislav/PersistentDao.java +++ b/src/main/java/ru/vk/itmo/danilinandrew/StorageDao.java @@ -1,4 +1,4 @@ -package ru.vk.itmo.bandurinvladislav; +package ru.vk.itmo.danilinandrew; import ru.vk.itmo.Config; import ru.vk.itmo.Dao; @@ -16,21 +16,23 @@ import java.util.NavigableMap; import java.util.concurrent.ConcurrentSkipListMap; -public class PersistentDao implements Dao> { +public class StorageDao implements Dao> { - private final Comparator comparator = PersistentDao::compare; + private final Comparator comparator = StorageDao::compare; private final NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); private final Arena arena; - private final DiskStorage diskStorage; + private final DiskStorageWithCompact diskStorage; private final Path path; - public PersistentDao(Config config) throws IOException { + public StorageDao(Config config) throws IOException { this.path = config.basePath().resolve("data"); Files.createDirectories(path); arena = Arena.ofShared(); - this.diskStorage = new DiskStorage(DiskStorage.loadOrRecover(path, arena)); + this.diskStorage = new DiskStorageWithCompact( + new DiskStorage(DiskStorage.loadOrRecover(path, arena)) + ); } static int compare(MemorySegment memorySegment1, MemorySegment memorySegment2) { @@ -96,6 +98,14 @@ public Entry get(MemorySegment key) { return null; } + @Override + public void compact() throws IOException { + Iterator> allElementsIterator1 = all(); + Iterator> allElementsIterator2 = all(); + diskStorage.compact(path, allElementsIterator1, allElementsIterator2); + storage.clear(); + } + @Override public void close() throws IOException { if (!arena.scope().isAlive()) { @@ -108,9 +118,4 @@ public void close() throws IOException { DiskStorage.save(path, storage.values()); } } - - @Override - public void compact() throws IOException { - diskStorage.compact(path, get(null, null)); - } } diff --git a/src/main/java/ru/vk/itmo/dyagayalexandra/CompactManager.java b/src/main/java/ru/vk/itmo/dyagayalexandra/CompactManager.java new file mode 100644 index 000000000..d66781a93 --- /dev/null +++ b/src/main/java/ru/vk/itmo/dyagayalexandra/CompactManager.java @@ -0,0 +1,166 @@ +package ru.vk.itmo.dyagayalexandra; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; + +public class CompactManager { + + private final String fileName; + private final String fileExtension; + private final String fileIndexName; + private Path compactedFile; + private Path compactedIndex; + private final EntryKeyComparator entryKeyComparator; + private final FileWriterManager fileWriterManager; + private static final String TEMP_FILE_NAME = "tempCompact.txt"; + private static final String TEMP_FILE_INDEX_NAME = "tempCompactIndex.txt"; + private static final String MAIN_FILE_NAME = "mainCompact.txt"; + private static final String MAIN_FILE_INDEX_NAME = "mainCompactIndex.txt"; + + public CompactManager(String fileName, String fileIndexName, String fileExtension, + EntryKeyComparator entryKeyComparator, FileWriterManager fileWriterManager) { + this.fileName = fileName; + this.fileIndexName = fileIndexName; + this.fileExtension = fileExtension; + this.entryKeyComparator = entryKeyComparator; + this.fileWriterManager = fileWriterManager; + } + + void compact(Path basePath, FileManager fileManager) throws IOException { + long indexOffset = Long.BYTES; + long offset; + compactedFile = basePath.resolve(TEMP_FILE_NAME); + compactedIndex = basePath.resolve(TEMP_FILE_INDEX_NAME); + + Iterator> iterator = + MergedIterator.createMergedIterator(fileManager.createIterators(null, null), entryKeyComparator); + + long storageSize = 0; + long tableSize = 0; + while (iterator.hasNext()) { + Entry currentItem = iterator.next(); + tableSize += 2 * Integer.BYTES + currentItem.key().byteSize(); + if (currentItem.value() != null) { + tableSize += currentItem.value().byteSize(); + } + + storageSize++; + } + + iterator = MergedIterator.createMergedIterator(fileManager.createIterators(null, null), entryKeyComparator); + long tableOffset = 0; + + try (FileChannel tableChannel = FileChannel.open(compactedFile, StandardOpenOption.READ, + StandardOpenOption.WRITE, StandardOpenOption.CREATE); + FileChannel indexChannel = FileChannel.open(compactedIndex, StandardOpenOption.READ, + StandardOpenOption.WRITE, StandardOpenOption.CREATE); + Arena arena = Arena.ofConfined()) { + MemorySegment tableMemorySegment = tableChannel.map(FileChannel.MapMode.READ_WRITE, + 0, tableSize, arena); + MemorySegment indexMemorySegment = indexChannel.map(FileChannel.MapMode.READ_WRITE, + 0, (storageSize + 1) * Long.BYTES, arena); + fileWriterManager.writeStorageSize(indexMemorySegment, storageSize); + while (iterator.hasNext()) { + Entry currentItem = iterator.next(); + offset = fileWriterManager.writeEntry(tableMemorySegment, tableOffset, currentItem); + fileWriterManager.writeIndexes(indexMemorySegment, indexOffset, tableOffset); + tableOffset = offset; + indexOffset += Long.BYTES; + } + } + + Files.move(compactedFile, basePath.resolve(MAIN_FILE_NAME), ATOMIC_MOVE); + Files.move(compactedIndex, basePath.resolve(MAIN_FILE_INDEX_NAME), ATOMIC_MOVE); + compactedFile = basePath.resolve(MAIN_FILE_NAME); + compactedIndex = basePath.resolve(MAIN_FILE_INDEX_NAME); + } + + void renameCompactFile(Path basePath) throws IOException { + Files.move(compactedFile, basePath.resolve(fileName + "0" + fileExtension), ATOMIC_MOVE); + Files.move(compactedIndex, basePath.resolve(fileIndexName + "0" + fileExtension), ATOMIC_MOVE); + } + + void deleteAllFiles(List ssTables, List ssIndexes) { + for (Path ssTable : ssTables) { + try { + Files.deleteIfExists(ssTable); + } catch (IOException e) { + throw new UncheckedIOException("Error deleting a ssTable file.", e); + } + } + + for (Path ssIndex : ssIndexes) { + try { + Files.deleteIfExists(ssIndex); + } catch (IOException e) { + throw new UncheckedIOException("Error deleting a ssIndex file.", e); + } + } + } + + boolean deleteTempFile(Path basePath, Path file, List files) throws IOException { + if (file.equals(basePath.resolve(TEMP_FILE_NAME))) { + Files.delete(basePath.resolve(TEMP_FILE_NAME)); + if (files.contains(basePath.resolve(TEMP_FILE_INDEX_NAME))) { + Files.delete(basePath.resolve(TEMP_FILE_INDEX_NAME)); + } + + return true; + } + + return false; + } + + boolean clearIfCompactFileExists(Path basePath, Path file, List files) throws IOException { + if (file.equals(basePath.resolve(MAIN_FILE_NAME))) { + if (files.contains(basePath.resolve(MAIN_FILE_INDEX_NAME))) { + List ssTables = new ArrayList<>(); + List ssIndexes = new ArrayList<>(); + for (Path currentFile : files) { + if (currentFile.getFileName().toString().startsWith(fileName)) { + ssTables.add(currentFile); + } + + if (currentFile.getFileName().toString().startsWith(fileIndexName)) { + ssIndexes.add(currentFile); + } + } + + ssTables.sort(new PathsComparator(fileName, fileExtension)); + ssIndexes.sort(new PathsComparator(fileIndexName, fileExtension)); + deleteFiles(ssTables); + deleteFiles(ssIndexes); + + Files.move(basePath.resolve(MAIN_FILE_NAME), + basePath.resolve(fileName + "0" + fileExtension), ATOMIC_MOVE); + Files.move(basePath.resolve(MAIN_FILE_INDEX_NAME), + basePath.resolve(fileIndexName + "0" + fileExtension), ATOMIC_MOVE); + } else { + Files.delete(basePath.resolve(MAIN_FILE_NAME)); + } + + return true; + } + + return false; + } + + private void deleteFiles(List filePaths) throws IOException { + for (Path filePath : filePaths) { + Files.delete(filePath); + } + } +} diff --git a/src/main/java/ru/vk/itmo/dyagayalexandra/EntryKeyComparator.java b/src/main/java/ru/vk/itmo/dyagayalexandra/EntryKeyComparator.java index 9ccd7f440..31b635741 100644 --- a/src/main/java/ru/vk/itmo/dyagayalexandra/EntryKeyComparator.java +++ b/src/main/java/ru/vk/itmo/dyagayalexandra/EntryKeyComparator.java @@ -7,7 +7,11 @@ public class EntryKeyComparator implements Comparator> { - private final MemorySegmentComparator memorySegmentComparator = new MemorySegmentComparator(); + private final MemorySegmentComparator memorySegmentComparator; + + public EntryKeyComparator(MemorySegmentComparator memorySegmentComparator) { + this.memorySegmentComparator = memorySegmentComparator; + } @Override public int compare(Entry entry1, Entry entry2) { diff --git a/src/main/java/ru/vk/itmo/dyagayalexandra/FileChecker.java b/src/main/java/ru/vk/itmo/dyagayalexandra/FileChecker.java new file mode 100644 index 000000000..3c66605b2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/dyagayalexandra/FileChecker.java @@ -0,0 +1,138 @@ +package ru.vk.itmo.dyagayalexandra; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class FileChecker { + + private final String fileName; + private final String fileExtension; + private final String fileIndexName; + private final CompactManager compactManager; + + public FileChecker(String fileName, String fileIndexName, String fileExtension, CompactManager compactManager) { + this.fileName = fileName; + this.fileIndexName = fileIndexName; + this.fileExtension = fileExtension; + this.compactManager = compactManager; + } + + public List> checkFiles(Path basePath, Arena arena) throws IOException { + List> allFiles = new ArrayList<>(); + List ssTablesPaths = new ArrayList<>(); + List ssIndexesPaths = new ArrayList<>(); + + List files = getAllFiles(basePath); + checkFile(basePath, files); + + List> allDataPaths = getAllDataPaths(basePath); + + for (Map.Entry entry : allDataPaths) { + ssTablesPaths.add(entry.getKey()); + ssIndexesPaths.add(entry.getValue()); + } + + ssTablesPaths.sort(new PathsComparator(fileName, fileExtension)); + ssIndexesPaths.sort(new PathsComparator(fileIndexName, fileExtension)); + + if (ssTablesPaths.size() != ssIndexesPaths.size()) { + throw new NoSuchFileException("Not all files found."); + } + + for (int i = 0; i < ssTablesPaths.size(); i++) { + MemorySegment data; + MemorySegment index; + Path dataPath = ssTablesPaths.get(i); + Path indexPath = ssIndexesPaths.get(i); + + try (FileChannel fileChannel = FileChannel.open(dataPath)) { + data = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, fileChannel.size(), arena); + } + + try (FileChannel fileChannel = FileChannel.open(indexPath)) { + index = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, fileChannel.size(), arena); + } + + checkFileMatch(dataPath, indexPath); + allFiles.add(new AbstractMap.SimpleEntry<>(data, index)); + } + + return allFiles; + } + + public List> getAllDataPaths(Path basePath) throws IOException { + List files = getAllFiles(basePath); + List ssTablesPaths = new ArrayList<>(); + List ssIndexesPaths = new ArrayList<>(); + List> filePathsMap = new ArrayList<>(); + for (Path file : files) { + if (String.valueOf(file.getFileName()).startsWith(fileName)) { + ssTablesPaths.add(file); + } + + if (String.valueOf(file.getFileName()).startsWith(fileIndexName)) { + ssIndexesPaths.add(file); + } + } + + ssTablesPaths.sort(new PathsComparator(fileName, fileExtension)); + ssIndexesPaths.sort(new PathsComparator(fileIndexName, fileExtension)); + + int size = ssTablesPaths.size(); + for (int i = 0; i < size; i++) { + filePathsMap.add(new AbstractMap.SimpleEntry<>(ssTablesPaths.get(i), ssIndexesPaths.get(i))); + } + + return filePathsMap; + } + + private void checkFile(Path basePath, List files) throws IOException { + for (Path file : files) { + if (compactManager.deleteTempFile(basePath, file, files)) { + break; + } + + if (compactManager.clearIfCompactFileExists(basePath, file, files)) { + break; + } + } + } + + private void checkFileMatch(Path data, Path index) throws IOException { + String dataString = data.toString(); + String indexString = index.toString(); + if (Integer.parseInt(dataString.substring( + dataString.indexOf(fileName) + fileName.length(), + dataString.indexOf(fileExtension))) + != Integer.parseInt(indexString.substring( + indexString.indexOf(fileIndexName) + fileIndexName.length(), + indexString.indexOf(fileExtension)))) { + throw new NoSuchFileException("The files don't match."); + } + } + + private List getAllFiles(Path basePath) throws IOException { + List files = new ArrayList<>(); + if (!Files.exists(basePath)) { + Files.createDirectory(basePath); + } + + try (DirectoryStream directoryStream = Files.newDirectoryStream(basePath)) { + for (Path path : directoryStream) { + files.add(path); + } + } + + return files; + } +} diff --git a/src/main/java/ru/vk/itmo/dyagayalexandra/FileIterator.java b/src/main/java/ru/vk/itmo/dyagayalexandra/FileIterator.java index bd7dc8aec..8cc4c3483 100644 --- a/src/main/java/ru/vk/itmo/dyagayalexandra/FileIterator.java +++ b/src/main/java/ru/vk/itmo/dyagayalexandra/FileIterator.java @@ -13,13 +13,15 @@ public class FileIterator implements Iterator> { private final MemorySegment ssIndex; private long currentIndex; private final long endIndex; + private final FileReaderManager fileReaderManager; public FileIterator(MemorySegment ssTable, MemorySegment ssIndex, MemorySegment from, - MemorySegment to, long indexSize) throws IOException { + MemorySegment to, long indexSize, FileReaderManager fileReaderManager) throws IOException { this.ssTable = ssTable; this.ssIndex = ssIndex; - currentIndex = from == null ? 0 : FileManager.getEntryIndex(ssTable, ssIndex, from, indexSize); - endIndex = to == null ? indexSize : FileManager.getEntryIndex(ssTable, ssIndex, to, indexSize); + currentIndex = from == null ? 0 : fileReaderManager.getEntryIndex(ssTable, ssIndex, from, indexSize); + endIndex = to == null ? indexSize : fileReaderManager.getEntryIndex(ssTable, ssIndex, to, indexSize); + this.fileReaderManager = fileReaderManager; } @Override @@ -35,10 +37,11 @@ public Entry next() { Entry entry; try { - entry = FileManager.getCurrentEntry(currentIndex, ssTable, ssIndex); + entry = fileReaderManager.getCurrentEntry(currentIndex, ssTable, ssIndex); } catch (IOException e) { throw new NoSuchElementException("There is no next element.", e); } + currentIndex++; return entry; } diff --git a/src/main/java/ru/vk/itmo/dyagayalexandra/FileManager.java b/src/main/java/ru/vk/itmo/dyagayalexandra/FileManager.java index a3e54fef7..e5c13d097 100644 --- a/src/main/java/ru/vk/itmo/dyagayalexandra/FileManager.java +++ b/src/main/java/ru/vk/itmo/dyagayalexandra/FileManager.java @@ -1,203 +1,186 @@ package ru.vk.itmo.dyagayalexandra; -import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Config; import ru.vk.itmo.Entry; import java.io.IOException; -import java.io.RandomAccessFile; import java.io.UncheckedIOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; -import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.ArrayList; +import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.NavigableMap; -import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class FileManager { - - private long filesCount; private static final String FILE_NAME = "data"; private static final String FILE_INDEX_NAME = "index"; private static final String FILE_EXTENSION = ".txt"; private final Path basePath; - private final List ssTables; - private final List ssIndexes; - private final Map ssTableIndexStorage; + private final List ssTables; + private final List ssIndexes; + private final List ssTablesPaths; + private final List ssIndexesPaths; + private final CompactManager compactManager; + private final FileWriterManager fileWriterManager; + private final FileReaderManager fileReaderManager; + private final MemorySegmentComparator memorySegmentComparator; private final Arena arena; + private final ReadWriteLock lock = new ReentrantReadWriteLock(); - public FileManager(Config config) { + public FileManager(Config config, MemorySegmentComparator memorySegmentComparator, + EntryKeyComparator entryKeyComparator) { basePath = config.basePath(); ssTables = new ArrayList<>(); ssIndexes = new ArrayList<>(); - ssTableIndexStorage = new ConcurrentHashMap<>(); + ssTablesPaths = new ArrayList<>(); + ssIndexesPaths = new ArrayList<>(); arena = Arena.ofShared(); - if (Files.exists(basePath)) { - try (DirectoryStream directoryStream = Files.newDirectoryStream(basePath)) { - for (Path path : directoryStream) { - String fileName = String.valueOf(path.getFileName()); - if (fileName.contains(FILE_NAME)) { - ssTables.add(path); - } else if (fileName.contains(FILE_INDEX_NAME)) { - ssIndexes.add(path); - } - } + this.memorySegmentComparator = memorySegmentComparator; + fileWriterManager = new FileWriterManager(FILE_NAME, FILE_EXTENSION, FILE_INDEX_NAME, basePath, arena); + fileReaderManager = new FileReaderManager(memorySegmentComparator); + compactManager = new CompactManager(FILE_NAME, FILE_INDEX_NAME, FILE_EXTENSION, entryKeyComparator, + fileWriterManager); + FileChecker fileChecker = new FileChecker(FILE_NAME, FILE_INDEX_NAME, FILE_EXTENSION, compactManager); + try { + List> allDataSegments = fileChecker.checkFiles(basePath, arena); + getData(allDataSegments, fileChecker.getAllDataPaths(basePath)); + } catch (IOException e) { + throw new UncheckedIOException("Error checking files.", e); + } + } + + Entry get(MemorySegment key) { + for (int i = 0; i < ssTables.size(); i++) { + FileIterator fileIterator; + try { + fileIterator = new FileIterator(ssTables.get(i), ssIndexes.get(i), key, null, + fileReaderManager.getIndexSize(ssIndexes.get(i)), fileReaderManager); } catch (IOException e) { - throw new UncheckedIOException("An error occurred while reading the file.", e); + throw new UncheckedIOException("Failed to create FileIterator", e); } - } - ssTables.sort(new PathsComparator(FILE_NAME, FILE_EXTENSION)); - ssIndexes.sort(new PathsComparator(FILE_INDEX_NAME, FILE_EXTENSION)); - filesCount = ssTables.size(); - for (int i = 0; i < filesCount; i++) { - ssTableIndexStorage.put(ssIndexes.get(i), getIndexSize(ssIndexes.get(i))); + if (fileIterator.hasNext()) { + Entry currentEntry = fileIterator.next(); + if (currentEntry != null && memorySegmentComparator.compare(currentEntry.key(), key) == 0) { + return currentEntry; + } + } } + + return null; } - public void save(NavigableMap> storage) throws IOException { - if (storage.isEmpty()) { + void performCompact() { + if (ssTables.size() <= 1) { return; } - saveData(storage); - saveIndexes(storage); - filesCount++; + try { + compactManager.compact(basePath, this); + compactManager.deleteAllFiles(ssTablesPaths, ssIndexesPaths); + compactManager.renameCompactFile(basePath); + afterCompact(); + } catch (IOException e) { + throw new UncheckedIOException("Failed to complete compact.", e); + } + } + + void flush(Collection> entryCollection) { + try { + FileWriterManager.SavedFilesInfo savedFilesInfo = + fileWriterManager.save(entryCollection, ssTables.size(), ssIndexes.size()); + lock.readLock().lock(); + try { + ssTables.addFirst(savedFilesInfo.getSSTable()); + ssIndexes.addFirst(savedFilesInfo.getSSIndex()); + ssTablesPaths.addFirst(savedFilesInfo.getSSTablePath()); + ssIndexesPaths.addFirst(savedFilesInfo.getSSIndexPath()); + } finally { + lock.readLock().unlock(); + } + } catch (IOException e) { + throw new UncheckedIOException("Error saving storage.", e); + } } - public Iterator> createIterators(MemorySegment from, MemorySegment to) { - List peekIterators = new ArrayList<>(); + List>> createIterators(MemorySegment from, MemorySegment to) { + List>> iterators = new ArrayList<>(); for (int i = 0; i < ssTables.size(); i++) { - Iterator> iterator = createFileIterator(ssTables.get(i), ssIndexes.get(i), from, to); - peekIterators.add(new PeekingIterator(i, iterator)); + iterators.add(createFileIterator(ssTables.get(i), ssIndexes.get(i), from, to)); } - return MergedIterator.createMergedIterator(peekIterators, new EntryKeyComparator()); + return iterators; } - private Iterator> createFileIterator(Path ssTable, Path ssIndex, + private Iterator> createFileIterator(MemorySegment ssTable, MemorySegment ssIndex, MemorySegment from, MemorySegment to) { - long indexSize = ssTableIndexStorage.get(ssIndex); - - FileIterator fileIterator; - try (FileChannel ssTableFileChannel = FileChannel.open(ssTable, StandardOpenOption.READ, - StandardOpenOption.WRITE); - FileChannel ssIndexFileChannel = FileChannel.open(ssIndex, StandardOpenOption.READ, - StandardOpenOption.WRITE)) { - MemorySegment ssTableMemorySegment = ssTableFileChannel.map(FileChannel.MapMode.READ_WRITE, - 0, ssTableFileChannel.size(), arena); - MemorySegment ssIndexMemorySegment = ssIndexFileChannel.map(FileChannel.MapMode.READ_WRITE, - 0, ssIndexFileChannel.size(), arena); - - fileIterator = new FileIterator(ssTableMemorySegment, ssIndexMemorySegment, from, to, indexSize); - return fileIterator; + try { + long indexSize = fileReaderManager.getIndexSize(ssIndex); + return new FileIterator(ssTable, ssIndex, from, to, indexSize, fileReaderManager); } catch (IOException e) { throw new UncheckedIOException("An error occurred while reading files.", e); } } - private void saveData(NavigableMap> storage) throws IOException { - Path filePath = basePath.resolve(FILE_NAME + filesCount + FILE_EXTENSION); - if (!Files.exists(filePath)) { - Files.createFile(filePath); + private void getData(List> allDataSegments, + List> allDataPaths) { + for (Map.Entry entry : allDataSegments) { + ssTables.add(entry.getKey()); + ssIndexes.add(entry.getValue()); } - try (RandomAccessFile randomAccessFile = new RandomAccessFile(String.valueOf(filePath), "rw")) { - for (Map.Entry> entry : storage.entrySet()) { - Entry entryValue = entry.getValue(); - randomAccessFile.writeInt((int) entryValue.key().byteSize()); - randomAccessFile.write(entryValue.key().toArray(ValueLayout.JAVA_BYTE)); - - if (entryValue.value() == null) { - randomAccessFile.writeInt(-1); - } else { - randomAccessFile.writeInt((int) entryValue.value().byteSize()); - randomAccessFile.write(entryValue.value().toArray(ValueLayout.JAVA_BYTE)); - } - } + for (Map.Entry entry : allDataPaths) { + ssTablesPaths.add(entry.getKey()); + ssIndexesPaths.add(entry.getValue()); } } - private void saveIndexes(NavigableMap> storage) throws IOException { - Path indexPath = basePath.resolve(FILE_INDEX_NAME + filesCount + FILE_EXTENSION); - if (!Files.exists(indexPath)) { - Files.createFile(indexPath); - } + private void afterCompact() throws IOException { + Path ssTablePath = basePath.resolve(FILE_NAME + "0" + FILE_EXTENSION); + Path ssIndexPath = basePath.resolve(FILE_INDEX_NAME + "0" + FILE_EXTENSION); - try (RandomAccessFile randomAccessFile = new RandomAccessFile(String.valueOf(indexPath), "rw")) { - randomAccessFile.writeLong(storage.size()); - long offset = 0; - for (Map.Entry> entry : storage.entrySet()) { - randomAccessFile.writeLong(offset); - Entry entryValue = entry.getValue(); - offset += Integer.BYTES + entryValue.key().byteSize(); - offset += Integer.BYTES; - if (entryValue.value() != null) { - offset += entryValue.value().byteSize(); - } - } - } - } + MemorySegment tableMemorySegment; + MemorySegment indexMemorySegment; - private long getIndexSize(Path indexPath) { - long size; - try (RandomAccessFile raf = new RandomAccessFile(indexPath.toString(), "r")) { - size = raf.readLong(); - } catch (IOException e) { - throw new UncheckedIOException("Unable to read file.", e); + try (FileChannel tableChannel = FileChannel.open(ssTablePath, StandardOpenOption.READ, + StandardOpenOption.WRITE, StandardOpenOption.CREATE)) { + tableMemorySegment = tableChannel.map(FileChannel.MapMode.READ_WRITE, + 0, Files.size(ssTablePath), arena); } - return size; - } - - static long getEntryIndex(MemorySegment ssTable, MemorySegment ssIndex, - MemorySegment key, long indexSize) throws IOException { - long low = 0; - long high = indexSize - 1; - long mid = (low + high) / 2; - while (low <= high) { - Entry current = getCurrentEntry(mid, ssTable, ssIndex); - int compare = new MemorySegmentComparator().compare(key, current.key()); - if (compare > 0) { - low = mid + 1; - } else if (compare < 0) { - high = mid - 1; - } else { - return mid; - } - mid = (low + high) / 2; + try (FileChannel indexChannel = FileChannel.open(ssIndexPath, StandardOpenOption.READ, + StandardOpenOption.WRITE, StandardOpenOption.CREATE)) { + indexMemorySegment = indexChannel.map(FileChannel.MapMode.READ_WRITE, + 0, Files.size(ssIndexPath), arena); } - return low; - } - - static Entry getCurrentEntry(long position, MemorySegment ssTable, - MemorySegment ssIndex) throws IOException { - long offset = ssIndex.asSlice((position + 1) * Long.BYTES, - Long.BYTES).asByteBuffer().getLong(); - int keyLength = ssTable.asSlice(offset, Integer.BYTES).asByteBuffer().getInt(); - offset += Integer.BYTES; - byte[] keyByteArray = ssTable.asSlice(offset, keyLength).toArray(ValueLayout.JAVA_BYTE); - offset += keyLength; - int valueLength = ssTable.asSlice(offset, Integer.BYTES).asByteBuffer().getInt(); - if (valueLength == -1) { - return new BaseEntry<>(MemorySegment.ofArray(keyByteArray), null); + lock.readLock().lock(); + try { + ssTables.clear(); + ssIndexes.clear(); + ssTablesPaths.clear(); + ssIndexesPaths.clear(); + ssTables.add(tableMemorySegment); + ssIndexes.add(indexMemorySegment); + ssTablesPaths.add(ssTablePath); + ssIndexesPaths.add(ssIndexPath); + } finally { + lock.readLock().unlock(); } - - offset += Integer.BYTES; - byte[] valueByteArray = ssTable.asSlice(offset, valueLength).toArray(ValueLayout.JAVA_BYTE); - return new BaseEntry<>(MemorySegment.ofArray(keyByteArray), MemorySegment.ofArray(valueByteArray)); } - public void closeArena() { + void closeArena() { + if (arena == null || !arena.scope().isAlive()) { + return; + } arena.close(); } } diff --git a/src/main/java/ru/vk/itmo/dyagayalexandra/FileReaderManager.java b/src/main/java/ru/vk/itmo/dyagayalexandra/FileReaderManager.java new file mode 100644 index 000000000..a9ec5b95f --- /dev/null +++ b/src/main/java/ru/vk/itmo/dyagayalexandra/FileReaderManager.java @@ -0,0 +1,64 @@ +package ru.vk.itmo.dyagayalexandra; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +public class FileReaderManager { + + private final MemorySegmentComparator memorySegmentComparator; + + public FileReaderManager(MemorySegmentComparator memorySegmentComparator) { + this.memorySegmentComparator = memorySegmentComparator; + } + + long getIndexSize(MemorySegment indexMemorySegment) { + return indexMemorySegment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + + long getEntryIndex(MemorySegment ssTable, MemorySegment ssIndex, + MemorySegment key, long indexSize) throws IOException { + long low = 0; + long high = indexSize - 1; + long mid = (low + high) / 2; + while (low <= high) { + Entry current = getCurrentEntry(mid, ssTable, ssIndex); + int compare = memorySegmentComparator.compare(key, current.key()); + if (compare > 0) { + low = mid + 1; + } else if (compare < 0) { + high = mid - 1; + } else { + return mid; + } + mid = (low + high) / 2; + } + + return low; + } + + Entry getCurrentEntry(long position, MemorySegment ssTable, + MemorySegment ssIndex) throws IOException { + long offset = ssIndex.get(ValueLayout.JAVA_LONG_UNALIGNED, (position + 1) * Long.BYTES); + + int keyLength = ssTable.get(ValueLayout.JAVA_INT_UNALIGNED, offset); + offset += Integer.BYTES; + + MemorySegment keyMemorySegment = ssTable.asSlice(offset, keyLength); + offset += keyLength; + + int valueLength = ssTable.get(ValueLayout.JAVA_INT_UNALIGNED, offset); + offset += Integer.BYTES; + + if (valueLength == -1) { + return new BaseEntry<>(keyMemorySegment, null); + } + + MemorySegment valueMemorySegment = ssTable.asSlice(offset, valueLength); + + return new BaseEntry<>(keyMemorySegment, valueMemorySegment); + } +} diff --git a/src/main/java/ru/vk/itmo/dyagayalexandra/FileWriterManager.java b/src/main/java/ru/vk/itmo/dyagayalexandra/FileWriterManager.java new file mode 100644 index 000000000..4f2cd0648 --- /dev/null +++ b/src/main/java/ru/vk/itmo/dyagayalexandra/FileWriterManager.java @@ -0,0 +1,148 @@ +package ru.vk.itmo.dyagayalexandra; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Collection; +import java.util.Iterator; + +public class FileWriterManager { + + private final String fileName; + private final String fileExtension; + private final String fileIndexName; + private final Path basePath; + private final Arena arena; + + public FileWriterManager(String fileName, String fileExtension, String fileIndexName, Path basePath, Arena arena) { + this.fileName = fileName; + this.fileExtension = fileExtension; + this.fileIndexName = fileIndexName; + this.basePath = basePath; + this.arena = arena; + } + + SavedFilesInfo save(Collection> entryCollection, + int tablesSize, int indexesSize) throws IOException { + Path tablePath = basePath.resolve(fileName + tablesSize + fileExtension); + Path indexPath = basePath.resolve(fileIndexName + indexesSize + fileExtension); + + long tableSize = 0; + + Iterator> storageIterator = entryCollection.iterator(); + long storageSize = 0; + + while (storageIterator.hasNext()) { + Entry entry = storageIterator.next(); + tableSize += 2 * Integer.BYTES + entry.key().byteSize(); + if (entry.value() != null) { + tableSize += entry.value().byteSize(); + } + storageSize++; + } + + MemorySegment tableMemorySegment; + MemorySegment indexMemorySegment; + + try (FileChannel tableChannel = FileChannel.open(tablePath, StandardOpenOption.READ, + StandardOpenOption.WRITE, StandardOpenOption.CREATE)) { + tableMemorySegment = tableChannel.map(FileChannel.MapMode.READ_WRITE, + 0, tableSize, arena); + } + + try (FileChannel indexChannel = FileChannel.open(indexPath, StandardOpenOption.READ, + StandardOpenOption.WRITE, StandardOpenOption.CREATE)) { + indexMemorySegment = indexChannel.map(FileChannel.MapMode.READ_WRITE, + 0, (storageSize + 1) * Long.BYTES, arena); + } + + long indexOffset = Long.BYTES; + long offset; + long tableOffset = 0; + + storageIterator = entryCollection.iterator(); + + writeStorageSize(indexMemorySegment, storageSize); + for (int i = 0; i < storageSize; i++) { + Entry entry = storageIterator.next(); + offset = writeEntry(tableMemorySegment, tableOffset, entry); + writeIndexes(indexMemorySegment, indexOffset, tableOffset); + tableOffset = offset; + indexOffset += Long.BYTES; + } + + return new SavedFilesInfo(tableMemorySegment, indexMemorySegment, tablePath, indexPath); + } + + long writeEntry(MemorySegment tableMemorySegment, long offset, Entry entry) { + long tableFileOffset = offset; + + int keyLength = (int) entry.key().byteSize(); + + tableMemorySegment.set(ValueLayout.JAVA_INT_UNALIGNED, tableFileOffset, keyLength); + tableFileOffset += Integer.BYTES; + + MemorySegment.copy(entry.key(), 0, tableMemorySegment, tableFileOffset, keyLength); + tableFileOffset += keyLength; + + if (entry.value() == null) { + tableMemorySegment.set(ValueLayout.JAVA_INT_UNALIGNED, tableFileOffset, -1); + tableFileOffset += Integer.BYTES; + } else { + int valueLength = (int) entry.value().byteSize(); + + tableMemorySegment.set(ValueLayout.JAVA_INT_UNALIGNED, tableFileOffset, valueLength); + tableFileOffset += Integer.BYTES; + + MemorySegment.copy(entry.value(), 0, tableMemorySegment, tableFileOffset, valueLength); + tableFileOffset += valueLength; + } + + return tableFileOffset; + } + + void writeIndexes(MemorySegment indexMemorySegment, long indexOffset, long offset) { + indexMemorySegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, offset); + } + + void writeStorageSize(MemorySegment indexMemorySegment, long storageSize) { + indexMemorySegment.set(ValueLayout.JAVA_LONG_UNALIGNED, 0, storageSize); + } + + public static class SavedFilesInfo { + + private final MemorySegment ssTable; + private final MemorySegment ssIndex; + private final Path ssTablePath; + private final Path ssIndexPath; + + public SavedFilesInfo(MemorySegment ssTable, MemorySegment ssIndex, Path ssTablePath, Path ssIndexPath) { + this.ssTable = ssTable; + this.ssIndex = ssIndex; + this.ssTablePath = ssTablePath; + this.ssIndexPath = ssIndexPath; + } + + public MemorySegment getSSTable() { + return ssTable; + } + + public MemorySegment getSSIndex() { + return ssIndex; + } + + public Path getSSTablePath() { + return ssTablePath; + } + + public Path getSSIndexPath() { + return ssIndexPath; + } + } +} diff --git a/src/main/java/ru/vk/itmo/dyagayalexandra/MergedIterator.java b/src/main/java/ru/vk/itmo/dyagayalexandra/MergedIterator.java index a9e765ffa..8300b4ed0 100644 --- a/src/main/java/ru/vk/itmo/dyagayalexandra/MergedIterator.java +++ b/src/main/java/ru/vk/itmo/dyagayalexandra/MergedIterator.java @@ -1,27 +1,22 @@ package ru.vk.itmo.dyagayalexandra; -import ru.vk.itmo.Entry; - -import java.lang.foreign.MemorySegment; import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.PriorityQueue; -public final class MergedIterator implements Iterator> { +public final class MergedIterator implements Iterator { - private final PriorityQueue iterators; - private final Comparator> comparator; + private final PriorityQueue> iterators; + private final Comparator comparator; - private MergedIterator(PriorityQueue iterators, Comparator> comparator) { + private MergedIterator(PriorityQueue> iterators, Comparator comparator) { this.iterators = iterators; this.comparator = comparator; } - public static Iterator> createMergedIterator(List iterators, - Comparator> comparator) { - + public static Iterator createMergedIterator(List> iterators, Comparator comparator) { if (iterators.isEmpty()) { return Collections.emptyIterator(); } @@ -30,22 +25,28 @@ public static Iterator> createMergedIterator(List queue = new PriorityQueue<>(iterators.size(), (iterator1, iterator2) -> { + PriorityQueue> queue = + new PriorityQueue<>(iterators.size(), (iterator1, iterator2) -> { int result = comparator.compare(iterator1.peek(), iterator2.peek()); if (result != 0) { return result; } - return Integer.compare(iterator1.getIndex(), iterator2.getIndex()); + return Integer.compare(iterator1.index, iterator2.index); }); - for (PeekingIterator iterator : iterators) { + int index = 0; + for (Iterator iterator : iterators) { + if (iterator == null) { + continue; + } + if (iterator.hasNext()) { - queue.add(iterator); + queue.add(new IteratorWrapper<>(index++, iterator)); } } - return new MergedIterator(queue, comparator); + return new MergedIterator<>(queue, comparator); } @Override @@ -54,11 +55,11 @@ public boolean hasNext() { } @Override - public Entry next() { - PeekingIterator iterator = iterators.remove(); - Entry next = iterator.next(); + public E next() { + IteratorWrapper iterator = iterators.remove(); + E next = iterator.next(); while (!iterators.isEmpty()) { - PeekingIterator candidate = iterators.peek(); + IteratorWrapper candidate = iterators.peek(); if (comparator.compare(next, candidate.peek()) != 0) { break; } @@ -76,4 +77,15 @@ public Entry next() { return next; } + + private static class IteratorWrapper extends PeekingIterator { + + final int index; + + public IteratorWrapper(int index, Iterator iterator) { + super(iterator); + this.index = index; + } + + } } diff --git a/src/main/java/ru/vk/itmo/dyagayalexandra/PeekingIterator.java b/src/main/java/ru/vk/itmo/dyagayalexandra/PeekingIterator.java index e92ded105..9ae244aae 100644 --- a/src/main/java/ru/vk/itmo/dyagayalexandra/PeekingIterator.java +++ b/src/main/java/ru/vk/itmo/dyagayalexandra/PeekingIterator.java @@ -1,26 +1,17 @@ package ru.vk.itmo.dyagayalexandra; -import ru.vk.itmo.Entry; - -import java.lang.foreign.MemorySegment; import java.util.Iterator; -public class PeekingIterator implements Iterator> { +public class PeekingIterator implements Iterator { - private final int index; - private final Iterator> iterator; - private Entry peekedEntry; + private final Iterator iterator; + private E peekedEntry; - public PeekingIterator(int index, Iterator> iterator) { - this.index = index; + public PeekingIterator(Iterator iterator) { this.iterator = iterator; } - public int getIndex() { - return index; - } - - public Entry peek() { + public E peek() { if (peekedEntry == null && iterator.hasNext()) { peekedEntry = iterator.next(); } @@ -34,8 +25,8 @@ public boolean hasNext() { } @Override - public Entry next() { - Entry result = peek(); + public E next() { + E result = peek(); peekedEntry = null; return result; } diff --git a/src/main/java/ru/vk/itmo/dyagayalexandra/SkipNullIterator.java b/src/main/java/ru/vk/itmo/dyagayalexandra/SkipNullIterator.java index dfa96e01c..50e2fce8a 100644 --- a/src/main/java/ru/vk/itmo/dyagayalexandra/SkipNullIterator.java +++ b/src/main/java/ru/vk/itmo/dyagayalexandra/SkipNullIterator.java @@ -8,19 +8,28 @@ public class SkipNullIterator implements Iterator> { - private final PeekingIterator iterator; + private final Iterator> iterator; + private Entry current; - public SkipNullIterator(PeekingIterator iterator) { + public SkipNullIterator(Iterator> iterator) { this.iterator = iterator; } @Override public boolean hasNext() { - while (iterator.hasNext() && iterator.peek().value() == null) { - iterator.next(); + if (current != null) { + return true; } - return iterator.hasNext(); + while (iterator.hasNext()) { + Entry entry = iterator.next(); + if (entry.value() != null) { + this.current = entry; + return true; + } + } + + return false; } @Override @@ -29,6 +38,8 @@ public Entry next() { throw new NoSuchElementException("There is no next element."); } - return iterator.next(); + Entry next = current; + current = null; + return next; } } diff --git a/src/main/java/ru/vk/itmo/dyagayalexandra/Storage.java b/src/main/java/ru/vk/itmo/dyagayalexandra/Storage.java index 8e625e4a6..c0513fe30 100644 --- a/src/main/java/ru/vk/itmo/dyagayalexandra/Storage.java +++ b/src/main/java/ru/vk/itmo/dyagayalexandra/Storage.java @@ -6,96 +6,241 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.NavigableMap; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class Storage implements Dao> { - private final NavigableMap> dataStorage; - private final FileManager fileManager; - private static final MemorySegmentComparator COMPARATOR = new MemorySegmentComparator(); + private FileManager fileManager; + private final AtomicBoolean isClosed = new AtomicBoolean(); + private final List> taskResults = new CopyOnWriteArrayList<>(); + private final ExecutorService service = + Executors.newSingleThreadExecutor(r -> new Thread(r, "BackgroundFlushAndCompact")); + private long flushThresholdBytes; + private State state; + private static final MemorySegmentComparator memorySegmentComparator = new MemorySegmentComparator(); + private final EntryKeyComparator entryKeyComparator = new EntryKeyComparator(memorySegmentComparator); + private final ReadWriteLock lock = new ReentrantReadWriteLock(); public Storage() { - dataStorage = new ConcurrentSkipListMap<>(new MemorySegmentComparator()); - fileManager = null; + // Default constructor } public Storage(Config config) { - dataStorage = new ConcurrentSkipListMap<>(new MemorySegmentComparator()); - fileManager = new FileManager(config); + fileManager = new FileManager(config, memorySegmentComparator, entryKeyComparator); + state = State.emptyState(fileManager); + flushThresholdBytes = config.flushThresholdBytes(); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { - Iterator> memoryIterator; + if (isClosed.get()) { + throw new IllegalStateException("Unable to get: close operation performed."); + } + + ArrayList>> iterators = new ArrayList<>(); if (from == null && to == null) { - memoryIterator = dataStorage.values().iterator(); + iterators.add(state.dataStorage.values().iterator()); } else if (from == null) { - memoryIterator = dataStorage.headMap(to).values().iterator(); + iterators.add(state.dataStorage.headMap(to).values().iterator()); } else if (to == null) { - memoryIterator = dataStorage.tailMap(from).values().iterator(); + iterators.add(state.dataStorage.tailMap(from).values().iterator()); } else { - memoryIterator = dataStorage.subMap(from, to).values().iterator(); + iterators.add(state.dataStorage.subMap(from, to).values().iterator()); } - Iterator> iterators = null; - if (fileManager != null) { - iterators = fileManager.createIterators(from, to); + iterators.add(state.getFlushingPairsIterator()); + iterators.addAll(state.fileManager.createIterators(from, to)); + Iterator> mergedIterator = + MergedIterator.createMergedIterator(iterators, entryKeyComparator); + return new SkipNullIterator(mergedIterator); + } + + @Override + public Entry get(MemorySegment key) { + if (isClosed.get()) { + throw new IllegalStateException("Unable to get: close operation performed."); } - if (iterators == null) { - return memoryIterator; - } else { - Iterator> mergeIterator = MergedIterator.createMergedIterator( - List.of( - new PeekingIterator(0, memoryIterator), - new PeekingIterator(1, iterators) - ), - new EntryKeyComparator() - ); + Entry result = state.dataStorage.get(key); + + if (result == null) { + result = state.flushingDataStorage.get(key); + } - return new SkipNullIterator(new PeekingIterator(0, mergeIterator)); + if (result == null) { + result = state.fileManager.get(key); } + + return (result == null || result.value() == null) ? null : result; } @Override - public Entry get(MemorySegment key) { - if (fileManager == null) { - return dataStorage.get(key); - } else { - Iterator> iterator = get(key, null); - if (!iterator.hasNext()) { - return null; - } + public void upsert(Entry entry) { + if (isClosed.get()) { + throw new IllegalStateException("Unable to upsert: close operation performed."); + } - Entry next = iterator.next(); - if (COMPARATOR.compare(key, next.key()) == 0) { - return next; + if (entry == null || entry.key() == null) { + throw new IllegalArgumentException("Attempt to upsert empty entry."); + } + + long entryValueLength = entry.value() == null ? 0 : entry.value().byteSize(); + long delta = 2 * entry.key().byteSize() + entryValueLength; + + if (state.getDataStorageByteSize() + delta > flushThresholdBytes) { + if (!state.flushingDataStorage.isEmpty()) { + throw new IllegalStateException("Unable to flush: another background flush performing."); } + flush(); + } else { + state.updateDataStorageByteSize(delta); + } - return null; + lock.readLock().lock(); + try { + state.dataStorage.put(entry.key(), entry); + } finally { + lock.readLock().unlock(); } } @Override - public void upsert(Entry entry) { - dataStorage.put(entry.key(), entry); + public void flush() { + if (isClosed.get()) { + throw new IllegalStateException("Unable to flush: close operation performed."); + } + + if (state.dataStorage.isEmpty() || !state.flushingDataStorage.isEmpty()) { + return; + } + + performBackgroundFlush(); } @Override - public void flush() { - throw new UnsupportedOperationException("Flush is not supported!"); + public void compact() { + if (isClosed.get()) { + throw new IllegalStateException("Unable to compact: close operation performed."); + } + + performCompact(); } @Override public void close() throws IOException { - if (fileManager != null) { - fileManager.save(dataStorage); - fileManager.closeArena(); + if (isClosed.get()) { + return; } - dataStorage.clear(); + if (fileManager == null) { + isClosed.set(true); + return; + } + + performClose(); + fileManager.closeArena(); + isClosed.set(true); + } + + private void performBackgroundFlush() { + lock.writeLock().lock(); + try { + state = state.beforeFlushState(); + } finally { + lock.writeLock().unlock(); + } + state.setZeroDataStorageByteSize(); + + taskResults.add(service.submit(() -> { + state.fileManager.flush(state.flushingDataStorage.values()); + + lock.writeLock().lock(); + try { + state = state.afterFlushState(); + } finally { + lock.writeLock().unlock(); + } + })); + } + + private void performCompact() { + taskResults.add(service.submit(state.fileManager::performCompact)); + } + + private void performClose() { + for (Future taskResult : taskResults) { + if (taskResult != null && !taskResult.isDone()) { + try { + taskResult.get(); + } catch (ExecutionException e) { + throw new IllegalStateException("Current thread execution error occurred.", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Current thread was interrupted.", e); + } + } + } + + flush(); + service.close(); + taskResults.clear(); + } + + private static class State { + private final NavigableMap> dataStorage; + private final NavigableMap> flushingDataStorage; + private final FileManager fileManager; + private final AtomicLong dataStorageByteSize; + + private State(NavigableMap> dataStorage, + NavigableMap> flushingDataStorage, + FileManager fileManager) { + this.dataStorage = dataStorage; + this.fileManager = fileManager; + this.flushingDataStorage = flushingDataStorage; + this.dataStorageByteSize = new AtomicLong(); + } + + private static State emptyState(FileManager fileManager) { + return new State(new ConcurrentSkipListMap<>(memorySegmentComparator), + new ConcurrentSkipListMap<>(memorySegmentComparator), fileManager); + } + + private State beforeFlushState() { + return new State(new ConcurrentSkipListMap<>(memorySegmentComparator), dataStorage, fileManager); + } + + private State afterFlushState() { + return new State(dataStorage, new ConcurrentSkipListMap<>(memorySegmentComparator), fileManager); + } + + private long getDataStorageByteSize() { + return dataStorageByteSize.get(); + } + + private void updateDataStorageByteSize(long delta) { + dataStorageByteSize.getAndAdd(delta); + } + + private void setZeroDataStorageByteSize() { + dataStorageByteSize.set(0); + } + + private Iterator> getFlushingPairsIterator() { + return flushingDataStorage == null ? null : flushingDataStorage.values().iterator(); + } } } diff --git a/src/main/java/ru/vk/itmo/grunskiialexey/Compaction.java b/src/main/java/ru/vk/itmo/grunskiialexey/Compaction.java new file mode 100644 index 000000000..af177a37b --- /dev/null +++ b/src/main/java/ru/vk/itmo/grunskiialexey/Compaction.java @@ -0,0 +1,179 @@ +package ru.vk.itmo.grunskiialexey; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NavigableMap; +import java.util.NoSuchElementException; +import java.util.stream.Stream; + +import static ru.vk.itmo.grunskiialexey.DiskStorage.NAME_INDEX_FILE; +import static ru.vk.itmo.grunskiialexey.DiskStorage.NAME_TMP_INDEX_FILE; + +public class Compaction { + private final List segmentList; + + public Compaction(List segmentList) { + this.segmentList = segmentList; + } + + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, MemorySegment to + ) { + List>> iterators = new ArrayList<>(segmentList.size() + 1); + for (MemorySegment memorySegment : segmentList) { + iterators.add(iterator(memorySegment, from, to)); + } + iterators.add(firstIterator); + + return new MergeIterator<>( + iterators, + Comparator.comparing(Entry::key, MemorySegmentDao::compare), + entry -> entry.value() == null + ); + } + + public void compact( + Path storagePath, + NavigableMap> iterable + ) throws IOException { + if (segmentList.isEmpty() || (segmentList.size() == 1 && iterable.isEmpty())) { + return; + } + + final Path indexTmp = storagePath.resolve(NAME_TMP_INDEX_FILE); + final Path indexFile = storagePath.resolve(NAME_INDEX_FILE); + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + final String compactionFileName = DiskStorage.getNewFileName(existedFiles); + final Path compactionFile = storagePath.resolve(compactionFileName); + + long startValuesOffset = 0; + long maxOffset = 0; + for (Iterator> it = range(iterable.values().iterator(), null, null); it.hasNext(); ) { + Entry entry = it.next(); + startValuesOffset++; + maxOffset += entry.key().byteSize() + entry.value().byteSize(); + } + startValuesOffset *= 2 * Long.BYTES; + maxOffset += startValuesOffset; + + try ( + FileChannel fileChannel = FileChannel.open( + compactionFile, + StandardOpenOption.WRITE, StandardOpenOption.READ, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING + ); + Arena writeArena = Arena.ofConfined() + ) { + final MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, 0, maxOffset, writeArena + ); + + long dataOffset = startValuesOffset; + int indexOffset = 0; + for (Iterator> it = range(iterable.values().iterator(), null, null); it.hasNext(); ) { + Entry entry = it.next(); + + MemorySegment key = entry.key(); + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + MemorySegment.copy(key, 0, fileSegment, dataOffset, key.byteSize()); + dataOffset += key.byteSize(); + indexOffset += Long.BYTES; + + MemorySegment value = entry.value(); + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + MemorySegment.copy(value, 0, fileSegment, dataOffset, value.byteSize()); + dataOffset += value.byteSize(); + indexOffset += Long.BYTES; + } + } + + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + Files.write( + indexFile, + List.of(compactionFileName), + StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING + ); + Files.delete(indexTmp); + + // Delete old data + removeExcessFiles(storagePath, compactionFileName); + segmentList.clear(); + iterable.clear(); + } + + private static void removeExcessFiles(Path storagePath, String compactionFile) throws IOException { + List excessFiles = new ArrayList<>(); + try (Stream stream = Files.walk(storagePath, 1)) { + stream.forEach(path -> { + String fileName = path.getFileName().toString(); + if (Files.isRegularFile(path) + && !fileName.equals(compactionFile) + && !fileName.equals(NAME_INDEX_FILE) + ) { + excessFiles.add(path); + } + }); + } + + for (Path excessFile : excessFiles) { + Files.delete(excessFile); + } + } + + private Iterator> iterator(MemorySegment page, MemorySegment from, MemorySegment to) { + long recordIndexFrom = from == null ? 0 : DiskStorage.normalize(DiskStorage.indexOf(page, from)); + long recordIndexTo = to == null + ? DiskStorage.recordsCount(page) + : DiskStorage.normalize(DiskStorage.indexOf(page, to)); + long recordsCount = DiskStorage.recordsCount(page); + + return new Iterator<>() { + long index = recordIndexFrom; + + @Override + public boolean hasNext() { + return index < recordIndexTo; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + MemorySegment key = DiskStorage.slice( + page, + DiskStorage.startOfKey(page, index), + DiskStorage.endOfKey(page, index) + ); + long startOfValue = DiskStorage.startOfValue(page, index); + MemorySegment value = + startOfValue < 0 + ? null + : DiskStorage.slice( + page, + startOfValue, + DiskStorage.endOfValue(page, index, recordsCount) + ); + index++; + return new BaseEntry<>(key, value); + } + }; + } +} diff --git a/src/main/java/ru/vk/itmo/grunskiialexey/DiskStorage.java b/src/main/java/ru/vk/itmo/grunskiialexey/DiskStorage.java new file mode 100644 index 000000000..679fa3e24 --- /dev/null +++ b/src/main/java/ru/vk/itmo/grunskiialexey/DiskStorage.java @@ -0,0 +1,230 @@ +package ru.vk.itmo.grunskiialexey; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; + +public final class DiskStorage { + public static final String NAME_TMP_INDEX_FILE = "index.tmp"; + public static final String NAME_INDEX_FILE = "index.idx"; + + private DiskStorage() { + } + + public static void save(Path storagePath, Iterable> iterable) + throws IOException { + final Path indexTmp = storagePath.resolve(NAME_TMP_INDEX_FILE); + final Path indexFile = storagePath.resolve(NAME_INDEX_FILE); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + String newFileName = getNewFileName(existedFiles); + + long dataSize = 0; + long count = 0; + for (Entry entry : iterable) { + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + if (value != null) { + dataSize += value.byteSize(); + } + count++; + } + long indexSize = count * 2 * Long.BYTES; + + try ( + FileChannel fileChannel = FileChannel.open( + storagePath.resolve(newFileName), + StandardOpenOption.WRITE, StandardOpenOption.READ, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING + ); + Arena writeArena = Arena.ofConfined() + ) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, 0, indexSize + dataSize, writeArena + ); + + // index: + // |key0_Start|value0_Start|key1_Start|value1_Start|key2_Start|value2_Start|... + // key0_Start = data start = end of index + long dataOffset = indexSize; + int indexOffset = 0; + for (Entry entry : iterable) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += entry.key().byteSize(); + indexOffset += Long.BYTES; + + MemorySegment value = entry.value(); + if (value == null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, tombstone(dataOffset)); + } else { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += value.byteSize(); + } + indexOffset += Long.BYTES; + } + + // data: + // |key0|value0|key1|value1|... + dataOffset = indexSize; + for (Entry entry : iterable) { + MemorySegment key = entry.key(); + MemorySegment.copy(key, 0, fileSegment, dataOffset, key.byteSize()); + dataOffset += key.byteSize(); + + MemorySegment value = entry.value(); + if (value != null) { + MemorySegment.copy(value, 0, fileSegment, dataOffset, value.byteSize()); + dataOffset += value.byteSize(); + } + } + } + + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + + List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + list.add(newFileName); + Files.write( + indexFile, + list, + StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.delete(indexTmp); + } + + public static String getNewFileName(List existedFiles) { + int newFileNumber = existedFiles.isEmpty() ? 0 : Integer.parseInt(existedFiles.getLast()) + 1; + return Integer.toString(newFileNumber); + } + + public static List loadOrRecover(Path storagePath, Arena arena) throws IOException { + Path indexTmp = storagePath.resolve(NAME_TMP_INDEX_FILE); + Path indexFile = storagePath.resolve(NAME_INDEX_FILE); + + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + } + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + List result = new ArrayList<>(existedFiles.size()); + for (String fileName : existedFiles) { + Path file = storagePath.resolve(fileName); + try (FileChannel fileChannel = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(file), + arena + ); + result.add(fileSegment); + } + } + + return result; + } + + public static long indexOf(MemorySegment segment, MemorySegment key) { + long recordsCount = recordsCount(segment); + + long left = 0; + long right = recordsCount - 1; + while (left <= right) { + long mid = (left + right) >>> 1; + + long startOfKey = startOfKey(segment, mid); + long endOfKey = endOfKey(segment, mid); + long mismatch = MemorySegment.mismatch(segment, startOfKey, endOfKey, key, 0, key.byteSize()); + if (mismatch == -1) { + return mid; + } + + if (mismatch == key.byteSize()) { + right = mid - 1; + continue; + } + + if (mismatch == endOfKey - startOfKey) { + left = mid + 1; + continue; + } + + int b1 = Byte.toUnsignedInt(segment.get(ValueLayout.JAVA_BYTE, startOfKey + mismatch)); + int b2 = Byte.toUnsignedInt(key.get(ValueLayout.JAVA_BYTE, mismatch)); + if (b1 > b2) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + return tombstone(left); + } + + static long recordsCount(MemorySegment segment) { + long indexSize = indexSize(segment); + return indexSize / Long.BYTES / 2; + } + + private static long indexSize(MemorySegment segment) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + + static MemorySegment slice(MemorySegment page, long start, long end) { + return page.asSlice(start, end - start); + } + + static long startOfKey(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES); + } + + static long endOfKey(MemorySegment segment, long recordIndex) { + return normalizedStartOfValue(segment, recordIndex); + } + + private static long normalizedStartOfValue(MemorySegment segment, long recordIndex) { + return normalize(startOfValue(segment, recordIndex)); + } + + static long startOfValue(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES + Long.BYTES); + } + + static long endOfValue(MemorySegment segment, long recordIndex, long recordsCount) { + if (recordIndex < recordsCount - 1) { + return startOfKey(segment, recordIndex + 1); + } + return segment.byteSize(); + } + + private static long tombstone(long offset) { + return 1L << 63 | offset; + } + + static long normalize(long value) { + return value & ~(1L << 63); + } +} diff --git a/src/main/java/ru/vk/itmo/grunskiialexey/MemorySegmentDao.java b/src/main/java/ru/vk/itmo/grunskiialexey/MemorySegmentDao.java index f62e4bbfc..01dc1ed8c 100644 --- a/src/main/java/ru/vk/itmo/grunskiialexey/MemorySegmentDao.java +++ b/src/main/java/ru/vk/itmo/grunskiialexey/MemorySegmentDao.java @@ -1,6 +1,5 @@ package ru.vk.itmo.grunskiialexey; -import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; @@ -9,163 +8,113 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; +import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.NavigableMap; import java.util.concurrent.ConcurrentSkipListMap; public class MemorySegmentDao implements Dao> { - private final Comparator comparator = (o1, o2) -> { - long firstMismatch = o1.mismatch(o2); - if (firstMismatch == -1) { - return 0; - } - if (firstMismatch == o1.byteSize()) { - return -1; - } - if (firstMismatch == o2.byteSize()) { - return 1; - } - - byte byte1 = o1.get(ValueLayout.JAVA_BYTE, firstMismatch); - byte byte2 = o2.get(ValueLayout.JAVA_BYTE, firstMismatch); - return Byte.compare(byte1, byte2); - }; - - private final NavigableMap> data = new ConcurrentSkipListMap<>(comparator); - private final Path filePath; + private final Comparator comparator = MemorySegmentDao::compare; + private final NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); private final Arena arena; - private final MemorySegment page; + private final Compaction diskStorage; + private final Path path; public MemorySegmentDao(Config config) throws IOException { - this.filePath = Paths.get(config.basePath().toString(), "file.db"); + this.path = config.basePath().resolve("data"); + Files.createDirectories(path); + arena = Arena.ofShared(); - long size; - try { - size = Files.size(filePath); - } catch (NoSuchFileException e) { - page = MemorySegment.NULL; - return; + this.diskStorage = new Compaction(DiskStorage.loadOrRecover(path, arena)); + } + + static int compare(MemorySegment memorySegment1, MemorySegment memorySegment2) { + long mismatch = memorySegment1.mismatch(memorySegment2); + if (mismatch == -1) { + return 0; + } + + if (mismatch == memorySegment1.byteSize()) { + return -1; } - MemorySegment currentPage = null; - try (FileChannel channel = FileChannel.open(filePath, StandardOpenOption.READ)) { - currentPage = channel.map(FileChannel.MapMode.READ_ONLY, 0, size, arena); - } catch (IOException e) { - arena.close(); - } finally { - page = currentPage; + if (mismatch == memorySegment2.byteSize()) { + return 1; } + byte b1 = memorySegment1.get(ValueLayout.JAVA_BYTE, mismatch); + byte b2 = memorySegment2.get(ValueLayout.JAVA_BYTE, mismatch); + return Byte.compare(b1, b2); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { + return diskStorage.range(getInMemory(from, to), from, to); + } + + private Iterator> getInMemory(MemorySegment from, MemorySegment to) { if (from == null && to == null) { - return data.values().iterator(); - } else if (from == null) { - return data.headMap(to).values().iterator(); - } else if (to == null) { - return data.tailMap(from).values().iterator(); - } else { - return data.subMap(from, to).values().iterator(); + return storage.values().iterator(); + } + if (from == null) { + return storage.headMap(to).values().iterator(); } + if (to == null) { + return storage.tailMap(from).values().iterator(); + } + return storage.subMap(from, to).values().iterator(); + } + + @Override + public void upsert(Entry entry) { + storage.put(entry.key(), entry); } @Override public Entry get(MemorySegment key) { - if (data.containsKey(key)) { - return data.get(key); + Entry entry = storage.get(key); + if (entry != null) { + if (entry.value() == null) { + return null; + } + return entry; } - if (page.equals(MemorySegment.NULL)) { + + Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + + if (!iterator.hasNext()) { return null; } - - long offset = 0; - while (offset < page.byteSize()) { - int keyLength = page.get(ValueLayout.JAVA_INT, offset); - offset += 4; - MemorySegment resultKey = MemorySegment.ofArray(new byte[keyLength]); - MemorySegment.copy(page, ValueLayout.JAVA_BYTE, offset, resultKey, ValueLayout.JAVA_BYTE, 0, keyLength); - offset += correctAlignedSize(keyLength); - - int valueLength = page.get(ValueLayout.JAVA_INT, offset); - offset += 4; - MemorySegment resultValue = MemorySegment.ofArray(new byte[valueLength]); - MemorySegment.copy(page, ValueLayout.JAVA_BYTE, offset, resultValue, ValueLayout.JAVA_BYTE, 0, valueLength); - offset += correctAlignedSize(valueLength); - - if (resultKey.mismatch(key) == -1) { - return new BaseEntry<>( - resultKey, - resultValue - ); - } + Entry next = iterator.next(); + if (compare(next.key(), key) == 0) { + return next; } return null; } @Override - public void upsert(Entry entry) { - if (entry.value() == null) { - data.remove(entry.key()); - } else { - data.put(entry.key(), entry); + public void flush() throws IOException { + if (!storage.isEmpty()) { + DiskStorage.save(path, storage.values()); } } + @Override + public void compact() throws IOException { + diskStorage.compact(path, storage); + } + @Override public void close() throws IOException { if (!arena.scope().isAlive()) { return; } - arena.close(); - - try ( - FileChannel channel = FileChannel.open( - filePath, - StandardOpenOption.CREATE, - StandardOpenOption.READ, - StandardOpenOption.WRITE - ); - Arena writeArena = Arena.ofConfined() - ) { - long allSize = data.values().stream().mapToLong(entry -> - correctAlignedSize(entry.key().byteSize()) + correctAlignedSize(entry.value().byteSize()) + 2 * 4 - ).sum(); - MemorySegment writePage = channel.map(FileChannel.MapMode.READ_WRITE, 0, allSize, writeArena); - - long offset = 0; - for (Entry entry : data.values()) { - long keyLength = entry.key().byteSize(); - writePage.set(ValueLayout.JAVA_INT, offset, (int) keyLength); - offset += 4; - MemorySegment.copy( - entry.key(), ValueLayout.JAVA_BYTE, 0, - writePage, ValueLayout.JAVA_BYTE, offset, keyLength - ); - offset += correctAlignedSize(keyLength); - - long valueLength = entry.value().byteSize(); - writePage.set(ValueLayout.JAVA_INT, offset, (int) valueLength); - offset += 4; - MemorySegment.copy( - entry.value(), ValueLayout.JAVA_BYTE, 0, - writePage, ValueLayout.JAVA_BYTE, offset, valueLength - ); - offset += correctAlignedSize(valueLength); - } - } - } + flush(); - private long correctAlignedSize(long offset) { - return offset % 4 == 0 ? offset : offset + 4 - (offset % 4); + arena.close(); } } diff --git a/src/main/java/ru/vk/itmo/grunskiialexey/MergeIterator.java b/src/main/java/ru/vk/itmo/grunskiialexey/MergeIterator.java new file mode 100644 index 000000000..7a822fa4f --- /dev/null +++ b/src/main/java/ru/vk/itmo/grunskiialexey/MergeIterator.java @@ -0,0 +1,148 @@ +package ru.vk.itmo.grunskiialexey; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.function.Function; + +public class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + private final Comparator comparator; + private final Function isSkipT; + private PeekIterator peek; + + private static class PeekIterator implements Iterator { + + public final int id; + private final Iterator delegate; + private T peek; + + private PeekIterator(int id, Iterator delegate) { + this.id = id; + this.delegate = delegate; + } + + @Override + public boolean hasNext() { + if (peek == null) { + return delegate.hasNext(); + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T element = peek(); + this.peek = null; + return element; + } + + private T peek() { + if (peek == null) { + if (!delegate.hasNext()) { + return null; + } + peek = delegate.next(); + } + return peek; + } + } + + public MergeIterator(Collection> iterators, Comparator comparator, Function isSkipT) { + this.comparator = comparator; + this.isSkipT = isSkipT; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + private PeekIterator peek() { + // while exists take + while (peek == null) { + // Getting peek for one file or in-memory + peek = priorityQueue.poll(); + if (peek == null) { + return null; + } + + while (true) { + // getting first thing + PeekIterator next = priorityQueue.peek(); + if (next == null) { + break; + } + + int compare = comparator.compare(peek.peek(), next.peek()); + if (compare == 0) { + skipEqualsElement(); + } else { + break; + } + } + + if (peek.peek() == null) { + peek = null; + continue; + } + + peek = skipRemovedEntry(peek); + } + + return peek; + } + + private void skipEqualsElement() { + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + } + } + + private PeekIterator skipRemovedEntry(PeekIterator peek) { + if (isSkipT.apply(peek.peek())) { + peek.next(); + if (peek.hasNext()) { + priorityQueue.add(peek); + } + return null; + } + return peek; + } + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator element = peek(); + if (element == null) { + throw new NoSuchElementException(); + } + T next = element.next(); + this.peek = null; + if (element.hasNext()) { + priorityQueue.add(element); + } + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/AbstractBasedOnSSTableDao.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/AbstractBasedOnSSTableDao.java index 571eba8be..ff8a8cf1e 100644 --- a/src/main/java/ru/vk/itmo/kovalchukvladislav/AbstractBasedOnSSTableDao.java +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/AbstractBasedOnSSTableDao.java @@ -1,189 +1,148 @@ package ru.vk.itmo.kovalchukvladislav; import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; import ru.vk.itmo.kovalchukvladislav.model.DaoIterator; import ru.vk.itmo.kovalchukvladislav.model.EntryExtractor; +import ru.vk.itmo.kovalchukvladislav.model.SimpleDaoLoggerUtility; +import ru.vk.itmo.kovalchukvladislav.storage.InMemoryStorage; +import ru.vk.itmo.kovalchukvladislav.storage.InMemoryStorageImpl; +import ru.vk.itmo.kovalchukvladislav.storage.SSTableStorage; +import ru.vk.itmo.kovalchukvladislav.storage.SSTableStorageImpl; import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.OpenOption; +import java.io.UncheckedIOException; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Objects; - -public abstract class AbstractBasedOnSSTableDao> extends AbstractInMemoryDao { - // =================================== - // Constants - // =================================== - private static final ValueLayout.OfLong LONG_LAYOUT = ValueLayout.JAVA_LONG_UNALIGNED; - private static final String OFFSETS_FILENAME_PREFIX = "offsets_"; - private static final String METADATA_FILENAME = "metadata"; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Logger; + +public abstract class AbstractBasedOnSSTableDao> implements Dao { + private final Logger logger = SimpleDaoLoggerUtility.createLogger(getClass()); private static final String DB_FILENAME_PREFIX = "db_"; - - // =================================== - // Variables - // =================================== + private static final String METADATA_FILENAME = "metadata"; + private static final String OFFSETS_FILENAME_PREFIX = "offsets_"; private final Path basePath; - private final Path metadataPath; - private final Arena arena = Arena.ofShared(); + private final long flushThresholdBytes; private final EntryExtractor extractor; - - // =================================== - // Storages - // =================================== - - private final int storagesCount; - private final List dbMappedSegments; - private final List offsetMappedSegments; + private final AtomicBoolean isClosed = new AtomicBoolean(false); + private final AtomicBoolean isFlushingOrCompacting = new AtomicBoolean(false); + private final ExecutorService flushOrCompactQueue = Executors.newSingleThreadExecutor(); + + /** + * В get(), upsert() и compact() для inMemoryStorage и ssTableStorage не требуется синхронизация между собой. + * Исключение составляет только flush() и compact(). + * Следует проследить что на любом этапе оба стораджа в сумме будут иметь полные данные. + */ + private final InMemoryStorage inMemoryStorage; + private final SSTableStorage ssTableStorage; protected AbstractBasedOnSSTableDao(Config config, EntryExtractor extractor) throws IOException { - super(extractor); this.extractor = extractor; + this.flushThresholdBytes = config.flushThresholdBytes(); this.basePath = Objects.requireNonNull(config.basePath()); - - if (!Files.exists(basePath)) { - Files.createDirectory(basePath); - } - this.metadataPath = basePath.resolve(METADATA_FILENAME); - - this.storagesCount = getCountFromMetadataOrCreate(); - this.dbMappedSegments = new ArrayList<>(storagesCount); - this.offsetMappedSegments = new ArrayList<>(storagesCount); - - for (int i = 0; i < storagesCount; i++) { - readFileAndMapToSegment(DB_FILENAME_PREFIX, i, dbMappedSegments); - readFileAndMapToSegment(OFFSETS_FILENAME_PREFIX, i, offsetMappedSegments); - } - } - - // =================================== - // Restoring state - // =================================== - private int getCountFromMetadataOrCreate() throws IOException { - if (!Files.exists(metadataPath)) { - Files.writeString(metadataPath, "0", StandardOpenOption.WRITE, StandardOpenOption.CREATE); - return 0; - } - return Integer.parseInt(Files.readString(metadataPath)); - } - - private void readFileAndMapToSegment(String filenamePrefix, int index, - List segments) throws IOException { - Path path = basePath.resolve(filenamePrefix + index); - try (FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) { - - MemorySegment segment = channel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(path), arena); - segments.add(segment); - } + this.inMemoryStorage = new InMemoryStorageImpl<>(extractor, config.flushThresholdBytes()); + this.ssTableStorage = new SSTableStorageImpl<>(basePath, METADATA_FILENAME, + DB_FILENAME_PREFIX, OFFSETS_FILENAME_PREFIX, extractor); } - // =================================== - // Finding in storage - // =================================== @Override public Iterator get(D from, D to) { - Iterator inMemotyIterator = super.get(from, to); - return new DaoIterator<>(from, to, inMemotyIterator, dbMappedSegments, offsetMappedSegments, extractor); + List> iterators = new ArrayList<>(); + iterators.addAll(inMemoryStorage.getIterators(from, to)); + iterators.addAll(ssTableStorage.getIterators(from, to)); + return new DaoIterator<>(iterators, extractor); } @Override public E get(D key) { - E e = dao.get(key); + E e = inMemoryStorage.get(key); if (e != null) { return e.value() == null ? null : e; } - E fromFile = findInStorages(key); + E fromFile = ssTableStorage.get(key); return (fromFile == null || fromFile.value() == null) ? null : fromFile; } - private E findInStorages(D key) { - for (int i = storagesCount - 1; i >= 0; i--) { - MemorySegment storage = dbMappedSegments.get(i); - MemorySegment offsets = offsetMappedSegments.get(i); - - long offset = extractor.findLowerBoundValueOffset(key, storage, offsets); - if (offset == -1) { - continue; - } - D lowerBoundKey = extractor.readValue(storage, offset); - - if (comparator.compare(lowerBoundKey, key) == 0) { - long valueOffset = offset + extractor.size(lowerBoundKey); - D value = extractor.readValue(storage, valueOffset); - return extractor.createEntry(lowerBoundKey, value); - } + @Override + public void upsert(E entry) { + long size = inMemoryStorage.upsertAndGetSize(entry); + if (size >= flushThresholdBytes) { + flush(); } - return null; } - // =================================== - // Writing data - // =================================== - private void writeData() throws IOException { - Path dbPath = basePath.resolve(DB_FILENAME_PREFIX + storagesCount); - Path offsetsPath = basePath.resolve(OFFSETS_FILENAME_PREFIX + storagesCount); - - OpenOption[] options = new OpenOption[] { - StandardOpenOption.READ, - StandardOpenOption.WRITE, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.CREATE - }; - - try (FileChannel db = FileChannel.open(dbPath, options); - FileChannel offsets = FileChannel.open(offsetsPath, options); - Arena confinedArena = Arena.ofConfined()) { - - long dbSize = getDAOBytesSize(); - long offsetsSize = (long) dao.size() * Long.BYTES; - MemorySegment fileSegment = db.map(FileChannel.MapMode.READ_WRITE, 0, dbSize, confinedArena); - MemorySegment offsetsSegment = offsets.map(FileChannel.MapMode.READ_WRITE, 0, offsetsSize, confinedArena); - - int i = 0; - long offset = 0; - for (E entry : dao.values()) { - offsetsSegment.setAtIndex(LONG_LAYOUT, i, offset); - i += 1; - offset = extractor.writeEntry(entry, fileSegment, offset); - } - fileSegment.load(); - offsetsSegment.load(); + @Override + public void flush() { + if (!isFlushingOrCompacting.compareAndSet(false, true)) { + logger.info("Flush or compact already in process"); + return; + } + Callable flushCallable = inMemoryStorage.prepareFlush( + basePath, + DB_FILENAME_PREFIX, + OFFSETS_FILENAME_PREFIX); + if (flushCallable == null) { + isFlushingOrCompacting.set(false); + return; } + submitFlushAndAddSSTable(flushCallable); } - private long getDAOBytesSize() { - long size = 0; - for (E entry : dao.values()) { - size += extractor.size(entry); - } - return size; + private void submitFlushAndAddSSTable(Callable flushCallable) { + flushOrCompactQueue.execute(() -> { + try { + String newTimestamp = flushCallable.call(); + ssTableStorage.addSSTableId(newTimestamp, true); + inMemoryStorage.completeFlush(); + } catch (Exception e) { + inMemoryStorage.failFlush(); + } finally { + isFlushingOrCompacting.set(false); + } + }); } - // =================================== - // Flush and close - // =================================== @Override - public synchronized void flush() throws IOException { - if (!dao.isEmpty()) { - writeData(); - Files.writeString(metadataPath, String.valueOf(storagesCount + 1)); + public void close() { + if (!isClosed.compareAndSet(false, true)) { + return; } + + flushOrCompactQueue.close(); + try { + String newTimestamp = inMemoryStorage.close(basePath, DB_FILENAME_PREFIX, OFFSETS_FILENAME_PREFIX); + if (newTimestamp != null) { + ssTableStorage.addSSTableId(newTimestamp, false); + } + } catch (Exception e) { + logger.severe(() -> "Error while flushing on close: " + e.getMessage()); + } + ssTableStorage.close(); } @Override - public synchronized void close() throws IOException { - if (arena.scope().isAlive()) { - arena.close(); + public void compact() { + if (!isFlushingOrCompacting.compareAndSet(false, true)) { + logger.info("Flush or compact already in process"); + return; } - flush(); + flushOrCompactQueue.execute(() -> { + try { + ssTableStorage.compact(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + isFlushingOrCompacting.set(false); + } + }); } } diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/AbstractInMemoryDao.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/AbstractInMemoryDao.java deleted file mode 100644 index 3f48b9981..000000000 --- a/src/main/java/ru/vk/itmo/kovalchukvladislav/AbstractInMemoryDao.java +++ /dev/null @@ -1,44 +0,0 @@ -package ru.vk.itmo.kovalchukvladislav; - -import ru.vk.itmo.Dao; -import ru.vk.itmo.Entry; - -import java.util.Comparator; -import java.util.Iterator; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; - -public abstract class AbstractInMemoryDao> implements Dao { - protected final ConcurrentNavigableMap dao; - protected final Comparator comparator; - - protected AbstractInMemoryDao(Comparator comparator) { - this.dao = new ConcurrentSkipListMap<>(comparator); - this.comparator = comparator; - } - - @Override - public Iterator get(D from, D to) { - ConcurrentNavigableMap subMap; - if (from == null && to == null) { - subMap = dao; - } else if (from == null) { - subMap = dao.headMap(to); - } else if (to == null) { - subMap = dao.tailMap(from); - } else { - subMap = dao.subMap(from, to); - } - return subMap.values().iterator(); - } - - @Override - public E get(D key) { - return dao.get(key); - } - - @Override - public void upsert(E entry) { - dao.put(entry.key(), entry); - } -} diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/MemorySegmentEntryExtractor.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/MemorySegmentEntryExtractor.java index f55e75fdb..4e25d24fa 100644 --- a/src/main/java/ru/vk/itmo/kovalchukvladislav/MemorySegmentEntryExtractor.java +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/MemorySegmentEntryExtractor.java @@ -80,6 +80,9 @@ public long size(MemorySegment value) { @Override public long size(Entry entry) { + if (entry == null) { + return 0; + } return size(entry.key()) + size(entry.value()); } diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/model/DaoIterator.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/model/DaoIterator.java index 47572a8ac..37ed6d2c4 100644 --- a/src/main/java/ru/vk/itmo/kovalchukvladislav/model/DaoIterator.java +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/model/DaoIterator.java @@ -2,50 +2,28 @@ import ru.vk.itmo.Entry; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.PriorityQueue; public class DaoIterator> implements Iterator { - private static final ValueLayout.OfLong LONG_LAYOUT = ValueLayout.JAVA_LONG_UNALIGNED; - private static final Integer IN_MEMORY_ITERATOR_ID = Integer.MAX_VALUE; - private final Iterator inMemoryIterator; + private final List> iterators; private final EntryExtractor extractor; private final PriorityQueue queue; - private final List storageIterators; - public DaoIterator(D from, D to, - Iterator inMemoryIterator, - List storageSegments, - List offsetsSegments, - EntryExtractor extractor) { + public DaoIterator(List> iteratorsSortedByPriority, EntryExtractor extractor) { + this.iterators = iteratorsSortedByPriority; this.extractor = extractor; - this.inMemoryIterator = inMemoryIterator; - this.storageIterators = getStorageIterators(from, to, storageSegments, offsetsSegments); - this.queue = new PriorityQueue<>(1 + storageIterators.size()); - addEntryByIteratorIdSafe(IN_MEMORY_ITERATOR_ID); - for (int i = 0; i < storageIterators.size(); i++) { - addEntryByIteratorIdSafe(i); + int size = iterators.size(); + this.queue = new PriorityQueue<>(size); + for (int i = 0; i < size; i++) { + addEntry(iterators.get(i), i); } cleanByNull(); } - private List getStorageIterators(D from, D to, - List storageSegments, - List offsetsSegments) { - int storagesCount = storageSegments.size(); - final List iterators = new ArrayList<>(storagesCount); - for (int i = 0; i < storagesCount; i++) { - iterators.add(new StorageIterator(storageSegments.get(i), offsetsSegments.get(i), from, to)); - } - return iterators; - } - @Override public boolean hasNext() { return !queue.isEmpty(); @@ -82,18 +60,18 @@ private void cleanByNull() { } private void addEntryByIteratorIdSafe(int iteratorId) { - Iterator iteratorById = getIteratorById(iteratorId); - if (iteratorById.hasNext()) { - E next = iteratorById.next(); - queue.add(new IndexedEntry(iteratorId, next)); + addEntry(getIteratorById(iteratorId), iteratorId); + } + + private void addEntry(Iterator iterator, int id) { + if (iterator.hasNext()) { + E next = iterator.next(); + queue.add(new IndexedEntry(id, next)); } } private Iterator getIteratorById(int id) { - if (id == IN_MEMORY_ITERATOR_ID) { - return inMemoryIterator; - } - return storageIterators.get(id); + return iterators.get(id); } private class IndexedEntry implements Comparable { @@ -111,83 +89,7 @@ public int compareTo(IndexedEntry other) { if (compared != 0) { return compared; } - return -Integer.compare(iteratorId, other.iteratorId); - } - } - - private class StorageIterator implements Iterator { - private final MemorySegment storageSegment; - private final long end; - private long start; - - public StorageIterator(MemorySegment storageSegment, MemorySegment offsetsSegment, D from, D to) { - this.storageSegment = storageSegment; - - if (offsetsSegment.byteSize() == 0) { - this.start = -1; - this.end = -1; - } else { - this.start = calculateStartPosition(offsetsSegment, from); - this.end = calculateEndPosition(offsetsSegment, to); - } - } - - private long calculateStartPosition(MemorySegment offsetsSegment, D from) { - if (from == null) { - return getFirstOffset(offsetsSegment); - } - long lowerBoundOffset = extractor.findLowerBoundValueOffset(from, storageSegment, offsetsSegment); - if (lowerBoundOffset == -1) { - // the smallest element and doesn't exist - return getFirstOffset(offsetsSegment); - } else { - // storage[lowerBoundOffset] <= from, we need >= only - return moveOffsetIfFirstKeyAreNotEqual(from, lowerBoundOffset); - } - } - - private long calculateEndPosition(MemorySegment offsetsSegment, D to) { - if (to == null) { - return getEndOffset(); - } - long lowerBoundOffset = extractor.findLowerBoundValueOffset(to, storageSegment, offsetsSegment); - if (lowerBoundOffset == -1) { - // the smallest element and doesn't exist - return getFirstOffset(offsetsSegment); - } - // storage[lowerBoundOffset] <= to, we need >= only - return moveOffsetIfFirstKeyAreNotEqual(to, lowerBoundOffset); - } - - private long getFirstOffset(MemorySegment offsetsSegment) { - return offsetsSegment.getAtIndex(LONG_LAYOUT, 0); - } - - private long getEndOffset() { - return storageSegment.byteSize(); - } - - private long moveOffsetIfFirstKeyAreNotEqual(D from, long lowerBoundOffset) { - long offset = lowerBoundOffset; - D lowerBoundKey = extractor.readValue(storageSegment, offset); - if (extractor.compare(lowerBoundKey, from) != 0) { - offset += extractor.size(lowerBoundKey); - D lowerBoundValue = extractor.readValue(storageSegment, offset); - offset += extractor.size(lowerBoundValue); - } - return offset; - } - - @Override - public boolean hasNext() { - return start < end; - } - - @Override - public E next() { - E entry = extractor.readEntry(storageSegment, start); - start += extractor.size(entry); - return entry; + return Integer.compare(iteratorId, other.iteratorId); } } } diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/model/MemoryOverflowException.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/model/MemoryOverflowException.java new file mode 100644 index 000000000..84b798886 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/model/MemoryOverflowException.java @@ -0,0 +1,7 @@ +package ru.vk.itmo.kovalchukvladislav.model; + +public class MemoryOverflowException extends RuntimeException { + public MemoryOverflowException(String message) { + super(message); + } +} diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/model/SimpleDaoLoggerUtility.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/model/SimpleDaoLoggerUtility.java new file mode 100644 index 000000000..7457d5091 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/model/SimpleDaoLoggerUtility.java @@ -0,0 +1,16 @@ +package ru.vk.itmo.kovalchukvladislav.model; + +import java.util.logging.Level; +import java.util.logging.Logger; + +// Логгер, который я включаю локально, но выключаю перед пушем, чтобы он не засорял гитхаб. +public final class SimpleDaoLoggerUtility { + private SimpleDaoLoggerUtility() { + } + + public static Logger createLogger(Class clazz) { + Logger logger = Logger.getLogger(clazz.getSimpleName()); + logger.setLevel(Level.OFF); + return logger; + } +} diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/model/StorageIterator.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/model/StorageIterator.java new file mode 100644 index 000000000..4454f1204 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/model/StorageIterator.java @@ -0,0 +1,91 @@ +package ru.vk.itmo.kovalchukvladislav.model; + +import ru.vk.itmo.Entry; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class StorageIterator> implements Iterator { + private final EntryExtractor extractor; + private final MemorySegment storageSegment; + private final long end; + private long start; + + public StorageIterator(D from, D to, + MemorySegment storageSegment, + MemorySegment offsetsSegment, + EntryExtractor extractor) { + this.storageSegment = storageSegment; + this.extractor = extractor; + + if (offsetsSegment.byteSize() == 0) { + this.start = -1; + this.end = -1; + } else { + this.start = calculateStartPosition(offsetsSegment, from); + this.end = calculateEndPosition(offsetsSegment, to); + } + } + + private long calculateStartPosition(MemorySegment offsetsSegment, D from) { + if (from == null) { + return getFirstOffset(offsetsSegment); + } + long lowerBoundOffset = extractor.findLowerBoundValueOffset(from, storageSegment, offsetsSegment); + if (lowerBoundOffset == -1) { + // the smallest element and doesn't exist + return getFirstOffset(offsetsSegment); + } else { + // storage[lowerBoundOffset] <= from, we need >= only + return moveOffsetIfFirstKeyAreNotEqual(from, lowerBoundOffset); + } + } + + private long calculateEndPosition(MemorySegment offsetsSegment, D to) { + if (to == null) { + return getEndOffset(); + } + long lowerBoundOffset = extractor.findLowerBoundValueOffset(to, storageSegment, offsetsSegment); + if (lowerBoundOffset == -1) { + // the smallest element and doesn't exist + return getFirstOffset(offsetsSegment); + } + // storage[lowerBoundOffset] <= to, we need >= only + return moveOffsetIfFirstKeyAreNotEqual(to, lowerBoundOffset); + } + + private long getFirstOffset(MemorySegment offsetsSegment) { + return offsetsSegment.getAtIndex(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + + private long getEndOffset() { + return storageSegment.byteSize(); + } + + private long moveOffsetIfFirstKeyAreNotEqual(D from, long lowerBoundOffset) { + long offset = lowerBoundOffset; + D lowerBoundKey = extractor.readValue(storageSegment, offset); + if (extractor.compare(lowerBoundKey, from) != 0) { + offset += extractor.size(lowerBoundKey); + D lowerBoundValue = extractor.readValue(storageSegment, offset); + offset += extractor.size(lowerBoundValue); + } + return offset; + } + + @Override + public boolean hasNext() { + return start < end; + } + + @Override + public E next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + E entry = extractor.readEntry(storageSegment, start); + start += extractor.size(entry); + return entry; + } +} diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/model/TableInfo.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/model/TableInfo.java new file mode 100644 index 000000000..38620f3b7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/model/TableInfo.java @@ -0,0 +1,4 @@ +package ru.vk.itmo.kovalchukvladislav.model; + +public record TableInfo(long recordsCount, long recordsSize) { +} diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/InMemoryStorage.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/InMemoryStorage.java new file mode 100644 index 000000000..99a7384a5 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/InMemoryStorage.java @@ -0,0 +1,43 @@ +package ru.vk.itmo.kovalchukvladislav.storage; + +import ru.vk.itmo.Entry; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.Callable; + +public interface InMemoryStorage> { + E get(D key); + + /** + * Вставляет значение и возвращает размер стораджа, по которому можно судить нужно ли планировать flush. + * Бросает MemoryOverflowException если достигнут порог по размеру, а предыдущий flush() еще выполняется или упал. + */ + long upsertAndGetSize(E entry); + + List> getIterators(D from, D to); + + /** + * Возвращает таску для flush(). Таска возвращает новый timestamp файлов в base path. + * Если уже выполняется flush, возвращает null. + * Если предыдущий flush() был завершен с ошибкой, возвратит таску на повторную попытку. + * Если нет данных, возвращает null. + */ + Callable prepareFlush(Path basePath, String dbFilenamePrefix, String offsetsFilenamePrefix); + + /** + * Помечает flush() завершенным с ошибкой. Позволяет повторить попытку при повторных flush(). + */ + void failFlush(); + + /** + * Завершает flush() и удаляет выгружаемое дао, которое хранится пока SSTableStorage не подхватит данные с диска. + */ + void completeFlush(); + + /** + * Синхронно и однопоточно делает flush().Возвращает новый timestamp или null. + */ + String close(Path basePath, String dbFilenamePrefix, String offsetsFilenamePrefix) throws IOException; +} diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/InMemoryStorageImpl.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/InMemoryStorageImpl.java new file mode 100644 index 000000000..54b3eb541 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/InMemoryStorageImpl.java @@ -0,0 +1,293 @@ +package ru.vk.itmo.kovalchukvladislav.storage; + +import ru.vk.itmo.Entry; +import ru.vk.itmo.kovalchukvladislav.model.EntryExtractor; +import ru.vk.itmo.kovalchukvladislav.model.MemoryOverflowException; +import ru.vk.itmo.kovalchukvladislav.model.SimpleDaoLoggerUtility; +import ru.vk.itmo.kovalchukvladislav.model.TableInfo; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.logging.Logger; + +public class InMemoryStorageImpl> implements InMemoryStorage { + private static final StandardCopyOption[] MOVE_OPTIONS = new StandardCopyOption[]{ + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + }; + private static final Logger logger = SimpleDaoLoggerUtility.createLogger(InMemoryStorageImpl.class); + private final long flushThresholdBytes; + private final EntryExtractor extractor; + + private volatile DaoState daoState; + private volatile FlushingDaoState flushingDaoState; + private final ReadWriteLock stateLock = new ReentrantReadWriteLock(); + + public InMemoryStorageImpl(EntryExtractor extractor, long flushThresholdBytes) { + this.extractor = extractor; + this.flushThresholdBytes = flushThresholdBytes; + this.daoState = createEmptyDaoState(); + this.flushingDaoState = createEmptyFlushingDaoState(); + + } + + private DaoState createEmptyDaoState() { + return new DaoState<>(new ConcurrentSkipListMap<>(extractor), new AtomicLong(0)); + } + + private FlushingDaoState createEmptyFlushingDaoState() { + return new FlushingDaoState<>(new ConcurrentSkipListMap<>(extractor), 0, FlushingState.NOT_RUNNING); + } + + /** + * daoSize допустимо увеличивать внутри readLock для большей производительности. + * Например, в upsert надо атомарно увеличить или уменьшить значение, а не узнать корректное. + * В операциях на чтение точного значения daoSize (flush) следует использовать writeLock. + * writeLock() гарантирует что в настоящее время нет readLock'ов, а значит и незаконченных операций изменения size. + */ + @SuppressWarnings("unused") + private record DaoState>(ConcurrentNavigableMap dao, AtomicLong daoSize) { + public FlushingDaoState toFlushingRunningDaoState() { + return new FlushingDaoState<>(dao, daoSize.get(), FlushingState.RUNNING); + } + } + + @SuppressWarnings("unused") + private record FlushingDaoState(ConcurrentNavigableMap dao, long daoSize, FlushingState flushingState) { + + public FlushingDaoState toFailed() { + return new FlushingDaoState<>(dao, daoSize, FlushingState.FAILED); + } + + public FlushingDaoState failedToTryAgain() { + if (flushingState != FlushingState.FAILED) { + throw new IllegalStateException("This method should be called when state is failed"); + } + return new FlushingDaoState<>(dao, daoSize, FlushingState.RUNNING); + } + } + + private enum FlushingState { + NOT_RUNNING, + RUNNING, + FAILED, + // Можно добавить еще одно состояние: данные выгружены, но произошло исключение при их релоаде в SSTableStorage. + // Позволит при повторном flush вернуть уже готовый timestamp, а не флашить опять в новый файл. + } + + @Override + public E get(D key) { + ConcurrentNavigableMap dao; + ConcurrentNavigableMap flushingDao; + + stateLock.readLock().lock(); + try { + dao = daoState.dao; + flushingDao = flushingDaoState.dao; + } finally { + stateLock.readLock().unlock(); + } + + E entry = dao.get(key); + if (entry == null && !flushingDao.isEmpty()) { + entry = flushingDao.get(key); + } + return entry; + } + + /** + * Возвращает не точное значение size в угоду перфомансу, иначе будет куча writeLock. + * При параллельных upsert(), в одном из них daoSize может не успеть инкрементироваться для вставляемого entry. + * Тем не менее "приблизительное size" можно использовать для детекта случаев когда надо делать flush(). + */ + @Override + public long upsertAndGetSize(E entry) { + stateLock.readLock().lock(); + try { + // "приблизительный" size, не пользуемся write lock'ами в угоду перфомансу + AtomicLong daoSize = getDaoSizeOrThrowMemoryOverflow(flushThresholdBytes, daoState, flushingDaoState); + + E oldEntry = daoState.dao.put(entry.key(), entry); + long delta = extractor.size(entry) - extractor.size(oldEntry); + return daoSize.addAndGet(delta); + } finally { + stateLock.readLock().unlock(); + } + } + + private static > AtomicLong getDaoSizeOrThrowMemoryOverflow( + long flushThresholdBytes, DaoState daoState, FlushingDaoState flushingDaoState) { + AtomicLong daoSize = daoState.daoSize; + if (daoSize.get() < flushThresholdBytes) { + return daoSize; + } + FlushingState flushingState = flushingDaoState.flushingState(); + if (flushingState == FlushingState.RUNNING) { + throw new MemoryOverflowException("There no free space." + + "daoSize is max, previous flush running and not completed"); + } else if (flushingState == FlushingState.FAILED) { + throw new MemoryOverflowException("There no free space." + + "daoSize is max, previous flush was failed. Try to repeat flush"); + } + return daoSize; + } + + @Override + public List> getIterators(D from, D to) { + ConcurrentNavigableMap dao; + ConcurrentNavigableMap flushingDao; + + stateLock.readLock().lock(); + try { + dao = daoState.dao; + flushingDao = flushingDaoState.dao; + } finally { + stateLock.readLock().unlock(); + } + + List> result = new ArrayList<>(2); + result.add(getIteratorDao(dao, from, to)); + result.add(getIteratorDao(flushingDao, from, to)); + return result; + } + + private Iterator getIteratorDao(ConcurrentNavigableMap dao, D from, D to) { + ConcurrentNavigableMap subMap; + if (from == null && to == null) { + subMap = dao; + } else if (from == null) { + subMap = dao.headMap(to); + } else if (to == null) { + subMap = dao.tailMap(from); + } else { + subMap = dao.subMap(from, to); + } + return subMap.values().iterator(); + } + + /** + * Возвращает таску для flush(). Таска возвращает новый таймстемп файлов в base path. + * Если уже выполняется flush, возвращает null. + * Если предыдущий flush() был завершен с ошибкой, возвратит таску на повторную попытку. + * Если нет данных, возвращает null. + */ + @Override + public Callable prepareFlush(Path basePath, String dbFilenamePrefix, String offsetsFilenamePrefix) { + FlushingDaoState newFlushingDaoState; + + stateLock.writeLock().lock(); + try { + switch (flushingDaoState.flushingState) { + case RUNNING -> { + return null; + } + case FAILED -> { + newFlushingDaoState = flushingDaoState.failedToTryAgain(); + flushingDaoState = newFlushingDaoState; + } + case NOT_RUNNING -> { + DaoState newDaoState = createEmptyDaoState(); + newFlushingDaoState = daoState.toFlushingRunningDaoState(); + + flushingDaoState = newFlushingDaoState; + daoState = newDaoState; + } + default -> throw new IllegalStateException("Unexpected state: " + flushingDaoState.flushingState); + } + } finally { + stateLock.writeLock().unlock(); + } + + return () -> { + ConcurrentNavigableMap dao = newFlushingDaoState.dao; + int recordsCount = dao.size(); + long daoSize = newFlushingDaoState.daoSize; + TableInfo tableInfo = new TableInfo(recordsCount, daoSize); + + return flushImpl(dao.values().iterator(), tableInfo, basePath, dbFilenamePrefix, offsetsFilenamePrefix); + }; + } + + private String flushImpl(Iterator immutableCollectionIterator, TableInfo info, Path basePath, + String dbPrefix, String offsetsPrefix) throws IOException { + String timestamp = String.valueOf(System.currentTimeMillis()); + Path tempDirectory = Files.createTempDirectory(null); + Path newSSTable = null; + + logger.info(() -> String.format("Flushing started to dir %s, timestamp %s, info %s", + tempDirectory, timestamp, info)); + try { + Path tmpSSTable = tempDirectory.resolve(dbPrefix + timestamp); + Path tmpOffsets = tempDirectory.resolve(offsetsPrefix + timestamp); + + StorageUtility.writeData(tmpSSTable, tmpOffsets, immutableCollectionIterator, info, extractor); + + newSSTable = Files.move(tmpSSTable, basePath.resolve(dbPrefix + timestamp), MOVE_OPTIONS); + Files.move(tmpOffsets, basePath.resolve(offsetsPrefix + timestamp), MOVE_OPTIONS); + } catch (Exception e) { + // newOffsets чистить не надо. Это последняя операция, если исключение то он точно не перемещен. + if (newSSTable != null) { + StorageUtility.deleteUnusedFiles(logger, newSSTable); + } + throw e; + } finally { + StorageUtility.deleteUnusedFilesInDirectory(logger, tempDirectory); + } + logger.info(() -> String.format("Flushed to dir %s, timestamp %s", basePath, timestamp)); + return timestamp; + } + + @Override + public void failFlush() { + stateLock.writeLock().lock(); + try { + // Помечаем как failed, но не чистим мапу и не теряем данные + flushingDaoState = flushingDaoState.toFailed(); + } finally { + stateLock.writeLock().unlock(); + } + } + + @Override + public void completeFlush() { + stateLock.writeLock().lock(); + try { + flushingDaoState = createEmptyFlushingDaoState(); + } finally { + stateLock.writeLock().unlock(); + } + } + + @Override + public String close(Path basePath, String dbFilenamePrefix, String offsetsFilenamePrefix) throws IOException { + ConcurrentNavigableMap dao; + long size; + + stateLock.writeLock().lock(); + try { + dao = daoState.dao; + size = daoState.daoSize.get(); + daoState = null; + } finally { + stateLock.writeLock().unlock(); + } + + if (dao.isEmpty()) { + return null; + } + int recordsCount = dao.size(); + TableInfo tableInfo = new TableInfo(recordsCount, size); + return flushImpl(dao.values().iterator(), tableInfo, basePath, dbFilenamePrefix, offsetsFilenamePrefix); + } +} diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/SSTableStorage.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/SSTableStorage.java new file mode 100644 index 000000000..f3ceffcd5 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/SSTableStorage.java @@ -0,0 +1,23 @@ +package ru.vk.itmo.kovalchukvladislav.storage; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +public interface SSTableStorage> { + /** + * Добавляет новый SSTable из basePath в метадату и рефрешит SSTableStorage. + * Используется при flush(), рефреш полезно выключать при flush() внутри close(), когда больше не будет запросов. + */ + void addSSTableId(String id, boolean needRefresh) throws IOException; + + E get(D key); + + List> getIterators(D from, D to); + + void compact() throws IOException; + + void close(); +} diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/SSTableStorageImpl.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/SSTableStorageImpl.java new file mode 100644 index 000000000..806c650ee --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/SSTableStorageImpl.java @@ -0,0 +1,243 @@ +package ru.vk.itmo.kovalchukvladislav.storage; + +import ru.vk.itmo.Entry; +import ru.vk.itmo.kovalchukvladislav.model.DaoIterator; +import ru.vk.itmo.kovalchukvladislav.model.EntryExtractor; +import ru.vk.itmo.kovalchukvladislav.model.SimpleDaoLoggerUtility; +import ru.vk.itmo.kovalchukvladislav.model.StorageIterator; +import ru.vk.itmo.kovalchukvladislav.model.TableInfo; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Logger; + +public class SSTableStorageImpl> implements SSTableStorage { + private static final StandardCopyOption[] MOVE_OPTIONS = new StandardCopyOption[]{ + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + }; + private static final Logger logger = SimpleDaoLoggerUtility.createLogger(SSTableStorageImpl.class); + private final Path basePath; + private final String metadataFilename; + private final String dataPrefix; + private final String offsetsPrefix; + private final Arena arena = Arena.ofShared(); + private final EntryExtractor extractor; + private final Set filesToDelete = ConcurrentHashMap.newKeySet(); + private volatile State state; + + public SSTableStorageImpl(Path basePath, + String metadataFilename, + String dbFilenamePrefix, + String offsetsPrefix, + EntryExtractor extractor) throws IOException { + this.basePath = basePath; + this.metadataFilename = metadataFilename; + this.dataPrefix = dbFilenamePrefix; + this.offsetsPrefix = offsetsPrefix; + this.extractor = extractor; + if (!Files.exists(basePath)) { + Files.createDirectory(basePath); + } + this.state = reloadSSTableIds(); + } + + @SuppressWarnings("unused") + // Компилятор ругается на unused переменные внутри record, хотя они очень даже used + private record State(List ssTableIds, List data, List offsets) { + public int getCount() { + return ssTableIds.size(); + } + } + + @Override + public E get(D key) { + State currentState = state; + for (int i = currentState.getCount() - 1; i >= 0; i--) { + MemorySegment storage = currentState.data.get(i); + MemorySegment offsets = currentState.offsets.get(i); + + long offset = extractor.findLowerBoundValueOffset(key, storage, offsets); + if (offset == -1) { + continue; + } + D lowerBoundKey = extractor.readValue(storage, offset); + + if (extractor.compare(lowerBoundKey, key) == 0) { + long valueOffset = offset + extractor.size(lowerBoundKey); + D value = extractor.readValue(storage, valueOffset); + return extractor.createEntry(lowerBoundKey, value); + } + } + return null; + } + + @Override + public List> getIterators(D from, D to) { + State currentState = state; + List> iterators = new ArrayList<>(currentState.getCount()); + for (int i = currentState.getCount() - 1; i >= 0; i--) { + MemorySegment storage = currentState.data.get(i); + MemorySegment offsets = currentState.offsets.get(i); + iterators.add(new StorageIterator<>(from, to, storage, offsets, extractor)); + } + return iterators; + } + + // State может поменяться только в addSSTableId (используется при flush() для обновления состояния) и compact(). + // Оба метода никогда не вызываются одновременно из AbstractBasedOnSSTableDao, синхронизация не нужна. + @Override + @SuppressWarnings("unused") + // Компилятор ругается на unused ignoredPath, хотя в названии переменной есть unused + public void addSSTableId(String id, boolean needRefresh) throws IOException { + Path ignoredPath = addSSTableId(basePath, id); + if (needRefresh) { + state = reloadSSTableIds(); + } + } + + @Override + public void compact() throws IOException { + List ssTableIds = state.ssTableIds; + if (ssTableIds.size() <= 1) { + logger.info("SSTables <= 1, not compacting: " + ssTableIds); + return; + } + + compactAndChangeMetadata(); + state = reloadSSTableIds(); + filesToDelete.addAll(convertSSTableIdsToPath(ssTableIds)); + } + + @Override + public void close() { + if (arena.scope().isAlive()) { + arena.close(); + StorageUtility.deleteUnusedFiles(logger, filesToDelete.toArray(Path[]::new)); + } + } + + private State reloadSSTableIds() throws IOException { + List ssTableIds = readSSTableIds(); + logger.info(() -> String.format("Reloading files from %s", basePath)); + List newDbMappedSegments = new ArrayList<>(ssTableIds.size()); + List newOffsetMappedSegments = new ArrayList<>(ssTableIds.size()); + + for (String ssTableId : ssTableIds) { + readFileAndMapToSegment(newDbMappedSegments, newOffsetMappedSegments, ssTableId); + } + logger.info(() -> String.format("Reloaded %d files", ssTableIds.size())); + + return new State(ssTableIds, newDbMappedSegments, newOffsetMappedSegments); + } + + private List readSSTableIds() throws IOException { + Path metadataPath = basePath.resolve(metadataFilename); + if (!Files.exists(metadataPath)) { + return Collections.emptyList(); + } + return Files.readAllLines(metadataPath, StandardCharsets.UTF_8); + } + + private void readFileAndMapToSegment(List dbMappedResult, + List offsetMappedResult, + String timestamp) throws IOException { + Path dbPath = basePath.resolve(dataPrefix + timestamp); + Path offsetsPath = basePath.resolve(offsetsPrefix + timestamp); + if (!Files.exists(dbPath) || !Files.exists(offsetsPath)) { + throw new FileNotFoundException("File under path " + dbPath + " or " + offsetsPath + " doesn't exists"); + } + + logger.info(() -> String.format("Reading files with timestamp %s", timestamp)); + + try (FileChannel dbChannel = FileChannel.open(dbPath, StandardOpenOption.READ); + FileChannel offsetChannel = FileChannel.open(offsetsPath, StandardOpenOption.READ)) { + + MemorySegment db = dbChannel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(dbPath), arena); + MemorySegment offsets = + offsetChannel.map(FileChannel.MapMode.READ_ONLY, 0, Files.size(offsetsPath), arena); + dbMappedResult.add(db); + offsetMappedResult.add(offsets); + } + logger.info(() -> String.format("Successfully read files with %s timestamp", timestamp)); + } + + private Path addSSTableId(Path path, String id) throws IOException { + return Files.writeString(path.resolve(metadataFilename), id + System.lineSeparator(), + StandardOpenOption.WRITE, StandardOpenOption.APPEND, StandardOpenOption.CREATE); + } + + private void compactAndChangeMetadata() throws IOException { + Path tempDirectory = Files.createTempDirectory(null); + String timestamp = String.valueOf(System.currentTimeMillis()); + + Path newSSTable = null; + Path newOffsetsTable = null; + Path tmpSSTable = tempDirectory.resolve(dataPrefix + timestamp); + Path tmpOffsetsTable = tempDirectory.resolve(offsetsPrefix + timestamp); + + try { + Iterator iterator = getIterator(); + TableInfo info = calculateStorageTableInfo(); + logger.info(() -> String.format("Compacting started to dir %s, timestamp %s, info %s", + tempDirectory, timestamp, info)); + + StorageUtility.writeData(tmpSSTable, tmpOffsetsTable, iterator, info, extractor); + Path tmpMetadata = addSSTableId(tempDirectory, timestamp); + Path newMetadata = basePath.resolve(metadataFilename); + + newSSTable = Files.move(tmpSSTable, basePath.resolve(dataPrefix + timestamp), MOVE_OPTIONS); + newOffsetsTable = Files.move(tmpOffsetsTable, basePath.resolve(offsetsPrefix + timestamp), + MOVE_OPTIONS); + Files.move(tmpMetadata, newMetadata, MOVE_OPTIONS); + } catch (Exception e) { + if (newOffsetsTable != null) { + StorageUtility.deleteUnusedFiles(logger, newSSTable, newOffsetsTable); + } else if (newSSTable != null) { + StorageUtility.deleteUnusedFiles(logger, newSSTable); + } + throw e; + } finally { + StorageUtility.deleteUnusedFiles(logger, tempDirectory); + } + logger.info(() -> String.format("Compacted to dir %s, timestamp %s", basePath, timestamp)); + } + + private TableInfo calculateStorageTableInfo() { + Iterator iterator = getIterator(); + long size = 0; + int count = 0; + while (iterator.hasNext()) { + count++; + size += extractor.size(iterator.next()); + } + return new TableInfo(count, size); + } + + private Iterator getIterator() { + return new DaoIterator<>(getIterators(null, null), extractor); + } + + private List convertSSTableIdsToPath(List ssTableIds) { + List result = new ArrayList<>(ssTableIds.size() * 2); + for (String ssTableId : ssTableIds) { + result.add(basePath.resolve(dataPrefix + ssTableId)); + result.add(basePath.resolve(offsetsPrefix + ssTableId)); + } + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/StorageUtility.java b/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/StorageUtility.java new file mode 100644 index 000000000..62bb7ba74 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalchukvladislav/storage/StorageUtility.java @@ -0,0 +1,84 @@ +package ru.vk.itmo.kovalchukvladislav.storage; + +import ru.vk.itmo.Entry; +import ru.vk.itmo.kovalchukvladislav.model.EntryExtractor; +import ru.vk.itmo.kovalchukvladislav.model.TableInfo; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Comparator; +import java.util.Iterator; +import java.util.logging.Logger; +import java.util.stream.Stream; + +public final class StorageUtility { + private static final OpenOption[] WRITE_OPTIONS = new OpenOption[] { + StandardOpenOption.READ, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.CREATE + }; + private static final int OFFSET_SIZE = Long.BYTES; + + private StorageUtility() { + } + + // Удаление ненужных файлов не является чем то критически важным + // Если произойдет исключение, лучше словить и вывести в лог, чем останавливать работу + public static void deleteUnusedFiles(Logger logger, Path... files) { + for (Path file : files) { + try { + boolean deleted = Files.deleteIfExists(file); + if (deleted) { + logger.info(() -> String.format("File %s was deleted", file)); + } else { + logger.severe(() -> String.format("File %s not deleted", file)); + } + } catch (IOException e) { + logger.severe(() -> String.format("Error while deleting file %s: %s", file, e.getMessage())); + } + } + } + + public static void deleteUnusedFilesInDirectory(Logger logger, Path directory) { + try (Stream files = Files.walk(directory)) { + Path[] array = files.sorted(Comparator.reverseOrder()).toArray(Path[]::new); + deleteUnusedFiles(logger, array); + } catch (Exception e) { + logger.severe(() -> String.format("Error while deleting directory %s: %s", directory, e.getMessage())); + } + } + + public static > void writeData(Path dbPath, Path offsetsPath, + Iterator daoIterator, TableInfo info, + EntryExtractor extractor) throws IOException { + + try (FileChannel db = FileChannel.open(dbPath, WRITE_OPTIONS); + FileChannel offsets = FileChannel.open(offsetsPath, WRITE_OPTIONS); + Arena arena = Arena.ofConfined()) { + + long offsetsSize = info.recordsCount() * OFFSET_SIZE; + MemorySegment fileSegment = db.map(FileChannel.MapMode.READ_WRITE, 0, info.recordsSize(), arena); + MemorySegment offsetsSegment = offsets.map(FileChannel.MapMode.READ_WRITE, 0, offsetsSize, arena); + + int i = 0; + long offset = 0; + while (daoIterator.hasNext()) { + E entry = daoIterator.next(); + offsetsSegment.setAtIndex(ValueLayout.JAVA_LONG_UNALIGNED, i, offset); + offset = extractor.writeEntry(entry, fileSegment, offset); + i += 1; + } + + fileSegment.load(); + offsetsSegment.load(); + } + } +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/DaoFileGet.java b/src/main/java/ru/vk/itmo/kovalevigor/DaoFileGet.java new file mode 100644 index 000000000..8bd36d490 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalevigor/DaoFileGet.java @@ -0,0 +1,13 @@ +package ru.vk.itmo.kovalevigor; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.util.Iterator; + +public interface DaoFileGet> { + + Iterator get(D from, D to) throws IOException; + + E get(D key) throws IOException; +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/DaoImpl.java b/src/main/java/ru/vk/itmo/kovalevigor/DaoImpl.java index dd6b807cb..31868e461 100644 --- a/src/main/java/ru/vk/itmo/kovalevigor/DaoImpl.java +++ b/src/main/java/ru/vk/itmo/kovalevigor/DaoImpl.java @@ -5,93 +5,240 @@ import ru.vk.itmo.Entry; import java.io.IOException; -import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; +import java.util.List; import java.util.Objects; +import java.util.SortedMap; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.logging.Level; +import java.util.logging.Logger; public class DaoImpl implements Dao> { - private final Arena memoryArena; - private final SSTable ssTable; - private final ConcurrentNavigableMap> storage; - public static final String SSTABLE_NAME = "sstable"; - public static final long TABLE_SIZE_LIMIT = 1024 * 8; // TODO: От балды + private final SSTableManager ssManager; + private static final ConcurrentNavigableMap> EMPTY_MAP = + new ConcurrentSkipListMap<>(SSTable.COMPARATOR); + private ConcurrentNavigableMap> flushedStorage; + private ConcurrentNavigableMap> currentStorage; + private final AtomicLong currentMemoryByteSize; + private final long flushThresholdBytes; + private final ExecutorService flushService; + private final ExecutorService compactService; + private Future flushFuture; + private final ReadWriteLock lock = new ReentrantReadWriteLock(); public DaoImpl(final Config config) throws IOException { - memoryArena = Arena.ofShared(); - ssTable = new SSTable(getSSTablePath(config.basePath())); - storage = new ConcurrentSkipListMap<>(ssTable.load(memoryArena, TABLE_SIZE_LIMIT)); + ssManager = new SSTableManager(config.basePath()); + currentStorage = new ConcurrentSkipListMap<>(SSTable.COMPARATOR); + flushedStorage = EMPTY_MAP; + flushThresholdBytes = config.flushThresholdBytes(); + flushService = Executors.newSingleThreadExecutor(); + compactService = Executors.newSingleThreadExecutor(); + currentMemoryByteSize = new AtomicLong(); } - private static Iterator getValuesIterator(final ConcurrentNavigableMap map) { + private static Iterator getValuesIterator(final SortedMap map) { return map.values().iterator(); } - @Override - public Iterator> get(final MemorySegment from, final MemorySegment to) { + private static Iterator> getIterator( + final SortedMap> sortedMap, + final MemorySegment from, + final MemorySegment to + ) { if (from == null) { if (to == null) { - return all(); + return getValuesIterator(sortedMap); + } else { + return getValuesIterator(sortedMap.headMap(to)); } - return allTo(to); } else if (to == null) { - return allFrom(from); + return getValuesIterator(sortedMap.tailMap(from)); + } else { + return getValuesIterator(sortedMap.subMap(from, to)); } - return getValuesIterator(storage.subMap(from, to)); } @Override - public void upsert(final Entry entry) { - Objects.requireNonNull(entry); - storage.put(entry.key(), entry); + public Iterator> get(final MemorySegment from, final MemorySegment to) { + final List>> iterators = new ArrayList<>(3); + lock.readLock().lock(); + try { + iterators.add(new MemEntryPriorityIterator(0, getIterator(currentStorage, from, to))); + iterators.add(new MemEntryPriorityIterator(1, getIterator(flushedStorage, from, to))); + } finally { + lock.readLock().unlock(); + } + try { + iterators.add(new MemEntryPriorityIterator(2, ssManager.get(from, to))); + } catch (IOException e) { + log(e); + } + return new MergeEntryIterator(iterators); } - @Override - public Iterator> allFrom(final MemorySegment from) { - Objects.requireNonNull(from); - return getValuesIterator(storage.tailMap(from)); + private static long getMemorySegmentSize(final MemorySegment memorySegment) { + return memorySegment == null ? 0 : memorySegment.byteSize(); } - @Override - public Iterator> allTo(final MemorySegment to) { - Objects.requireNonNull(to); - return getValuesIterator(storage.headMap(to)); + private static long getEntrySize(final Entry entry) { + return getMemorySegmentSize(entry.key()) + getMemorySegmentSize(entry.value()); } @Override - public Iterator> all() { - return getValuesIterator(storage); + public void upsert(final Entry entry) { + Objects.requireNonNull(entry); + final long entrySize = getEntrySize(entry); + + lock.readLock().lock(); + try { + currentStorage.put(entry.key(), entry); + currentMemoryByteSize.addAndGet(entrySize); + } finally { + lock.readLock().unlock(); + } + final long newSize = currentMemoryByteSize.get() + entrySize; + if (newSize >= flushThresholdBytes) { + if (!flushedStorage.isEmpty()) { + throw new IllegalStateException("Limit is reached. U should wait"); + } + flush(); + } } @Override public Entry get(final MemorySegment key) { Objects.requireNonNull(key); - final Entry result = storage.get(key); + Entry result; + lock.readLock().lock(); + try { + result = currentStorage.get(key); + if (result == null) { + result = flushedStorage.get(key); + } + } finally { + lock.readLock().unlock(); + } + if (result != null) { + if (result.value() == null) { + return null; + } return result; } + try { - return ssTable.get(key, memoryArena); + return ssManager.get(key); } catch (IOException e) { return null; } } - private Path getSSTablePath(final Path base) { - return base.resolve(SSTABLE_NAME); + @Override + public void flush() { + if (!flushedStorage.isEmpty()) { + return; + } + + final ConcurrentNavigableMap> storage = + new ConcurrentSkipListMap<>(SSTable.COMPARATOR); + lock.writeLock().lock(); + try { + + if (currentStorage.isEmpty()) { + return; + } + + flushedStorage = currentStorage; + currentStorage = storage; + currentMemoryByteSize.set(0); + } finally { + lock.writeLock().unlock(); + } + + flushFuture = flushService.submit(() -> { + String name = null; + try { + name = ssManager.write(flushedStorage); + } catch (IOException e) { + log(e); + } finally { + lock.writeLock().lock(); + try { + if (name != null) { + ssManager.addSSTable(name); + } + flushedStorage = EMPTY_MAP; + } catch (IOException e) { + log(e); + } finally { + lock.writeLock().unlock(); + } + } + }, null); + } + + private static void awaitShutdown(ExecutorService service) { + while (true) { + try { + if (service.awaitTermination(1, TimeUnit.SECONDS)) { + return; + } + } catch (InterruptedException e) { + log(e); + Thread.currentThread().interrupt(); + } + } + } + + @Override + public void compact() throws IOException { + compactService.execute(() -> { + try { + ssManager.compact(); + } catch (IOException e) { + log(e); + } + }); } @Override public void close() throws IOException { - if (!memoryArena.scope().isAlive()) { - return; + try { + if (flushFuture != null) { + flushFuture.get(); + } + } catch (InterruptedException e) { + log(e); + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + log(e); + } + flush(); + compactService.shutdown(); + flushService.shutdown(); + awaitShutdown(compactService); + awaitShutdown(flushService); + currentStorage.clear(); + flushedStorage.clear(); + ssManager.close(); + } + + private static void log(Exception e) { + if (Logger.getAnonymousLogger().isLoggable(Level.WARNING)) { + Logger.getAnonymousLogger().log(Level.WARNING, Arrays.toString(e.getStackTrace())); } - ssTable.write(storage); - memoryArena.close(); - storage.clear(); } } diff --git a/src/main/java/ru/vk/itmo/kovalevigor/Dumper.java b/src/main/java/ru/vk/itmo/kovalevigor/Dumper.java new file mode 100644 index 000000000..d54b59e2e --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalevigor/Dumper.java @@ -0,0 +1,17 @@ +package ru.vk.itmo.kovalevigor; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.nio.file.Path; + +public abstract class Dumper extends SegmentWriter implements AutoCloseable { + + protected Dumper(final Path path, final long fileSize, final Arena arena) throws IOException { + super(path, fileSize, arena); + } + + protected abstract void writeHead(); + + @Override + public abstract void close() throws IOException; +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/IndexDumper.java b/src/main/java/ru/vk/itmo/kovalevigor/IndexDumper.java new file mode 100644 index 000000000..b9bb1bbac --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalevigor/IndexDumper.java @@ -0,0 +1,72 @@ +package ru.vk.itmo.kovalevigor; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.ValueLayout; +import java.nio.file.Path; + +public class IndexDumper extends Dumper { + + public static final long HEAD_SIZE = 2 * ValueLayout.JAVA_LONG.byteSize(); + public static final long ENTRY_SIZE = 2 * ValueLayout.JAVA_LONG.byteSize(); + public long keysSize; + public long valuesSize; + private long nullCount; + + protected IndexDumper( + final long entryCount, + final Path path, + final Arena arena + ) throws IOException { + super(path, getSize(entryCount), arena); + this.keysSize = 0; + this.valuesSize = 0; + this.nullCount = 0; + offset = HEAD_SIZE; + } + + public static long getSize(final long entryCount) { + return HEAD_SIZE + entryCount * ENTRY_SIZE; + } + + public void setKeysSize(final long keysSize) { + this.keysSize = keysSize; + } + + public void setValuesSize(final long valuesSize) { + this.valuesSize = valuesSize; + } + + @Override + protected void writeHead() { + writeLong(memorySegment, 0, keysSize); + writeLong(memorySegment, ValueLayout.JAVA_LONG.byteSize(), valuesSize); + } + + private void fillNulls(final long valueOffset) { + long curOffset = offset - ValueLayout.JAVA_LONG.byteSize(); + while (nullCount > 0) { + writeLong(memorySegment, curOffset, -valueOffset - 1); + curOffset -= ENTRY_SIZE; + nullCount -= 1; + } + } + + public void writeEntry(final long keyOffset, final long valueOffset) { + if (valueOffset < -1) { + throw new IllegalArgumentException("Invalid valueOffset value. Should be equal or greater than -1"); + } + if (valueOffset == -1) { + nullCount += 1; + } else { + fillNulls(valueOffset); + } + writeLong(keyOffset); + writeLong(valueOffset); + } + + @Override + public void close() { + fillNulls(valuesSize); + } +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/IndexList.java b/src/main/java/ru/vk/itmo/kovalevigor/IndexList.java new file mode 100644 index 000000000..eea2f5bdb --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalevigor/IndexList.java @@ -0,0 +1,118 @@ +package ru.vk.itmo.kovalevigor; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.AbstractList; +import java.util.RandomAccess; + +public class IndexList extends AbstractList> implements RandomAccess { + + public static final long INDEX_ENTRY_SIZE = IndexDumper.ENTRY_SIZE; + public static final long META_INFO_SIZE = IndexDumper.HEAD_SIZE; + public static long MAX_BYTE_SIZE = META_INFO_SIZE + Integer.MAX_VALUE * INDEX_ENTRY_SIZE; + + private final MemorySegment indexSegment; + private final MemorySegment dataSegment; + private final long keysSize; + private final long valuesSize; + + public IndexList(final MemorySegment indexSegment, final MemorySegment dataSegment) { + if (indexSegment.byteSize() > MAX_BYTE_SIZE) { + this.indexSegment = indexSegment.asSlice(0, MAX_BYTE_SIZE); + } else { + this.indexSegment = indexSegment; + } + + this.dataSegment = dataSegment; + this.keysSize = readOffset(0); + this.valuesSize = readOffset(ValueLayout.JAVA_LONG.byteSize()); + } + + private long getEntryOffset(final int index) { + if (size() <= index) { + return -1; + } + return META_INFO_SIZE + INDEX_ENTRY_SIZE * index; + } + + private long readOffset(final long offset) { + return indexSegment.get(ValueLayout.JAVA_LONG, offset); + } + + public class LazyEntry implements Entry { + + private final MemorySegment key; + private final int index; + private MemorySegment value; + + private LazyEntry(MemorySegment key, int index) { + this.key = key; + this.index = index; + } + + @Override + public MemorySegment key() { + return key; + } + + @Override + public MemorySegment value() { + if (value == null) { + value = getValue(index); + } + return value; + } + + } + + private MemorySegment getValue(final int index) { + final long offset = getEntryOffset(index); + final long valueOffset = readOffset(offset + ValueLayout.JAVA_LONG.byteSize()); + final long nextEntryOffset = getEntryOffset(index + 1); + + final MemorySegment value; + if (valueOffset >= 0) { + + long valueSize; + if (nextEntryOffset == -1) { + valueSize = dataSegment.byteSize() - keysSize(); + } else { + final long nextValueOffset = readOffset(nextEntryOffset + ValueLayout.JAVA_LONG.byteSize()); + valueSize = (nextValueOffset > 0 ? nextValueOffset : -(nextValueOffset + 1)); + } + value = dataSegment.asSlice(valueOffset + keysSize(), valueSize - valueOffset); + } else { + value = null; + } + return value; + } + + @Override + public LazyEntry get(final int index) { + final long offset = getEntryOffset(index); + + final long keyOffset = readOffset(offset); + final long nextEntryOffset = getEntryOffset(index + 1); + + long keySize = (nextEntryOffset == -1 ? keysSize : readOffset(nextEntryOffset)) - keyOffset; + + final MemorySegment key = dataSegment.asSlice(keyOffset, keySize); + return new LazyEntry(key, index); + } + + @Override + public int size() { + return (int)((indexSegment.byteSize() - META_INFO_SIZE) / INDEX_ENTRY_SIZE); + } + + public long keysSize() { + return keysSize; + } + + public long valuesSize() { + return valuesSize; + } + +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/MemEntryPriorityIterator.java b/src/main/java/ru/vk/itmo/kovalevigor/MemEntryPriorityIterator.java new file mode 100644 index 000000000..ab391a0bc --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalevigor/MemEntryPriorityIterator.java @@ -0,0 +1,28 @@ +package ru.vk.itmo.kovalevigor; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; + +public class MemEntryPriorityIterator extends PriorityShiftedIterator> { + + public MemEntryPriorityIterator(int priority) { + super(priority); + } + + public MemEntryPriorityIterator(int priority, Iterator> iterator) { + super(priority, iterator); + } + + @Override + public int compareTo(final PriorityShiftedIterator rhs) { + if (rhs instanceof MemEntryPriorityIterator rhsEntry) { + final int comp = UtilsMemorySegment.compareEntry(value, rhsEntry.value); + if (comp != 0) { + return comp; + } + } + return super.compareTo(rhs); + } +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/MergeEntryIterator.java b/src/main/java/ru/vk/itmo/kovalevigor/MergeEntryIterator.java new file mode 100644 index 000000000..cfa579421 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalevigor/MergeEntryIterator.java @@ -0,0 +1,37 @@ +package ru.vk.itmo.kovalevigor; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Collection; + +public class MergeEntryIterator extends MergeIterator< + Entry, + PriorityShiftedIterator> + > { + public MergeEntryIterator(Collection>> collection) { + super(collection); + } + + @Override + public boolean hasNext() { + checkAndSkip(); + return super.hasNext(); + } + + @Override + protected boolean checkEquals( + final PriorityShiftedIterator> lhs, + final PriorityShiftedIterator> rhs + ) { + return UtilsMemorySegment.compareEntry(lhs.value, rhs.value) == 0; + } + + private void checkAndSkip() { + PriorityShiftedIterator> next = queue.peek(); + while (next != null && super.hasNext() && next.value.value() == null) { + next(); + next = queue.peek(); + } + } +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/MergeIterator.java b/src/main/java/ru/vk/itmo/kovalevigor/MergeIterator.java new file mode 100644 index 000000000..d13cf9daa --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalevigor/MergeIterator.java @@ -0,0 +1,49 @@ +package ru.vk.itmo.kovalevigor; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.PriorityQueue; + +public abstract class MergeIterator> implements Iterator { + + protected final PriorityQueue queue; + + protected MergeIterator(final Collection collection) { + final List filteredCollection = new ArrayList<>(collection.size()); + for (final T iterator : collection) { + if (iterator.hasNext()) { + filteredCollection.add(iterator); + } + } + queue = new PriorityQueue<>(filteredCollection); + } + + @Override + public boolean hasNext() { + return !queue.isEmpty(); + } + + @Override + public E next() { + final T iterator = queue.remove(); + + T nextIterator = queue.peek(); + while (nextIterator != null && checkEquals(iterator, nextIterator)) { + shiftAdd(queue.remove()); + nextIterator = queue.peek(); + } + return shiftAdd(iterator); + } + + protected abstract boolean checkEquals(T lhs, T rhs); + + private E shiftAdd(final T iterator) { + final E value = iterator.next(); + if (iterator.hasNext()) { + queue.add(iterator); + } + return value; + } +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/PriorityShiftedIterator.java b/src/main/java/ru/vk/itmo/kovalevigor/PriorityShiftedIterator.java new file mode 100644 index 000000000..410da1f29 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalevigor/PriorityShiftedIterator.java @@ -0,0 +1,23 @@ +package ru.vk.itmo.kovalevigor; + +import java.util.Iterator; + +public class PriorityShiftedIterator extends ShiftedIterator implements Comparable> { + + private final int priority; + + public PriorityShiftedIterator(final int priority) { + super(); + this.priority = priority; + } + + public PriorityShiftedIterator(final int priority, final Iterator iterator) { + super(iterator); + this.priority = priority; + } + + @Override + public int compareTo(final PriorityShiftedIterator rhs) { + return Integer.compare(priority, rhs.priority); + } +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/SSTable.java b/src/main/java/ru/vk/itmo/kovalevigor/SSTable.java index f262f1f63..5e9a77b63 100644 --- a/src/main/java/ru/vk/itmo/kovalevigor/SSTable.java +++ b/src/main/java/ru/vk/itmo/kovalevigor/SSTable.java @@ -5,110 +5,147 @@ import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; +import java.util.Collections; import java.util.Comparator; +import java.util.Iterator; import java.util.Map; import java.util.SortedMap; -import java.util.TreeMap; -public class SSTable { +public class SSTable implements DaoFileGet> { - private final Path path; public static final Comparator COMPARATOR = UtilsMemorySegment::compare; + public static final Comparator> ENTRY_COMPARATOR = UtilsMemorySegment::compareEntry; - public SSTable(final Path path) { - this.path = path; + private Path indexPath; + private Path dataPath; + private final IndexList indexList; + + private SSTable(final Path indexPath, final Path dataPath, final Arena arena) throws IOException { + this.indexPath = indexPath; + this.dataPath = dataPath; + this.indexList = new IndexList( + UtilsMemorySegment.mapReadSegment(indexPath, arena), + UtilsMemorySegment.mapReadSegment(dataPath, arena) + ); } - public SortedMap> load(final Arena arena, final long limit) throws IOException { - final SortedMap> map = new TreeMap<>(COMPARATOR); - if (Files.notExists(path)) { - return map; + public static SSTable create(final Path root, final String name, final Arena arena) throws IOException { + final Path indexPath = getIndexPath(root, name); + final Path dataPath = getDataPath(root, name); + if (Files.notExists(indexPath) || Files.notExists(dataPath)) { + return null; } + return new SSTable(indexPath, dataPath, arena); + } - try (FileChannel readerChannel = FileChannel.open( - path, - StandardOpenOption.READ) - ) { - final MemorySegment memorySegment = readerChannel.map( - FileChannel.MapMode.READ_ONLY, - 0, - readerChannel.size(), - arena - ); - final MemoryEntryReader reader = new MemoryEntryReader(memorySegment); - - Entry entry = reader.readEntry(limit); - while (entry != null) { - map.put(entry.key(), entry); - entry = reader.readEntry(limit); - } - return map; - } + public static Path getDataPath(final Path root, final String name) { + return root.resolve(name); } - public SortedMap> load(final Arena arena) throws IOException { - return load(arena, 0); + public static Path getIndexPath(final Path root, final String name) { + return root.resolve(name + "_index"); } - public Entry get(final MemorySegment key, final Arena segmentArena) throws IOException { - if (Files.notExists(path)) { + private static final class KeyEntry implements Entry { + + final MemorySegment key; + + public KeyEntry(final MemorySegment key) { + this.key = key; + } + + @Override + public MemorySegment key() { + return key; + } + + @Override + public MemorySegment value() { return null; } + } - try ( - Arena readerArena = Arena.ofConfined(); - FileChannel readerChannel = FileChannel.open(path, StandardOpenOption.READ) - ) { - - final MemorySegment memorySegment = readerChannel.map( - FileChannel.MapMode.READ_ONLY, - 0, - readerChannel.size(), - readerArena - ); - final MemoryEntryReader reader = new MemoryEntryReader(memorySegment); - - long prevOffset = reader.getOffset(); - Entry entry = reader.readEntry(); - while (entry != null) { - if (UtilsMemorySegment.findDiff(key, entry.key()) == -1) { - return MemoryEntryReader.mapEntry( - readerChannel, - prevOffset, - entry.key().byteSize(), - entry.value().byteSize(), - segmentArena - ); - } - prevOffset = reader.getOffset(); - entry = reader.readEntry(); - } + private int binarySearch(final MemorySegment key) { + return Collections.binarySearch(indexList, new KeyEntry(key), ENTRY_COMPARATOR); + } + + @Override + public Entry get(final MemorySegment key) throws IOException { + final int pos = binarySearch(key); + return pos >= 0 ? indexList.get(pos) : null; + } + + @Override + public Iterator> get(final MemorySegment from, final MemorySegment to) throws IOException { + int startPos = 0; + if (from != null && (startPos = binarySearch(from)) < 0) { + startPos = -(startPos + 1); + } + int endPos = indexList.size(); + if (to != null && (endPos = binarySearch(to)) < 0) { + endPos = -(endPos + 1); + } + return indexList.subList(startPos, endPos).iterator(); + } + + public static SizeInfo getMapSize(final SortedMap> map) { + long keysSize = 0; + long valuesSize = 0; + for (Map.Entry> entry : map.entrySet()) { + final MemorySegment value = entry.getValue().value(); + + keysSize += entry.getKey().byteSize(); + valuesSize += value == null ? 0 : value.byteSize(); } - return null; + return new SizeInfo(map.size(), keysSize, valuesSize); } - public void write(final SortedMap> map) throws IOException { - try (Arena arena = Arena.ofConfined(); FileChannel writerChannel = FileChannel.open( - path, - StandardOpenOption.CREATE, - StandardOpenOption.READ, - StandardOpenOption.WRITE) - ) { - final MemorySegment memorySegment = writerChannel.map( - FileChannel.MapMode.READ_WRITE, - 0, - MemoryEntryWriter.getTotalMapSize(map), - arena - ); - final MemoryEntryWriter writer = new MemoryEntryWriter(memorySegment); - - for (Map.Entry> entry : map.entrySet()) { - writer.putEntry(entry); + public static void write( + final SortedMap> map, + final Path path, + final String name + ) throws IOException { + try (Arena arena = Arena.ofConfined(); SStorageDumper dumper = new SStorageDumper( + getMapSize(map), + getDataPath(path, name), + getIndexPath(path, name), + arena + )) { + for (final Entry entry: map.values()) { + dumper.writeEntry(entry); } } } + + public void move( + final Path path, + final String name + ) throws IOException { + final Path targetDataPath = getDataPath(path, name); + final Path targetIndexPath = getIndexPath(path, name); + Files.move(dataPath, targetDataPath); + try { + Files.move(indexPath, targetIndexPath); + } catch (IOException e) { + Files.move(targetDataPath, dataPath); + throw e; + } + dataPath = targetDataPath; + indexPath = targetIndexPath; + } + + public SizeInfo getSizeInfo() { + return new SizeInfo(indexList.size(), indexList.keysSize(), indexList.valuesSize()); + } + + public void delete() throws IOException { + try { + Files.delete(dataPath); + } finally { + Files.delete(indexPath); + } + } + } diff --git a/src/main/java/ru/vk/itmo/kovalevigor/SSTableManager.java b/src/main/java/ru/vk/itmo/kovalevigor/SSTableManager.java new file mode 100644 index 000000000..cb0784f45 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalevigor/SSTableManager.java @@ -0,0 +1,220 @@ +package ru.vk.itmo.kovalevigor; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.SortedMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class SSTableManager implements DaoFileGet>, AutoCloseable { + + public static final String SSTABLE_NAME = "sstable"; + + private final Path root; + private final Arena arena; + private final List ssTables; + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final Set deadSSTables; + private int totalSize; + + public SSTableManager(final Path root) throws IOException { + this.root = root; + this.arena = Arena.ofShared(); + this.ssTables = readTables(); + this.deadSSTables = new HashSet<>(); + this.totalSize = ssTables.size(); + } + + private List readTables() throws IOException { + final List tables = new ArrayList<>(); + SSTable table; + while ((table = readTable(getFormattedSSTableName(tables.size()))) != null) { + tables.add(table); + } + return new CopyOnWriteArrayList<>(tables.reversed()); + } + + private SSTable readTable(final String name) throws IOException { + return SSTable.create(root, name, arena); + } + + private static String getFormattedSSTableName(final int size) { + return SSTABLE_NAME + size; + } + + private synchronized String getNextSSTableName() { + return getFormattedSSTableName(totalSize++); + } + + public String write(SortedMap> map) throws IOException { + if (map.isEmpty()) { + return null; + } + + final String name = getNextSSTableName(); + SSTable.write(map, root, name); + return name; + } + + public void addSSTable(final String name) throws IOException { + lock.writeLock().lock(); + try { + ssTables.addFirst(SSTable.create(root, name, arena)); + } finally { + lock.writeLock().unlock(); + } + } + + public static Iterator> get( + Collection ssTables, + final MemorySegment from, + final MemorySegment to + ) throws IOException { + List>> iterators = new ArrayList<>(); + int i = 0; + for (final SSTable ssTable : ssTables) { + iterators.add(new MemEntryPriorityIterator(i, ssTable.get(from, to))); + i++; + } + return new MergeEntryIterator(iterators); + } + + @Override + public Iterator> get(final MemorySegment from, final MemorySegment to) throws IOException { + return get(ssTables, from, to); + } + + @Override + public Entry get(final MemorySegment key) throws IOException { + Entry value = null; + for (final SSTable ssTable : ssTables) { + value = ssTable.get(key); + if (value != null) { + if (value.value() == null) { + value = null; + } + break; + } + } + return value; + } + + private static SizeInfo getTotalInfoSize(final Collection ssTables) { + final SizeInfo result = new SizeInfo(); + for (SSTable ssTable: ssTables) { + result.add(ssTable.getSizeInfo()); + } + return result; + } + + public synchronized void compact() throws IOException { + final List compactTables = new ArrayList<>(ssTables); + if (compactTables.size() <= 1) { + return; + } + + Path tableTmpPath = null; + Path indexTmpPath = null; + final SizeInfo sizes = getTotalInfoSize(compactTables); + try { + tableTmpPath = Files.createTempFile(null, null); + indexTmpPath = Files.createTempFile(null, null); + final SizeInfo realSize = new SizeInfo(); + try (Arena tmpArena = Arena.ofConfined(); SStorageDumper dumper = new SStorageDumper( + sizes, + tableTmpPath, + indexTmpPath, + tmpArena + )) { + final Iterator> iterator = new MergeEntryIterator(List.of( + new MemEntryPriorityIterator(0, get(compactTables, null, null)) + )); + while (iterator.hasNext()) { + final Entry entry = iterator.next(); + if (entry.value() != null) { + dumper.writeEntry(entry); + realSize.size += 1; + realSize.keysSize += entry.key().byteSize(); + realSize.valuesSize += entry.value().byteSize(); + } + } + dumper.setKeysSize(realSize.keysSize); + dumper.setValuesSize(realSize.valuesSize); + } + + try (FileChannel dataChannel = FileChannel.open(tableTmpPath, StandardOpenOption.WRITE); + FileChannel indexChannel = FileChannel.open(indexTmpPath, StandardOpenOption.WRITE) + ) { + dataChannel.truncate(SStorageDumper.getSize(realSize.keysSize, realSize.valuesSize)); + indexChannel.truncate(SStorageDumper.getIndexSize(realSize.size)); + } + + final String sstableName = getNextSSTableName(); + + final Path dataPath = SSTable.getDataPath(root, sstableName); + final Path indexPath = SSTable.getIndexPath(root, sstableName); + + Files.copy(tableTmpPath, dataPath); + try { + Files.copy(indexTmpPath, indexPath); + } catch (IOException e) { + Files.deleteIfExists(dataPath); + throw e; + } + + lock.writeLock().lock(); + try { + ssTables.add(SSTable.create(root, sstableName, arena)); + ssTables.removeAll(compactTables); + deadSSTables.addAll(compactTables); + } finally { + lock.writeLock().unlock(); + } + } finally { + if (tableTmpPath != null) { + Files.deleteIfExists(tableTmpPath); + } + if (indexTmpPath != null) { + Files.deleteIfExists(indexTmpPath); + } + } + } + + @Override + public void close() throws IOException { + + if (!arena.scope().isAlive()) { + return; + } + arena.close(); + + if (!deadSSTables.isEmpty()) { + for (final SSTable ssTable : deadSSTables) { + ssTable.delete(); + } + + int size = 0; + for (final SSTable ssTable : ssTables.reversed()) { + ssTable.move(root, getFormattedSSTableName(size)); + size += 1; + } + } + deadSSTables.clear(); + + ssTables.clear(); + } +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/SStorageDumper.java b/src/main/java/ru/vk/itmo/kovalevigor/SStorageDumper.java new file mode 100644 index 000000000..20fdbf25d --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalevigor/SStorageDumper.java @@ -0,0 +1,105 @@ +package ru.vk.itmo.kovalevigor; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.file.Files; +import java.nio.file.Path; + +public class SStorageDumper extends Dumper { + + protected final IndexDumper indexDumper; + private final SegmentWriter keysWriter; + private final SegmentWriter valuesWriter; + + protected SStorageDumper( + final SizeInfo sizeInfo, + final Path storagePath, + final Path indexPath, + final Arena arena + ) throws IOException { + super(storagePath, getSize(sizeInfo.keysSize, sizeInfo.valuesSize), arena); + + keysWriter = new SegmentWriter( + Files.createTempFile(null, null), + sizeInfo.keysSize, + arena + ); + try { + valuesWriter = new SegmentWriter( + Files.createTempFile(null, null), + sizeInfo.valuesSize, + arena + ); + } catch (IOException e) { + Files.deleteIfExists(keysWriter.path); + throw e; + } + try { + indexDumper = new IndexDumper(sizeInfo.size, indexPath, arena); + } catch (IOException e) { + deleteSupportFiles(); + throw e; + } + + indexDumper.setKeysSize(sizeInfo.keysSize); + indexDumper.setValuesSize(sizeInfo.valuesSize); + } + + public static long getSize(final long keysSize, final long valuesSize) { + return keysSize + valuesSize; + } + + public static long getIndexSize(final long entryCount) { + return IndexDumper.getSize(entryCount); + } + + public void setKeysSize(final long keysSize) { + indexDumper.setKeysSize(keysSize); + } + + public void setValuesSize(final long valuesSize) { + indexDumper.setValuesSize(valuesSize); + } + + @Override + protected void writeHead() { + indexDumper.writeHead(); + } + + public void writeEntry(final Entry entry) { + final long keyOffset = keysWriter.offset; + keysWriter.writeMemorySegment(entry.key()); + + final long valueOffset; + final MemorySegment valueSegment = entry.value(); + if (valueSegment == null) { + valueOffset = -1; + } else { + valueOffset = valuesWriter.offset; + valuesWriter.writeMemorySegment(valueSegment); + } + indexDumper.writeEntry(keyOffset, valueOffset); + } + + private void deleteSupportFiles() throws IOException { + try { + Files.deleteIfExists(keysWriter.path); + } finally { + Files.deleteIfExists(valuesWriter.path); + } + } + + @Override + public void close() throws IOException { + writeHead(); + offset = writeMemorySegment(keysWriter.memorySegment, offset, indexDumper.keysSize); + offset = writeMemorySegment(valuesWriter.memorySegment, offset, indexDumper.valuesSize); + + indexDumper.close(); + + deleteSupportFiles(); + } +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/SegmentWriter.java b/src/main/java/ru/vk/itmo/kovalevigor/SegmentWriter.java new file mode 100644 index 000000000..84415df61 --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalevigor/SegmentWriter.java @@ -0,0 +1,44 @@ +package ru.vk.itmo.kovalevigor; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Path; + +public class SegmentWriter { + + public final MemorySegment memorySegment; + public final Path path; + public long offset; + + public SegmentWriter(final Path path, final long size, final Arena arena) throws IOException { + this.memorySegment = UtilsMemorySegment.mapWriteSegment(path, size, arena); + this.path = path; + this.offset = 0; + } + + protected static long writeLong(final MemorySegment memorySegment, final long offset, final long value) { + memorySegment.set(ValueLayout.JAVA_LONG, offset, value); + return offset + ValueLayout.JAVA_LONG.byteSize(); + } + + protected void writeLong(final long value) { + offset = writeLong(memorySegment, offset, value); + } + + protected long writeMemorySegment(final MemorySegment segment, final long offset, final long size) { + MemorySegment.copy( + segment, + 0, + memorySegment, + offset, + size + ); + return offset + size; + } + + protected void writeMemorySegment(final MemorySegment segment) { + offset = writeMemorySegment(segment, offset, segment.byteSize()); + } +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/ShiftedIterator.java b/src/main/java/ru/vk/itmo/kovalevigor/ShiftedIterator.java new file mode 100644 index 000000000..6b548efdc --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalevigor/ShiftedIterator.java @@ -0,0 +1,38 @@ +package ru.vk.itmo.kovalevigor; + +import java.util.Iterator; + +public class ShiftedIterator implements Iterator { + + private final Iterator iterator; + protected E value; + private boolean finished; + + public ShiftedIterator() { + this.iterator = null; + this.value = null; + this.finished = true; + } + + public ShiftedIterator(final Iterator iterator) { + this.iterator = iterator; + this.value = this.iterator.hasNext() ? this.iterator.next() : null; + this.finished = this.value == null; + } + + @Override + public boolean hasNext() { + return !finished; + } + + @Override + public E next() { + final E result = value; + if (iterator.hasNext()) { + value = iterator.next(); + } else { + finished = true; + } + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/SizeInfo.java b/src/main/java/ru/vk/itmo/kovalevigor/SizeInfo.java new file mode 100644 index 000000000..fd4562e8b --- /dev/null +++ b/src/main/java/ru/vk/itmo/kovalevigor/SizeInfo.java @@ -0,0 +1,32 @@ +package ru.vk.itmo.kovalevigor; + +public class SizeInfo { + + public long size; + public long keysSize; + public long valuesSize; + + public SizeInfo(final long size, final long keysSize, final long valuesSize) { + this.size = size; + this.keysSize = keysSize; + this.valuesSize = valuesSize; + } + + public SizeInfo(final long size, final long keysSize) { + this(size, keysSize, 0); + } + + public SizeInfo(final long size) { + this(size, 0); + } + + public SizeInfo() { + this(0); + } + + public void add(final SizeInfo sizeInfo) { + size += sizeInfo.size; + keysSize += sizeInfo.keysSize; + valuesSize += sizeInfo.valuesSize; + } +} diff --git a/src/main/java/ru/vk/itmo/kovalevigor/UtilsMemorySegment.java b/src/main/java/ru/vk/itmo/kovalevigor/UtilsMemorySegment.java index 6bdf5f113..bcc1f84e6 100644 --- a/src/main/java/ru/vk/itmo/kovalevigor/UtilsMemorySegment.java +++ b/src/main/java/ru/vk/itmo/kovalevigor/UtilsMemorySegment.java @@ -1,7 +1,15 @@ package ru.vk.itmo.kovalevigor; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; public final class UtilsMemorySegment { @@ -28,4 +36,55 @@ public static int compare(final MemorySegment lhs, final MemorySegment rhs) { } return Byte.compare(getByte(lhs, mismatch), getByte(rhs, mismatch)); } + + public static int compareEntry(final Entry lhs, final Entry rhs) { + return compare(lhs.key(), rhs.key()); + } + + public static MemorySegment mapSegment( + final Path path, + final long fileSize, + final Arena arena, + final FileChannel.MapMode mapMode, + final StandardOpenOption... options + ) throws IOException { + try (FileChannel writerChannel = FileChannel.open(path, options)) { + return writerChannel.map( + mapMode, + 0, + fileSize, + arena + ); + } + } + + public static MemorySegment mapReadSegment( + final Path path, + final Arena arena + ) throws IOException { + return mapSegment( + path, + Files.size(path), + arena, + FileChannel.MapMode.READ_ONLY, + StandardOpenOption.READ + ); + } + + public static MemorySegment mapWriteSegment( + final Path path, + final long fileSize, + final Arena arena + ) throws IOException { + return mapSegment( + path, + fileSize, + arena, + FileChannel.MapMode.READ_WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.READ, + StandardOpenOption.WRITE + ); + } } diff --git a/src/main/java/ru/vk/itmo/novichkovandrew/Candidate.java b/src/main/java/ru/vk/itmo/novichkovandrew/Candidate.java new file mode 100644 index 000000000..543c282d2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/novichkovandrew/Candidate.java @@ -0,0 +1,49 @@ +package ru.vk.itmo.novichkovandrew; + +import ru.vk.itmo.Entry; +import ru.vk.itmo.novichkovandrew.iterator.TableIterator; + +import java.util.Comparator; + +public class Candidate implements Comparable> { + private Entry entry; + private final TableIterator iterator; + private final Comparator comparator; + + public Candidate(TableIterator iterator, Comparator comparator) { + this.iterator = iterator; + this.comparator = comparator; + this.entry = iterator.hasNext() ? iterator.next() : null; + } + + public Entry entry() { + return entry; + } + + public void update() { + this.entry = iterator.hasNext() ? iterator.next() : null; + } + + @Override + public int compareTo(Candidate candidate) { + if (candidate.entry == null) { + return -1; + } + if (entry == null) { + return 1; + } + int memoryComparison = comparator.compare(this.entry.key(), candidate.entry.key()); + if (memoryComparison == 0) { + return Integer.compare(candidate.iterator.getTableNumber(), this.iterator.getTableNumber()); + } + return memoryComparison; + } + + public boolean nonLast() { + return this.entry != null; + } + + public TableIterator iterator() { + return iterator; + } +} diff --git a/src/main/java/ru/vk/itmo/novichkovandrew/TablesOrganizer.java b/src/main/java/ru/vk/itmo/novichkovandrew/TablesOrganizer.java new file mode 100644 index 000000000..11d6e0a70 --- /dev/null +++ b/src/main/java/ru/vk/itmo/novichkovandrew/TablesOrganizer.java @@ -0,0 +1,112 @@ +package ru.vk.itmo.novichkovandrew; + +import ru.vk.itmo.Entry; +import ru.vk.itmo.novichkovandrew.iterator.MergeIterator; +import ru.vk.itmo.novichkovandrew.iterator.TableIterator; +import ru.vk.itmo.novichkovandrew.table.AbstractTable; +import ru.vk.itmo.novichkovandrew.table.MemTable; +import ru.vk.itmo.novichkovandrew.table.SSTable; + +import java.io.Closeable; +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static ru.vk.itmo.novichkovandrew.Utils.copyToSegment; + +/** + * Class, that can organize different sst tables. + */ +public class TablesOrganizer implements Closeable { + /** + * Path associated with SSTables. + */ + private final Path path; + private final List tables; + private final MemTable memTable; + + private final StandardOpenOption[] openOptions = new StandardOpenOption[]{ + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.READ, + StandardOpenOption.WRITE, + }; + + public TablesOrganizer(Path path, MemTable memTable) { + this.memTable = memTable; + this.path = path; + this.tables = createTablesList(); + } + + public Iterator> mergeIterator(MemorySegment from, boolean fromInclusive, + MemorySegment to, boolean toInclusive) { + return new MergeIterator( + createIteratorsList(from, fromInclusive, to, toInclusive), + memTable.comparator() + ); + } + + public void flushMemTable() throws IOException { + Path sstPath = sstTablePath(tables.size()); + try (FileChannel sst = FileChannel.open(sstPath, openOptions); + Arena arena = Arena.ofConfined() + ) { + long metaSize = memTable.getMetaDataSize(); + long sstOffset = 0L; + long indexOffset = Utils.writeLong(sst, 0L, memTable.size()); + MemorySegment sstMap = sst.map(FileChannel.MapMode.READ_WRITE, metaSize, memTable.byteSize(), arena); + for (Entry entry : memTable) { + long keyOffset = sstOffset + metaSize; + long valueOffset = keyOffset + entry.key().byteSize(); + valueOffset *= memTable.isTombstone(entry.key()) ? -1 : 1; + indexOffset = writePosToFile(sst, indexOffset, keyOffset, valueOffset); + sstOffset = copyToSegment(sstMap, entry.key(), sstOffset); + sstOffset = copyToSegment(sstMap, entry.value(), sstOffset); + } + writePosToFile(sst, indexOffset, sstOffset + metaSize, 0L); + } + } + + @Override + public void close() throws IOException { + for (var table : tables) { + table.close(); + } + tables.clear(); + } + + private List createTablesList() { + Stream memStream = Stream.of(memTable); + Stream sstStream = IntStream + .rangeClosed(1, Utils.filesCount(path)) + .mapToObj(n -> new SSTable(sstTablePath(n), n)); + return Stream.concat(memStream, sstStream).collect(Collectors.toList()); + } + + private List> createIteratorsList(MemorySegment from, boolean fromInclusive, + MemorySegment to, boolean toInclusive) { + return tables.stream() + .map(t -> t.tableIterator(from, fromInclusive, to, toInclusive)) + .collect(Collectors.toList()); + } + + private synchronized Path sstTablePath(long suffix) { + String fileName = String.format("data-%s.txt", suffix); + return path.resolve(Path.of(fileName)); + } + + private long writePosToFile(FileChannel channel, long rawOffset, long keyOff, long valOff) { + long offset = rawOffset; + offset = Utils.writeLong(channel, offset, keyOff); + offset = Utils.writeLong(channel, offset, valOff); + return offset; + } +} diff --git a/src/main/java/ru/vk/itmo/novichkovandrew/Utils.java b/src/main/java/ru/vk/itmo/novichkovandrew/Utils.java index fbd1d228f..40efe2e2b 100644 --- a/src/main/java/ru/vk/itmo/novichkovandrew/Utils.java +++ b/src/main/java/ru/vk/itmo/novichkovandrew/Utils.java @@ -1,14 +1,39 @@ package ru.vk.itmo.novichkovandrew; +import ru.vk.itmo.novichkovandrew.exceptions.ReadFailureException; +import ru.vk.itmo.novichkovandrew.exceptions.WriteFailureException; + import java.io.IOException; import java.lang.foreign.MemorySegment; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; public final class Utils { + public static final MemorySegment LEFT = MemorySegment.NULL; + public static final MemorySegment RIGHT = MemorySegment.NULL; + + /** + * No instances. + */ private Utils() { } + /** + * Executes amount of files by path. + * Stream does not create an explicit list of files + * but simply counts the number of files in the directory. + */ + public static int filesCount(Path path) { + try (Stream files = Files.list(path)) { + return Math.toIntExact(files.count()); + } catch (IOException ex) { + return 0; + } + } + /** * Copy from one MemorySegment to another and return new offset of two segments. */ @@ -22,23 +47,31 @@ public static long copyToSegment(MemorySegment to, MemorySegment from, long offs * Writes long value into file opened in FileChannel. * Returns new offset of fileChannel. */ - public static long writeLong(FileChannel channel, long offset, long value) throws IOException { - channel.position(offset); - ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); - buffer.putLong(value); - buffer.flip(); - int bytes = channel.write(buffer); - return offset + bytes; + public static long writeLong(FileChannel channel, long offset, long value) { + try { + channel.position(offset); + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(value); + buffer.flip(); + int bytes = channel.write(buffer); + return offset + bytes; + } catch (IOException ex) { + throw new WriteFailureException("Failed to write long" + value + "value from position" + offset, ex); + } } /** * Read long value from file opened in FileChannel. */ - public static long readLong(FileChannel channel, long offset) throws IOException { - channel.position(offset); - ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); - channel.read(buffer); - buffer.flip(); - return buffer.getLong(); + public static long readLong(FileChannel channel, long offset) { + try { + channel.position(offset); + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + channel.read(buffer); + buffer.flip(); + return buffer.getLong(); + } catch (IOException ex) { + throw new ReadFailureException("Failed to write long value from position" + offset, ex); + } } } diff --git a/src/main/java/ru/vk/itmo/novichkovandrew/dao/InMemoryDao.java b/src/main/java/ru/vk/itmo/novichkovandrew/dao/InMemoryDao.java index 6b6b683be..4c4013acb 100644 --- a/src/main/java/ru/vk/itmo/novichkovandrew/dao/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/novichkovandrew/dao/InMemoryDao.java @@ -2,68 +2,36 @@ import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; +import ru.vk.itmo.novichkovandrew.table.MemTable; +import java.io.IOException; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.util.Comparator; import java.util.Iterator; -import java.util.NavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.atomic.AtomicLong; public class InMemoryDao implements Dao> { - protected final ConcurrentSkipListMap> entriesMap; - protected final Comparator comparator = (first, second) -> { - if (first == null || second == null) return -1; - long missIndex = first.mismatch(second); - if (missIndex == first.byteSize()) { - return -1; - } - if (missIndex == second.byteSize()) { - return 1; - } - return missIndex == -1 ? 0 : Byte.compare( - first.getAtIndex(ValueLayout.JAVA_BYTE, missIndex), - second.getAtIndex(ValueLayout.JAVA_BYTE, missIndex) - ); - }; - - protected final AtomicLong tableByteSize = new AtomicLong(0); + protected final MemTable memTable; public InMemoryDao() { - this.entriesMap = new ConcurrentSkipListMap<>(comparator); - } - - private NavigableMap> getSubMap(MemorySegment from, MemorySegment to) { - if (from != null && to != null) { - return entriesMap.subMap(from, to); - } - if (from != null) { - return entriesMap.tailMap(from, true); - } - if (to != null) { - return entriesMap.headMap(to, false); - } - return entriesMap; + this.memTable = new MemTable(); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { - return getSubMap(from, to).values().iterator(); + return memTable.tableIterator(from, true, to, false); } @Override public Entry get(MemorySegment key) { - return entriesMap.get(key); + return memTable.tableIterator(key, true, key, true).next(); } @Override public void upsert(Entry entry) { - tableByteSize.addAndGet(entry.key().byteSize() + entry.value().byteSize()); - entriesMap.put(entry.key(), entry); + memTable.upsert(entry); } - protected long getMetaDataSize() { - return 2L * (entriesMap.size() + 1) * Long.BYTES + Long.BYTES; + @Override + public void close() throws IOException { + memTable.close(); } } diff --git a/src/main/java/ru/vk/itmo/novichkovandrew/dao/PersistentDao.java b/src/main/java/ru/vk/itmo/novichkovandrew/dao/PersistentDao.java index dbbf05e17..76689f7f9 100644 --- a/src/main/java/ru/vk/itmo/novichkovandrew/dao/PersistentDao.java +++ b/src/main/java/ru/vk/itmo/novichkovandrew/dao/PersistentDao.java @@ -1,127 +1,58 @@ package ru.vk.itmo.novichkovandrew.dao; -import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Entry; +import ru.vk.itmo.novichkovandrew.TablesOrganizer; import ru.vk.itmo.novichkovandrew.Utils; -import ru.vk.itmo.novichkovandrew.exceptions.FileChannelException; import java.io.IOException; -import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.nio.channels.FileChannel; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; +import java.util.Iterator; public class PersistentDao extends InMemoryDao { /** - * File with SSTable path. + * Organizer class, that controls all sst tables. */ - private final Path path; - - private final Arena arena; - private final StandardOpenOption[] openOptions = new StandardOpenOption[]{ - StandardOpenOption.WRITE, - StandardOpenOption.READ, - StandardOpenOption.CREATE - }; + private final TablesOrganizer organizer; public PersistentDao(Path path) { - this.path = path.resolve("data.txt"); - this.arena = Arena.ofConfined(); + this.organizer = new TablesOrganizer(path, memTable); } @Override public void flush() throws IOException { - try (FileChannel sst = FileChannel.open(path, openOptions)) { - long metaSize = super.getMetaDataSize(); - long sstOffset = 0L; - long indexOffset = Utils.writeLong(sst, 0L, entriesMap.size()); - MemorySegment sstMap = sst.map(FileChannel.MapMode.READ_WRITE, metaSize, tableByteSize.get(), arena); - for (Entry entry : entriesMap.values()) { - long keyOffset = sstOffset + metaSize; - long valueOffset = keyOffset + entry.key().byteSize(); - indexOffset = writePosToFile(sst, indexOffset, keyOffset, valueOffset); - sstOffset = Utils.copyToSegment(sstMap, entry.key(), sstOffset); - sstOffset = Utils.copyToSegment(sstMap, entry.value(), sstOffset); - } - writePosToFile(sst, indexOffset, sstOffset + metaSize, 0L); - } - } - - private long writePosToFile(FileChannel channel, long rawOffset, long keyOff, long valOff) throws IOException { - long offset = rawOffset; - offset = Utils.writeLong(channel, offset, keyOff); - offset = Utils.writeLong(channel, offset, valOff); - return offset; + organizer.flushMemTable(); } @Override public void close() throws IOException { - if (!entriesMap.isEmpty()) flush(); - if (arena.scope().isAlive()) { - arena.close(); - } + if (memTable.size() != 0) flush(); + organizer.close(); + super.close(); } @Override - public Entry get(MemorySegment key) { - Entry entry = super.get(key); - if (entry != null) { - return entry; - } - if (Files.notExists(path)) { - return null; - } - try (FileChannel sstChannel = FileChannel.open(path, StandardOpenOption.READ)) { - return binarySearch(sstChannel, key); - } catch (IOException e) { - throw new FileChannelException("Couldn't open file " + path, e); - } - } - - private Entry binarySearch(FileChannel sst, MemorySegment key) throws IOException { - int l = 0; - int r = Math.toIntExact(Utils.readLong(sst, 0L)); - while (l < r) { - int mid = l + (r - l) / 2; - MemorySegment middle = getKeyByIndex(sst, mid); - if (comparator.compare(key, middle) <= 0) { - r = mid; - } else { - l = mid + 1; - } - } - var resultKey = getKeyByIndex(sst, l); - if (comparator.compare(key, resultKey) == 0) { - return new BaseEntry<>(resultKey, getValueByIndex(sst, l)); - } - return null; + public Iterator> get(MemorySegment from, MemorySegment to) { + return organizer.mergeIterator(from, true, to, false); } - private MemorySegment getKeyByIndex(FileChannel sst, int index) throws IOException { - long keyOffset = getKeyOffset(sst, index); - long valueOffset = getValueOffset(sst, index); - if (valueOffset == 0) return null; - return sst.map(FileChannel.MapMode.READ_ONLY, keyOffset, valueOffset - keyOffset, arena); + @Override + public Entry get(MemorySegment key) { + return organizer.mergeIterator(key, true, key, true).next(); } - private MemorySegment getValueByIndex(FileChannel sst, int index) throws IOException { - long valueOffset = getValueOffset(sst, index); - if (valueOffset < 0) { - return null; - } - long nextKeyOffset = getKeyOffset(sst, index + 1); - return sst.map(FileChannel.MapMode.READ_ONLY, valueOffset, nextKeyOffset - valueOffset, arena); + @Override + public Iterator> allFrom(MemorySegment from) { + return get(from, Utils.RIGHT); } - private long getKeyOffset(FileChannel sst, int index) throws IOException { - long rawOffset = (2L * index + 1) * (long) Long.BYTES; - return Utils.readLong(sst, rawOffset); + @Override + public Iterator> allTo(MemorySegment to) { + return get(Utils.LEFT, to); } - private long getValueOffset(FileChannel sst, int index) throws IOException { - long rawOffset = (2L * index + 2) * (long) Long.BYTES; - return Utils.readLong(sst, rawOffset); + @Override + public Iterator> all() { + return get(Utils.LEFT, Utils.RIGHT); } } diff --git a/src/main/java/ru/vk/itmo/novichkovandrew/exceptions/ReadFailureException.java b/src/main/java/ru/vk/itmo/novichkovandrew/exceptions/ReadFailureException.java new file mode 100644 index 000000000..223425a49 --- /dev/null +++ b/src/main/java/ru/vk/itmo/novichkovandrew/exceptions/ReadFailureException.java @@ -0,0 +1,11 @@ +package ru.vk.itmo.novichkovandrew.exceptions; + +public class ReadFailureException extends RuntimeException { + public ReadFailureException(String message) { + super(message); + } + + public ReadFailureException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/ru/vk/itmo/novichkovandrew/exceptions/WriteFailureException.java b/src/main/java/ru/vk/itmo/novichkovandrew/exceptions/WriteFailureException.java new file mode 100644 index 000000000..33b84c2ab --- /dev/null +++ b/src/main/java/ru/vk/itmo/novichkovandrew/exceptions/WriteFailureException.java @@ -0,0 +1,11 @@ +package ru.vk.itmo.novichkovandrew.exceptions; + +public class WriteFailureException extends RuntimeException { + public WriteFailureException(String message) { + super(message); + } + + public WriteFailureException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/ru/vk/itmo/novichkovandrew/iterator/MergeIterator.java b/src/main/java/ru/vk/itmo/novichkovandrew/iterator/MergeIterator.java new file mode 100644 index 000000000..bc8e2d0e5 --- /dev/null +++ b/src/main/java/ru/vk/itmo/novichkovandrew/iterator/MergeIterator.java @@ -0,0 +1,66 @@ +package ru.vk.itmo.novichkovandrew.iterator; + +import ru.vk.itmo.Entry; +import ru.vk.itmo.novichkovandrew.Candidate; + +import java.lang.foreign.MemorySegment; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.PriorityBlockingQueue; +import java.util.stream.Collectors; + +public class MergeIterator implements Iterator> { + private final Queue> queue; + private Entry minEntry; + private MemorySegment lowerBoundKey; + private final Comparator memoryComparator; + + public MergeIterator(List> iterators, Comparator memoryComparator) { + this.memoryComparator = memoryComparator; + this.queue = iterators.stream() + .map(it -> new Candidate<>(it, memoryComparator)) + .filter(Candidate::nonLast) + .collect(Collectors.toCollection(PriorityBlockingQueue::new)); + minEntry = getMinEntry(); + } + + private Entry getMinEntry() { + if (queue.isEmpty()) { + return null; + } + while (!queue.isEmpty()) { + var candidate = queue.poll(); + if (lowerBoundKey == null || memoryComparator.compare(lowerBoundKey, candidate.entry().key()) < 0) { + lowerBoundKey = candidate.entry().key(); + var entry = candidate.entry(); + candidate.update(); + if (candidate.nonLast()) { + queue.add(candidate); + } + if (entry.value() != null) { // Tombstone check. + return entry; + } + } else { + candidate.update(); + if (candidate.nonLast()) { + queue.add(candidate); + } + } + } + return null; + } + + @Override + public boolean hasNext() { + return this.minEntry != null; + } + + @Override + public Entry next() { + var entry = this.minEntry; + this.minEntry = getMinEntry(); + return entry; + } +} diff --git a/src/main/java/ru/vk/itmo/novichkovandrew/iterator/TableIterator.java b/src/main/java/ru/vk/itmo/novichkovandrew/iterator/TableIterator.java new file mode 100644 index 000000000..711740ee6 --- /dev/null +++ b/src/main/java/ru/vk/itmo/novichkovandrew/iterator/TableIterator.java @@ -0,0 +1,9 @@ +package ru.vk.itmo.novichkovandrew.iterator; + +import ru.vk.itmo.Entry; + +import java.util.Iterator; + +public interface TableIterator extends Iterator> { + int getTableNumber(); +} diff --git a/src/main/java/ru/vk/itmo/novichkovandrew/table/AbstractTable.java b/src/main/java/ru/vk/itmo/novichkovandrew/table/AbstractTable.java new file mode 100644 index 000000000..7a136682b --- /dev/null +++ b/src/main/java/ru/vk/itmo/novichkovandrew/table/AbstractTable.java @@ -0,0 +1,36 @@ +package ru.vk.itmo.novichkovandrew.table; + +import ru.vk.itmo.novichkovandrew.Utils; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Comparator; +import java.util.Objects; + +public abstract class AbstractTable implements Table { + protected final Comparator comparator = (first, second) -> { + Objects.requireNonNull(first, "First segment is null in memory comparing"); + Objects.requireNonNull(second, "Second segment is null in memory comparing"); + if (first == Utils.LEFT || second == Utils.RIGHT) { + return -1; + } + if (first == Utils.RIGHT || second == Utils.LEFT) { + return 1; + } + long missIndex = first.mismatch(second); + if (missIndex == first.byteSize()) { + return -1; + } + if (missIndex == second.byteSize()) { + return 1; + } + return missIndex == -1 ? 0 : Byte.compare( + first.getAtIndex(ValueLayout.JAVA_BYTE, missIndex), + second.getAtIndex(ValueLayout.JAVA_BYTE, missIndex) + ); + }; + + public Comparator comparator() { + return comparator; + } +} diff --git a/src/main/java/ru/vk/itmo/novichkovandrew/table/MemTable.java b/src/main/java/ru/vk/itmo/novichkovandrew/table/MemTable.java new file mode 100644 index 000000000..dbf4e30a7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/novichkovandrew/table/MemTable.java @@ -0,0 +1,100 @@ +package ru.vk.itmo.novichkovandrew.table; + +import ru.vk.itmo.Entry; +import ru.vk.itmo.novichkovandrew.iterator.TableIterator; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicLong; + +public class MemTable extends AbstractTable implements Iterable> { + + private final ConcurrentSkipListMap> entriesMap; + private final AtomicLong byteSize; + + public MemTable() { + this.entriesMap = new ConcurrentSkipListMap<>(comparator); + this.byteSize = new AtomicLong(); + } + + public void upsert(Entry entry) { + byteSize.addAndGet(entry.key().byteSize() + (entry.value() == null ? 0 : entry.value().byteSize())); + entriesMap.put(entry.key(), entry); + } + + @Override + public int size() { + return entriesMap.size(); + } + + @Override + public TableIterator tableIterator(MemorySegment from, boolean fromInclusive, + MemorySegment to, boolean toInclusive) { + + return new TableIterator<>() { + final Iterator> it = getSubMap(from, fromInclusive, to, toInclusive) + .values().iterator(); + + @Override + public int getTableNumber() { + return Integer.MAX_VALUE; + } + + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public Entry next() { + return it.next(); + } + }; + } + + public long byteSize() { + return this.byteSize.get(); + } + + /** + * Return metadata length of SSTable file. + * Metadata contains amount of entries in sst, offsets and size of keys. + * It has the following format: size keyOff1:valOff1 keyOff2:valOff2 ... + * keyOff_n:valOff_n keyOff_n+1:valOff_n+1 + * without any : and spaces. + */ + public long getMetaDataSize() { + return 2L * (entriesMap.size() + 1) * Long.BYTES + Long.BYTES; + } + + private NavigableMap> getSubMap(MemorySegment from, boolean fromInclusive, + MemorySegment to, boolean toInclusive) { + if (from != null && to != null) { + return entriesMap.subMap(from, fromInclusive, to, toInclusive); + } + if (from != null) { + return entriesMap.tailMap(from, fromInclusive); + } + if (to != null) { + return entriesMap.headMap(to, toInclusive); + } + return entriesMap; + } + + @Override + public void close() { + entriesMap.clear(); + byteSize.set(0); + } + + public boolean isTombstone(MemorySegment key) { + return entriesMap.containsKey(key) && entriesMap.get(key).value() == null; + } + + @Override + public Iterator> iterator() { + return entriesMap.values().iterator(); + } +} diff --git a/src/main/java/ru/vk/itmo/novichkovandrew/table/SSTable.java b/src/main/java/ru/vk/itmo/novichkovandrew/table/SSTable.java new file mode 100644 index 000000000..2cb5b4bfc --- /dev/null +++ b/src/main/java/ru/vk/itmo/novichkovandrew/table/SSTable.java @@ -0,0 +1,135 @@ +package ru.vk.itmo.novichkovandrew.table; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; +import ru.vk.itmo.novichkovandrew.Utils; +import ru.vk.itmo.novichkovandrew.exceptions.FileChannelException; +import ru.vk.itmo.novichkovandrew.iterator.TableIterator; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Objects; + +public class SSTable extends AbstractTable { + + private final FileChannel sstChannel; + /** + * Constable size of SSTable. + */ + private final int size; + + /** + * Unique number of this SST. + */ + private final int sstNumber; + private final Arena arena; + + public SSTable(Path path, int sstNumber) { + try { + this.sstChannel = FileChannel.open(path, StandardOpenOption.READ); + this.size = Math.toIntExact(Utils.readLong(sstChannel, 0L)); + this.sstNumber = sstNumber; + this.arena = Arena.ofShared(); + } catch (IOException ex) { + throw new FileChannelException("Couldn't create FileChannel by path" + path, ex); + } + } + + @Override + public void close() { + try { + this.sstChannel.close(); + if (arena.scope().isAlive()) { + arena.close(); + } + } catch (IOException e) { + throw new FileChannelException("Couldn't close file channel " + sstChannel, e); + } + } + + @Override + public int size() { + return size; + } + + @Override + public TableIterator tableIterator(MemorySegment from, boolean fromInclusive, + MemorySegment to, boolean toInclusive) { + return new TableIterator<>() { + int start = (from == null ? 0 : binarySearch(from)) + Boolean.compare(!fromInclusive, false); + final int end = (to == null ? size : binarySearch(to)) + Boolean.compare(toInclusive, true); + + @Override + public int getTableNumber() { + return sstNumber; + } + + @Override + public boolean hasNext() { + return start < size && start <= end; + } + + @Override + public Entry next() { + return new BaseEntry<>(getKeyByIndex(start), getValueByIndex(start++)); + } + }; + } + + private MemorySegment getKeyByIndex(int index) { + Objects.checkIndex(index, size); + long keyOffset = getKeyOffset(index); + long valueOffset = Math.abs(getValueOffset(index)); + return copyToArena(keyOffset, valueOffset); + } + + private MemorySegment getValueByIndex(int index) { + Objects.checkIndex(index, size); + long valueOffset = getValueOffset(index); + if (valueOffset < 0) { + return null; + } + long nextKeyOffset = getKeyOffset(index + 1); + return copyToArena(valueOffset, nextKeyOffset); + } + + private MemorySegment copyToArena(long valOffset, long nextOffset) { + try (Arena mapArena = Arena.ofShared()) { + var mappedMem = sstChannel.map(FileChannel.MapMode.READ_ONLY, valOffset, nextOffset - valOffset, mapArena); + var nativeMem = arena.allocate(mappedMem.byteSize()); + Utils.copyToSegment(nativeMem, mappedMem, 0); + return nativeMem; + } catch (IOException ex) { + throw new FileChannelException("Couldn't map file from channel " + sstChannel, ex); + } + } + + private int binarySearch(MemorySegment key) { + int l = 0; + int r = size - 1; + while (l <= r) { + int mid = l + (r - l) / 2; + MemorySegment middle = getKeyByIndex(mid); + if (comparator.compare(key, middle) <= 0) { + r = mid - 1; + } else { + l = mid + 1; + } + } + return l; + } + + private long getKeyOffset(int index) { + long rawOffset = (2L * index + 1) * (long) Long.BYTES; + return Utils.readLong(sstChannel, rawOffset); + } + + private long getValueOffset(int index) { + long rawOffset = (2L * index + 2) * (long) Long.BYTES; + return Utils.readLong(sstChannel, rawOffset); + } +} diff --git a/src/main/java/ru/vk/itmo/novichkovandrew/table/Table.java b/src/main/java/ru/vk/itmo/novichkovandrew/table/Table.java new file mode 100644 index 000000000..87cfe6164 --- /dev/null +++ b/src/main/java/ru/vk/itmo/novichkovandrew/table/Table.java @@ -0,0 +1,13 @@ +package ru.vk.itmo.novichkovandrew.table; + +import ru.vk.itmo.novichkovandrew.iterator.TableIterator; + +import java.io.Closeable; +import java.io.IOException; + +public interface Table extends Closeable { + + int size() throws IOException; + + TableIterator tableIterator(K from, boolean fromInclusive, K to, boolean toInclusive); +} diff --git a/src/main/java/ru/vk/itmo/osipovdaniil/DiskStorage.java b/src/main/java/ru/vk/itmo/osipovdaniil/DiskStorage.java new file mode 100644 index 000000000..0eff1e2c5 --- /dev/null +++ b/src/main/java/ru/vk/itmo/osipovdaniil/DiskStorage.java @@ -0,0 +1,75 @@ +package ru.vk.itmo.osipovdaniil; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +public class DiskStorage { + + private final List segmentList; + + public DiskStorage(final List segmentList) { + this.segmentList = segmentList; + } + + static Iterator> iterator(final MemorySegment page, + final MemorySegment from, + final MemorySegment to) { + long recordIndexFrom = from == null + ? 0 : DiskStorageUtilsSimple.normalize(DiskStorageUtils.indexOf(page, from)); + long recordIndexTo = to == null + ? DiskStorageUtilsSimple.recordsCount(page) : + DiskStorageUtilsSimple.normalize(DiskStorageUtils.indexOf(page, to)); + final long recordsCount = DiskStorageUtilsSimple.recordsCount(page); + return new Iterator<>() { + long index = recordIndexFrom; + + @Override + public boolean hasNext() { + return index < recordIndexTo; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + MemorySegment key = DiskStorageUtilsSimple.slice(page, + DiskStorageUtilsSimple.startOfKey(page, index), + DiskStorageUtilsSimple.endOfKey(page, index)); + long startOfValue = DiskStorageUtilsSimple.startOfValue(page, index); + MemorySegment value = + startOfValue < 0 + ? null + : DiskStorageUtilsSimple.slice(page, startOfValue, + DiskStorageUtilsSimple.endOfValue(page, index, recordsCount)); + index++; + return new BaseEntry<>(key, value); + } + }; + } + + public Iterator> range( + final Iterator> firstIterator, + final MemorySegment from, + final MemorySegment to) { + final List>> iterators = new ArrayList<>(segmentList.size() + 1); + for (MemorySegment memorySegment : segmentList) { + iterators.add(iterator(memorySegment, from, to)); + } + iterators.add(firstIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, Utils::compareMemorySegments)) { + @Override + protected boolean skip(final Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } +} diff --git a/src/main/java/ru/vk/itmo/osipovdaniil/DiskStorageUtils.java b/src/main/java/ru/vk/itmo/osipovdaniil/DiskStorageUtils.java new file mode 100644 index 000000000..ee341ec34 --- /dev/null +++ b/src/main/java/ru/vk/itmo/osipovdaniil/DiskStorageUtils.java @@ -0,0 +1,229 @@ +package ru.vk.itmo.osipovdaniil; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +public final class DiskStorageUtils { + + public static final String SSTABLE_PREFIX = "sstable_"; + + private DiskStorageUtils() { + } + + public static List loadOrRecover(final Path storagePath, final Arena arena) throws IOException { + if (Files.exists(compactionFile(storagePath))) { + finalizeCompaction(storagePath); + } + final Path indexTmp = DiskStorageUtilsSimple.getIndexTmpPath(storagePath); + final Path indexFile = DiskStorageUtilsSimple.getIndexPath(storagePath); + if (!Files.exists(indexFile)) { + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + Files.createFile(indexFile); + } + } + final List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + final List result = new ArrayList<>(existedFiles.size()); + for (String fileName : existedFiles) { + final Path file = storagePath.resolve(fileName); + try (FileChannel fileChannel = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(file), + arena); + result.add(fileSegment); + } + } + return result; + } + + static long indexOf(final MemorySegment segment, final MemorySegment key) { + final long recordsCount = DiskStorageUtilsSimple.recordsCount(segment); + + long left = 0; + long right = recordsCount - 1; + while (left <= right) { + long mid = (left + right) >>> 1; + long startOfKey = DiskStorageUtilsSimple.startOfKey(segment, mid); + long endOfKey = DiskStorageUtilsSimple.endOfKey(segment, mid); + long mismatch = MemorySegment.mismatch(segment, startOfKey, endOfKey, key, 0, key.byteSize()); + if (mismatch == -1) { + return mid; + } + if (mismatch == key.byteSize()) { + right = mid - 1; + continue; + } + if (mismatch == endOfKey - startOfKey) { + left = mid + 1; + continue; + } + int b1 = Byte.toUnsignedInt(segment.get(ValueLayout.JAVA_BYTE, startOfKey + mismatch)); + int b2 = Byte.toUnsignedInt(key.get(ValueLayout.JAVA_BYTE, mismatch)); + if (b1 > b2) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + return DiskStorageUtilsSimple.tombstone(left); + } + + public static void save(final Path storagePath, final Iterable> iterable) + throws IOException { + final Path indexTmp = DiskStorageUtilsSimple.getIndexTmpPath(storagePath); + final Path indexFile = DiskStorageUtilsSimple.getIndexPath(storagePath); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + final List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + final String newFileName = SSTABLE_PREFIX + existedFiles.size(); + final Path newFilePath = storagePath.resolve(newFileName); + dump(iterable, newFilePath); + final List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + list.add(newFileName); + Files.write(indexTmp, + list, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + Files.deleteIfExists(indexFile); + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE); + } + + private static void dump(final Iterable> iterable, + final Path newFilePath) throws IOException { + long dataSize = 0; + long count = 0; + for (Entry entry : iterable) { + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + if (value != null) { + dataSize += value.byteSize(); + } + count++; + } + final long indexSize = count * 2 * Long.BYTES; + try (FileChannel fileChannel = FileChannel.open( + newFilePath, + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE); + Arena writeArena = Arena.ofConfined()) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + indexSize + dataSize, + writeArena + ); + // index: + // |key0_Start|value0_Start|key1_Start|value1_Start|key2_Start|value2_Start|... + // key0_Start = data start = end of index + long dataOffset = indexSize; + int indexOffset = 0; + for (Entry entry : iterable) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += entry.key().byteSize(); + indexOffset += Long.BYTES; + + MemorySegment value = entry.value(); + if (value == null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, + DiskStorageUtilsSimple.tombstone(dataOffset)); + } else { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += value.byteSize(); + } + indexOffset += Long.BYTES; + } + // data: + // |key0|value0|key1|value1|... + dataOffset = indexSize; + for (Entry entry : iterable) { + MemorySegment key = entry.key(); + MemorySegment.copy(key, 0, fileSegment, dataOffset, key.byteSize()); + dataOffset += key.byteSize(); + + MemorySegment value = entry.value(); + if (value != null) { + MemorySegment.copy(value, 0, fileSegment, dataOffset, value.byteSize()); + dataOffset += value.byteSize(); + } + } + } + } + + public static void compact(final Path storagePath, final Iterable> iterable) + throws IOException { + final String newFileName = "compaction.tmp"; + final Path compactionTmpFile = storagePath.resolve(newFileName); + final Path newFilePath = storagePath.resolve(newFileName); + dump(iterable, newFilePath); + Files.move(compactionTmpFile, + storagePath.resolve("compaction"), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + finalizeCompaction(storagePath); + } + + private static void finalizeCompaction(Path storagePath) throws IOException { + final Path compactionFile = compactionFile(storagePath); + try (Stream stream = Files.find(storagePath, 1, + (path, attr) -> path.getFileName().toString().startsWith(SSTABLE_PREFIX))) { + stream.forEach(p -> { + try { + Files.delete(p); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + final Path indexTmp = storagePath.resolve("index.tmp"); + final Path indexFile = storagePath.resolve("index.idx"); + + Files.deleteIfExists(indexFile); + Files.deleteIfExists(indexTmp); + + boolean noData = Files.size(compactionFile) == 0; + + Files.write(indexTmp, + noData ? Collections.emptyList() : Collections.singleton(SSTABLE_PREFIX + "0"), + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE); + if (noData) { + Files.delete(compactionFile); + } else { + Files.move(compactionFile, storagePath.resolve(SSTABLE_PREFIX + "0"), StandardCopyOption.ATOMIC_MOVE); + } + } + + private static Path compactionFile(Path storagePath) { + return storagePath.resolve("compaction"); + } +} diff --git a/src/main/java/ru/vk/itmo/osipovdaniil/DiskStorageUtilsSimple.java b/src/main/java/ru/vk/itmo/osipovdaniil/DiskStorageUtilsSimple.java new file mode 100644 index 000000000..f5f94d34b --- /dev/null +++ b/src/main/java/ru/vk/itmo/osipovdaniil/DiskStorageUtilsSimple.java @@ -0,0 +1,66 @@ +package ru.vk.itmo.osipovdaniil; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Path; + +public final class DiskStorageUtilsSimple { + + private static final String INDEX = "index.idx"; + private static final String INDEX_TMP = "index.tmp"; + + private DiskStorageUtilsSimple() { + } + + public static Path getIndexTmpPath(final Path storagePath) { + return storagePath.resolve(INDEX_TMP); + } + + public static Path getIndexPath(final Path storagePath) { + return storagePath.resolve(INDEX); + } + + static long recordsCount(final MemorySegment segment) { + long indexSize = indexSize(segment); + return indexSize / Long.BYTES / 2; + } + + static long indexSize(final MemorySegment segment) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + + static MemorySegment slice(final MemorySegment page, final long start, final long end) { + return page.asSlice(start, end - start); + } + + static long startOfKey(final MemorySegment segment, final long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES); + } + + static long endOfKey(final MemorySegment segment, final long recordIndex) { + return normalizedStartOfValue(segment, recordIndex); + } + + static long normalizedStartOfValue(final MemorySegment segment, final long recordIndex) { + return normalize(startOfValue(segment, recordIndex)); + } + + static long startOfValue(final MemorySegment segment, final long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES + Long.BYTES); + } + + static long endOfValue(final MemorySegment segment, final long recordIndex, final long recordsCount) { + if (recordIndex < recordsCount - 1) { + return startOfKey(segment, recordIndex + 1); + } + return segment.byteSize(); + } + + static long tombstone(final long offset) { + return 1L << 63 | offset; + } + + static long normalize(final long value) { + return value & ~(1L << 63); + } +} diff --git a/src/main/java/ru/vk/itmo/osipovdaniil/InMemoryDao.java b/src/main/java/ru/vk/itmo/osipovdaniil/InMemoryDao.java index 693973c7e..61bcb9f12 100644 --- a/src/main/java/ru/vk/itmo/osipovdaniil/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/osipovdaniil/InMemoryDao.java @@ -1,75 +1,44 @@ package ru.vk.itmo.osipovdaniil; -import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; import java.io.IOException; -import java.io.UncheckedIOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Comparator; +import java.util.Collections; import java.util.Iterator; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; public class InMemoryDao implements Dao> { - private static final String SSTABLE = "sstable.txt"; + private static final String DATA = "data"; - private final Path ssTablePath; + private final Path path; - private final Arena arena = Arena.ofShared(); + private final Arena arena; - private final MemorySegment mappedMemorySegment; + private final DiskStorage diskStorage; private final ConcurrentNavigableMap> memorySegmentMap - = new ConcurrentSkipListMap<>(memSegmentComparator); - - private static final MemorySegmentComparator memSegmentComparator = new MemorySegmentComparator(); - - private static final class MemorySegmentComparator implements Comparator { - - @Override - public int compare(final MemorySegment a, final MemorySegment b) { - long mismatchOffset = a.mismatch(b); - if (mismatchOffset == -1) { - return 0; - } else if (mismatchOffset == a.byteSize()) { - return -1; - } else if (mismatchOffset == b.byteSize()) { - return 1; - } else { - return Byte.compare(a.getAtIndex(ValueLayout.JAVA_BYTE, mismatchOffset), - b.getAtIndex(ValueLayout.JAVA_BYTE, mismatchOffset)); - } - } - } + = new ConcurrentSkipListMap<>(Utils::compareMemorySegments); - public InMemoryDao() { - this.ssTablePath = null; - this.mappedMemorySegment = null; + public InMemoryDao(final Config config) throws IOException { + this.path = config.basePath().resolve(DATA); + Files.createDirectories(path); + this.arena = Arena.ofShared(); + this.diskStorage = new DiskStorage(DiskStorageUtils.loadOrRecover(path, arena)); } - public InMemoryDao(final Config config) { - this.ssTablePath = config.basePath().resolve(SSTABLE); - if (!Files.exists(ssTablePath)) { - this.mappedMemorySegment = null; - return; - } - try (FileChannel fileChannel = FileChannel.open(ssTablePath, StandardOpenOption.READ)) { - long ssTableFileSize = Files.size(ssTablePath); - this.mappedMemorySegment = fileChannel.map( - FileChannel.MapMode.READ_ONLY, 0, ssTableFileSize, arena); - } catch (IOException e) { - arena.close(); - throw new UncheckedIOException(e); - } + /** + * Compacts data (no-op by default). + */ + @Override + public void compact() throws IOException { + DiskStorageUtils.compact(path, this::all); } /** @@ -81,15 +50,20 @@ public InMemoryDao(final Config config) { */ @Override public Iterator> get(final MemorySegment from, final MemorySegment to) { + return diskStorage.range(getInMemory(from, to), from, to); + } + + private Iterator> getInMemory(final MemorySegment from, final MemorySegment to) { if (from == null && to == null) { return memorySegmentMap.values().iterator(); - } else if (from == null) { + } + if (from == null) { return memorySegmentMap.headMap(to).values().iterator(); - } else if (to == null) { + } + if (to == null) { return memorySegmentMap.tailMap(from).values().iterator(); - } else { - return memorySegmentMap.subMap(from, to).values().iterator(); } + return memorySegmentMap.subMap(from, to).values().iterator(); } /** @@ -100,33 +74,22 @@ public Iterator> get(final MemorySegment from, final Memory */ @Override public Entry get(final MemorySegment key) { - final Entry memorySegmentEntry = memorySegmentMap.get(key); - if (memorySegmentEntry != null) { - return memorySegmentEntry; + final Entry entry = memorySegmentMap.get(key); + if (entry != null) { + if (entry.value() == null) { + return null; + } + return entry; } - if (mappedMemorySegment == null) { + + final Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + + if (!iterator.hasNext()) { return null; } - long offset = 0; - MemorySegment lastValue = null; - while (offset < mappedMemorySegment.byteSize()) { - long keyLength = mappedMemorySegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; - if (keyLength != key.byteSize()) { - offset += keyLength; - offset += mappedMemorySegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset) + Long.BYTES; - continue; - } - final MemorySegment expectedKey = mappedMemorySegment.asSlice(offset, keyLength); - offset += keyLength; - long valueLength = mappedMemorySegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - if (key.mismatch(expectedKey) == -1) { - lastValue = mappedMemorySegment.asSlice(offset + Long.BYTES, valueLength); - } - offset += Long.BYTES + valueLength; - } - if (lastValue != null) { - return new BaseEntry<>(key, lastValue); + Entry next = iterator.next(); + if (next.key().mismatch(key) == -1) { + return next; } return null; } @@ -141,43 +104,22 @@ public void upsert(final Entry entry) { memorySegmentMap.put(entry.key(), entry); } - private long getSSTableFileSize() { - long sz = 0; - for (final Entry entry : memorySegmentMap.values()) { - sz += entry.key().byteSize() + entry.value().byteSize() + 2 * Long.BYTES; + /** + * Persists data (no-op by default). + */ + @Override + public void flush() throws IOException { + if (!memorySegmentMap.isEmpty()) { + DiskStorageUtils.save(path, memorySegmentMap.values()); } - return sz; } @Override public void close() throws IOException { - try (FileChannel fileChannel = FileChannel.open(ssTablePath, - StandardOpenOption.READ, - StandardOpenOption.WRITE, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.CREATE); - Arena writeArena = Arena.ofConfined()) { - long fileSize = getSSTableFileSize(); - final MemorySegment writeMemorySegment = fileChannel.map( - FileChannel.MapMode.READ_WRITE, 0, fileSize, writeArena); - long offset = 0; - for (final Entry entry : memorySegmentMap.values()) { - offset = writeMemorySegment(entry.key(), writeMemorySegment, offset); - offset = writeMemorySegment(entry.value(), writeMemorySegment, offset); - } - writeMemorySegment.load(); - } finally { - arena.close(); + if (!arena.scope().isAlive()) { + return; } - } - - private long writeMemorySegment(final MemorySegment srcMemorySegment, - final MemorySegment dstMemorySegment, - final long offset) { - long srcMemorySegmentSize = srcMemorySegment.byteSize(); - dstMemorySegment.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, srcMemorySegmentSize); - MemorySegment.copy(srcMemorySegment, 0, dstMemorySegment, offset + Long.BYTES, - srcMemorySegmentSize); - return offset + Long.BYTES + srcMemorySegmentSize; + arena.close(); + flush(); } } diff --git a/src/main/java/ru/vk/itmo/osipovdaniil/MergeIterator.java b/src/main/java/ru/vk/itmo/osipovdaniil/MergeIterator.java new file mode 100644 index 000000000..128977d11 --- /dev/null +++ b/src/main/java/ru/vk/itmo/osipovdaniil/MergeIterator.java @@ -0,0 +1,137 @@ +package ru.vk.itmo.osipovdaniil; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; + +public class MergeIterator implements Iterator { + + private final PriorityQueue> priorityQueue; + private final Comparator comparator; + + PeekIterator peek; + + private static class PeekIterator implements Iterator { + + public final int id; + private final Iterator delegate; + private T peek; + + private PeekIterator(int id, Iterator delegate) { + this.id = id; + this.delegate = delegate; + } + + @Override + public boolean hasNext() { + if (peek == null) { + return delegate.hasNext(); + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T pk = peek(); + this.peek = null; + return pk; + } + + private T peek() { + if (peek == null) { + if (!delegate.hasNext()) { + return null; + } + peek = delegate.next(); + } + return peek; + } + } + + public MergeIterator(final Collection> iterators, final Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + priorityQueue = new PriorityQueue<>( + iterators.size(), + peekComp.thenComparing(o -> -o.id) + ); + + int id = 0; + for (Iterator iterator : iterators) { + if (iterator.hasNext()) { + priorityQueue.add(new PeekIterator<>(id++, iterator)); + } + } + } + + private PeekIterator peek() { + while (peek == null) { + peek = priorityQueue.poll(); + if (peek == null) { + return null; + } + fillingQueue(); + if (peek.peek() == null) { + peek = null; + continue; + } + if (skip(peek.peek())) { + peek.next(); + if (peek.hasNext()) { + priorityQueue.add(peek); + } + peek = null; + } + } + return peek; + } + + private void fillingQueue() { + while (true) { + final PeekIterator next = priorityQueue.peek(); + if (next == null) { + break; + } + int compare = comparator.compare(peek.peek(), next.peek()); + if (compare != 0) { + break; + } + PeekIterator poll = priorityQueue.poll(); + if (poll == null) { + continue; + } + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + } + } + + protected boolean skip(final T t) { + return t == null; + } + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + final PeekIterator pk = peek(); + if (pk == null) { + throw new NoSuchElementException(); + } + final T next = pk.next(); + this.peek = null; + if (pk.hasNext()) { + priorityQueue.add(pk); + } + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/osipovdaniil/Utils.java b/src/main/java/ru/vk/itmo/osipovdaniil/Utils.java new file mode 100644 index 000000000..c28b52c5f --- /dev/null +++ b/src/main/java/ru/vk/itmo/osipovdaniil/Utils.java @@ -0,0 +1,24 @@ +package ru.vk.itmo.osipovdaniil; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +public final class Utils { + + private Utils() { + } + + public static int compareMemorySegments(final MemorySegment a, final MemorySegment b) { + long mismatchOffset = a.mismatch(b); + if (mismatchOffset == -1) { + return 0; + } else if (mismatchOffset == a.byteSize()) { + return -1; + } else if (mismatchOffset == b.byteSize()) { + return 1; + } else { + return Byte.compare(a.getAtIndex(ValueLayout.JAVA_BYTE, mismatchOffset), + b.getAtIndex(ValueLayout.JAVA_BYTE, mismatchOffset)); + } + } +} diff --git a/src/main/java/ru/vk/itmo/osokindmitry/DiskStorage.java b/src/main/java/ru/vk/itmo/osokindmitry/DiskStorage.java new file mode 100644 index 000000000..ec803501e --- /dev/null +++ b/src/main/java/ru/vk/itmo/osokindmitry/DiskStorage.java @@ -0,0 +1,250 @@ +package ru.vk.itmo.osokindmitry; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; + +public class DiskStorage { + + private final List tableList; + private static final String INDEX_FILE_NAME = "index"; + private static final String SSTABLE_EXT = ".sstable"; + private static final String TMP_EXT = ".tmp"; + + public DiskStorage(List ssTablePaths, Arena arena) throws IOException { + tableList = new ArrayList<>(); + for (Path path : ssTablePaths) { + tableList.add(new SsTable(path, arena)); + } + } + + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to) { + + List>> iterators = new ArrayList<>(tableList.size() + 1); + for (SsTable ssTable : tableList) { + iterators.add(ssTable.iterator(from, to)); + } + iterators.add(firstIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, PersistentDao::compare)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + /** + * Stores memTable as follows: + * index: + * |key0_Start|value0_Start|key1_Start|value1_Start|key2_Start|value2_Start|... + * key0_Start = data start = end of index + * data: + * |key0|value0|key1|value1|... + */ + public static void save(Path storagePath, Iterable> iterable) + throws IOException { + final Path indexTmp = storagePath.resolve(INDEX_FILE_NAME + TMP_EXT); + final Path indexFile = storagePath.resolve(INDEX_FILE_NAME + SSTABLE_EXT); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + + String newFileName = existedFiles.size() + SSTABLE_EXT; + + long dataSize = 0; + long count = 0; + + for (Entry entry : iterable) { + dataSize += entry.key().byteSize(); + + if (entry.value() != null) { + dataSize += entry.value().byteSize(); + } + count++; + } + long indexSize = count * 2 * Long.BYTES; + + try ( + FileChannel fileChannel = FileChannel.open( + storagePath.resolve(newFileName), + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE + ); + Arena writeArena = Arena.ofConfined() + ) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + indexSize + dataSize, + writeArena + ); + + long dataOffset = indexSize; + int indexOffset = 0; + for (Entry entry : iterable) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + MemorySegment.copy(entry.key(), 0, fileSegment, dataOffset, entry.key().byteSize()); + dataOffset += entry.key().byteSize(); + indexOffset += Long.BYTES; + + if (entry.value() == null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, SsTable.tombstone(dataOffset)); + } else { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + MemorySegment.copy(entry.value(), 0, fileSegment, dataOffset, entry.value().byteSize()); + dataOffset += entry.value().byteSize(); + } + indexOffset += Long.BYTES; + } + } + + updateIndex(indexFile, indexTmp, existedFiles, newFileName); + } + + private static void updateIndex(Path indexFile, Path indexTmp, List existedFiles, String newFileName) + throws IOException { + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + + List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + list.add(newFileName); + Files.write( + indexFile, + list, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + Files.delete(indexTmp); + } + + public void compact(Path storagePath) throws IOException { + if (tableList.isEmpty()) { + return; + } + final Path indexFile = storagePath.resolve(INDEX_FILE_NAME + SSTABLE_EXT); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + + MergeIterator> mergeIterator = getMergeIterator(); + long dataSize = 0; + long count = 0; + while (mergeIterator.hasNext()) { + count++; + Entry next = mergeIterator.next(); + dataSize += next.key().byteSize() + next.value().byteSize(); + } + dataSize += count * Long.BYTES * 2; + + MemorySegment fileSegment; + try ( + FileChannel fileChannel = FileChannel.open( + storagePath.resolve("0" + TMP_EXT), + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE + ); + Arena writeArena = Arena.ofConfined() + ) { + fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + dataSize, + writeArena + ); + + mergeIterator = getMergeIterator(); + long dataOffset = count * 2 * Long.BYTES; + int indexOffset = 0; + while (mergeIterator.hasNext()) { + Entry entry = mergeIterator.next(); + + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + MemorySegment.copy(entry.key(), 0, fileSegment, dataOffset, entry.key().byteSize()); + dataOffset += entry.key().byteSize(); + indexOffset += Long.BYTES; + + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + MemorySegment.copy(entry.value(), 0, fileSegment, dataOffset, entry.value().byteSize()); + dataOffset += entry.value().byteSize(); + indexOffset += Long.BYTES; + } + } + + updateIndexAndCleanUp(storagePath, indexFile); + tableList.clear(); + tableList.add(new SsTable(fileSegment)); + } + + private void updateIndexAndCleanUp(Path storagePath, Path indexFile) throws IOException { + + final List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + final Path indexTmp = storagePath.resolve(INDEX_FILE_NAME + TMP_EXT); + + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + Files.move(storagePath.resolve("0" + TMP_EXT), storagePath.resolve("0" + SSTABLE_EXT), + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + + Files.writeString( + indexFile, + "0" + SSTABLE_EXT, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + try { + for (int i = 1; i < existedFiles.size(); i++) { + Files.delete(storagePath.resolve(existedFiles.get(i))); + } + } catch (IOException e) { + // If we fail during delete, db will recover with index.tmp that points to deleted files + Files.delete(indexTmp); + throw e; + } + + Files.delete(indexTmp); + } + + private MergeIterator> getMergeIterator() { + List>> iterators = new ArrayList<>(tableList.size() + 1); + for (SsTable ssTable : tableList) { + iterators.add(ssTable.iterator(null, null)); + } + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, PersistentDao::compare)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + +} diff --git a/src/main/java/ru/vk/itmo/osokindmitry/InMemoryDao.java b/src/main/java/ru/vk/itmo/osokindmitry/InMemoryDao.java deleted file mode 100644 index 948195a75..000000000 --- a/src/main/java/ru/vk/itmo/osokindmitry/InMemoryDao.java +++ /dev/null @@ -1,166 +0,0 @@ -package ru.vk.itmo.osokindmitry; - -import ru.vk.itmo.BaseEntry; -import ru.vk.itmo.Config; -import ru.vk.itmo.Dao; -import ru.vk.itmo.Entry; - -import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.channels.FileChannel; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Collections; -import java.util.Iterator; -import java.util.Set; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; - -public class InMemoryDao implements Dao> { - - private final ConcurrentNavigableMap> storage; - private final ConcurrentNavigableMap> cachedValues; - - private final Arena arena; - private final Path path; - private final MemorySegment mappedFile; - - private static final String FILE_NAME = "sstable.txt"; - - public InMemoryDao(Config config) { - path = config.basePath().resolve(FILE_NAME); - arena = Arena.ofConfined(); - storage = new ConcurrentSkipListMap<>(InMemoryDao::compare); - cachedValues = new ConcurrentSkipListMap<>(InMemoryDao::compare); - mappedFile = mapFile(path); - } - - @Override - public Entry get(MemorySegment key) { - Entry entry = storage.get(key); - // avoiding extra file operations by checking cached values - if (entry == null) { - entry = cachedValues.get(key); - } - // if value is still null then searching in file - if (entry == null && mappedFile != null) { - entry = searchInSlice(mappedFile, key); - } - - return entry; - } - - @Override - public Iterator> get(MemorySegment from, MemorySegment to) { - if (storage.isEmpty()) { - return Collections.emptyIterator(); - } - boolean empty = to == null; - MemorySegment first = from == null ? storage.firstKey() : from; - MemorySegment last = to == null ? storage.lastKey() : to; - return storage.subMap(first, true, last, empty).values().iterator(); - } - - @Override - public void upsert(Entry entry) { - storage.put(entry.key(), entry); - } - - @Override - public void flush() throws IOException { - try ( - FileChannel fc = FileChannel.open( - path, - Set.of(StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE)) - ) { - - long ssTableSize = Long.BYTES * 2L * storage.size(); - for (Entry value : storage.values()) { - ssTableSize += value.key().byteSize() + value.value().byteSize(); - } - - MemorySegment ssTable = fc.map(FileChannel.MapMode.READ_WRITE, 0, ssTableSize, arena); - long offset = 0; - - for (Entry value : storage.values()) { - offset = writeEntry(value.key(), ssTable, offset); - offset = writeEntry(value.value(), ssTable, offset); - } - } - } - - @Override - public void close() throws IOException { - flush(); - if (arena.scope().isAlive()) { - arena.close(); - } - } - - private long writeEntry(MemorySegment entry, MemorySegment ssTable, long offset) { - long curOffset = offset; - ssTable.set(ValueLayout.JAVA_LONG_UNALIGNED, curOffset, entry.byteSize()); - curOffset += Long.BYTES; - MemorySegment.copy(entry, 0, ssTable, curOffset, entry.byteSize()); - curOffset += entry.byteSize(); - return curOffset; - } - - private MemorySegment mapFile(Path path) { - if (path.toFile().exists()) { - try ( - FileChannel fc = FileChannel.open( - path, - Set.of(StandardOpenOption.CREATE, StandardOpenOption.READ)) - ) { - if (fc.size() != 0) { - return fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size(), arena); - } - } catch (IOException e) { - return null; - } - } - return null; - } - - private Entry searchInSlice(MemorySegment mappedSegment, MemorySegment key) { - long offset = 0; - while (offset < mappedSegment.byteSize() - Long.BYTES) { - - long size = mappedSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; - MemorySegment slicedKey = mappedSegment.asSlice(offset, size); - offset += size; - - long mismatch = key.mismatch(slicedKey); - size = mappedSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; - - if (mismatch == -1) { - MemorySegment slicedValue = mappedSegment.asSlice(offset, size); - BaseEntry entry = new BaseEntry<>(slicedKey, slicedValue); - cachedValues.put(slicedKey, entry); - return entry; - } - offset += size; - } - return null; - } - - private static int compare(MemorySegment segment1, MemorySegment segment2) { - long offset = segment1.mismatch(segment2); - if (offset == -1) { - return 0; - } else if (offset == segment1.byteSize()) { - return -1; - } else if (offset == segment2.byteSize()) { - return 1; - } - byte b1 = segment1.get(ValueLayout.JAVA_BYTE, offset); - byte b2 = segment2.get(ValueLayout.JAVA_BYTE, offset); - return Byte.compare(b1, b2); - } - -} diff --git a/src/main/java/ru/vk/itmo/bandurinvladislav/MergeIterator.java b/src/main/java/ru/vk/itmo/osokindmitry/MergeIterator.java similarity index 85% rename from src/main/java/ru/vk/itmo/bandurinvladislav/MergeIterator.java rename to src/main/java/ru/vk/itmo/osokindmitry/MergeIterator.java index f5cf12d16..b80fb06cf 100644 --- a/src/main/java/ru/vk/itmo/bandurinvladislav/MergeIterator.java +++ b/src/main/java/ru/vk/itmo/osokindmitry/MergeIterator.java @@ -1,4 +1,4 @@ -package ru.vk.itmo.bandurinvladislav; +package ru.vk.itmo.osokindmitry; import java.util.Collection; import java.util.Comparator; @@ -10,8 +10,7 @@ public class MergeIterator implements Iterator { private final PriorityQueue> priorityQueue; private final Comparator comparator; - - private PeekIterator peekIterator; + PeekIterator peekIterator; private static class PeekIterator implements Iterator { @@ -37,9 +36,9 @@ public T next() { if (!hasNext()) { throw new NoSuchElementException(); } - T peekValue = peek(); - this.peek = null; - return peekValue; + T entry = peek(); + peek = null; + return entry; } private T peek() { @@ -75,8 +74,7 @@ private PeekIterator peek() { if (peekIterator == null) { return null; } - - refreshIterator(); + moveIterator(); if (peekIterator.peek() == null) { peekIterator = null; @@ -91,11 +89,11 @@ private PeekIterator peek() { peekIterator = null; } } + return peekIterator; } - @SuppressWarnings("DataFlowIssue") - private void refreshIterator() { + private void moveIterator() { while (true) { PeekIterator next = priorityQueue.peek(); if (next == null) { @@ -103,14 +101,17 @@ private void refreshIterator() { } int compare = comparator.compare(peekIterator.peek(), next.peek()); - if (compare != 0) { + if (compare == 0) { + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } + } + } else { break; } - PeekIterator poll = priorityQueue.poll(); - poll.next(); - if (poll.hasNext()) { - priorityQueue.add(poll); - } } } @@ -136,4 +137,5 @@ public T next() { } return next; } + } diff --git a/src/main/java/ru/vk/itmo/osokindmitry/PersistentDao.java b/src/main/java/ru/vk/itmo/osokindmitry/PersistentDao.java new file mode 100644 index 000000000..5660af819 --- /dev/null +++ b/src/main/java/ru/vk/itmo/osokindmitry/PersistentDao.java @@ -0,0 +1,119 @@ +package ru.vk.itmo.osokindmitry; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Iterator; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +public class PersistentDao implements Dao> { + + private final Arena arena; + private final Path path; + private final DiskStorage diskStorage; + private ConcurrentNavigableMap> storage; + + public PersistentDao(Config config) throws IOException { + path = config.basePath().resolve("data"); + Files.createDirectories(path); + + arena = Arena.ofShared(); + storage = new ConcurrentSkipListMap<>(PersistentDao::compare); + + this.diskStorage = new DiskStorage(Utils.loadOrRecover(path), arena); + } + + @Override + public Entry get(MemorySegment key) { + Entry entry = storage.get(key); + if (entry != null) { + if (entry.value() == null) { + return null; + } + return entry; + } + + Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + + if (!iterator.hasNext()) { + return null; + } + Entry next = iterator.next(); + if (compare(next.key(), key) == 0) { + return next; + } + return null; + } + + static int compare(MemorySegment segment1, MemorySegment segment2) { + long offset = segment1.mismatch(segment2); + if (offset == -1) { + return 0; + } else if (offset == segment1.byteSize()) { + return -1; + } else if (offset == segment2.byteSize()) { + return 1; + } + byte b1 = segment1.get(ValueLayout.JAVA_BYTE, offset); + byte b2 = segment2.get(ValueLayout.JAVA_BYTE, offset); + return Byte.compare(b1, b2); + } + + private Iterator> getInMemory(MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return storage.values().iterator(); + } + if (from == null) { + return storage.headMap(to).values().iterator(); + } + if (to == null) { + return storage.tailMap(from).values().iterator(); + } + return storage.subMap(from, to).values().iterator(); + } + + @Override + public Iterator> get(MemorySegment from, MemorySegment to) { + return diskStorage.range(getInMemory(from, to), from, to); + } + + @Override + public void upsert(Entry entry) { + storage.put(entry.key(), entry); + } + + @Override + public void flush() throws IOException { + if (!storage.isEmpty()) { + DiskStorage.save(path, storage.values()); + } + } + + @Override + public void compact() throws IOException { + if (!storage.isEmpty()) { + flush(); + storage = new ConcurrentSkipListMap<>(PersistentDao::compare); + } + diskStorage.compact(path); + } + + @Override + public void close() throws IOException { + if (!arena.scope().isAlive()) { + return; + } + arena.close(); + flush(); + } + +} diff --git a/src/main/java/ru/vk/itmo/osokindmitry/SsTable.java b/src/main/java/ru/vk/itmo/osokindmitry/SsTable.java new file mode 100644 index 000000000..13e263128 --- /dev/null +++ b/src/main/java/ru/vk/itmo/osokindmitry/SsTable.java @@ -0,0 +1,146 @@ +package ru.vk.itmo.osokindmitry; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class SsTable { + private final MemorySegment segment; + + public SsTable(Path fullPath, Arena arena) throws IOException { + try (FileChannel fileChannel = FileChannel.open(fullPath, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + segment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(fullPath), + arena + ); + } + } + + public SsTable(MemorySegment segment) { + this.segment = segment; + } + + private static long indexOf(MemorySegment segment, MemorySegment key) { + long recordsCount = recordsCount(segment); + + long left = 0; + long right = recordsCount - 1; + while (left <= right) { + long mid = (left + right) >>> 1; + + long startOfKey = startOfKey(segment, mid); + long endOfKey = endOfKey(segment, mid); + long mismatch = MemorySegment.mismatch(segment, startOfKey, endOfKey, key, 0, key.byteSize()); + if (mismatch == -1) { + return mid; + } + + if (mismatch == key.byteSize()) { + right = mid - 1; + continue; + } + + if (mismatch == endOfKey - startOfKey) { + left = mid + 1; + continue; + } + + int b1 = Byte.toUnsignedInt(segment.get(ValueLayout.JAVA_BYTE, startOfKey + mismatch)); + int b2 = Byte.toUnsignedInt(key.get(ValueLayout.JAVA_BYTE, mismatch)); + if (b1 > b2) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + return tombstone(left); + } + + private static long recordsCount(MemorySegment segment) { + long indexSize = indexSize(segment); + return indexSize / Long.BYTES / 2; + } + + private static long indexSize(MemorySegment segment) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); + } + + public static long tombstone(long offset) { + return 1L << 63 | offset; + } + + private static long normalize(long value) { + return value & ~(1L << 63); + } + + private static MemorySegment slice(MemorySegment page, long start, long end) { + return page.asSlice(start, end - start); + } + + private static long startOfKey(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES); + } + + private static long endOfKey(MemorySegment segment, long recordIndex) { + return normalizedStartOfValue(segment, recordIndex); + } + + private static long normalizedStartOfValue(MemorySegment segment, long recordIndex) { + return normalize(startOfValue(segment, recordIndex)); + } + + private static long startOfValue(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES + Long.BYTES); + } + + private static long endOfValue(MemorySegment segment, long recordIndex, long recordsCount) { + if (recordIndex < recordsCount - 1) { + return startOfKey(segment, recordIndex + 1); + } + return segment.byteSize(); + } + + public Iterator> iterator(MemorySegment from, MemorySegment to) { + long recordIndexFrom = from == null ? 0 : normalize(indexOf(segment, from)); + long recordIndexTo = to == null ? recordsCount(segment) : normalize(indexOf(segment, to)); + long recordsCount = recordsCount(segment); + + return new Iterator<>() { + long index = recordIndexFrom; + + @Override + public boolean hasNext() { + return index < recordIndexTo; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + MemorySegment key = slice(segment, startOfKey(segment, index), endOfKey(segment, index)); + long startOfValue = startOfValue(segment, index); + MemorySegment value = + startOfValue < 0 + ? null + : slice(segment, startOfValue, endOfValue(segment, index, recordsCount)); + index++; + return new BaseEntry<>(key, value); + } + }; + } + +} diff --git a/src/main/java/ru/vk/itmo/osokindmitry/Utils.java b/src/main/java/ru/vk/itmo/osokindmitry/Utils.java new file mode 100644 index 000000000..901d781d9 --- /dev/null +++ b/src/main/java/ru/vk/itmo/osokindmitry/Utils.java @@ -0,0 +1,44 @@ +package ru.vk.itmo.osokindmitry; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; + +public final class Utils { + private static final String INDEX_FILE_NAME = "index"; + private static final String SSTABLE_EXT = ".sstable"; + private static final String TMP_EXT = ".tmp"; + + private Utils() { + throw new UnsupportedOperationException("This is a utility class and cannot be instantiated"); + } + + public static List loadOrRecover(Path storagePath) throws IOException { + final Path indexTmp = storagePath.resolve(INDEX_FILE_NAME + TMP_EXT); + final Path indexFile = storagePath.resolve(INDEX_FILE_NAME + SSTABLE_EXT); + + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + } + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + List result = new ArrayList<>(existedFiles.size()); + for (String fileName : existedFiles) { + Path file = storagePath.resolve(fileName); + result.add(file); + } + return result; + } + +} diff --git a/src/main/java/ru/vk/itmo/prokopyevnikita/AlreadyFlushingInBg.java b/src/main/java/ru/vk/itmo/prokopyevnikita/AlreadyFlushingInBg.java new file mode 100644 index 000000000..945e1a188 --- /dev/null +++ b/src/main/java/ru/vk/itmo/prokopyevnikita/AlreadyFlushingInBg.java @@ -0,0 +1,4 @@ +package ru.vk.itmo.prokopyevnikita; + +public class AlreadyFlushingInBg extends RuntimeException{ +} diff --git a/src/main/java/ru/vk/itmo/prokopyevnikita/DaoImpl.java b/src/main/java/ru/vk/itmo/prokopyevnikita/DaoImpl.java index 7a37d3cec..e3e6c8ae2 100644 --- a/src/main/java/ru/vk/itmo/prokopyevnikita/DaoImpl.java +++ b/src/main/java/ru/vk/itmo/prokopyevnikita/DaoImpl.java @@ -7,8 +7,11 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; import java.util.Iterator; -import java.util.NavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -16,114 +19,184 @@ public class DaoImpl implements Dao> { private final Config config; private final ReadWriteLock lock = new ReentrantReadWriteLock(); - - private Storage storage; - - private NavigableMap> map = - new ConcurrentSkipListMap<>(MemorySegmentComparator::compare); + private final ExecutorService executorService = Executors.newSingleThreadExecutor(r -> { + return new Thread(r, "flusher"); + }); + private volatile State state; public DaoImpl(Config config) throws IOException { this.config = config; - storage = Storage.load(config); + this.state = State.initState(config, Storage.load(config)); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { - lock.readLock().lock(); - try { - if (from == null) { - return storage.getIterator( - MemorySegment.NULL, to, - getMemoryIterator(MemorySegment.NULL, to) - ); - } + return combinedIterator(Objects.requireNonNullElse(from, MemorySegment.NULL), to, true); + } - return storage.getIterator(from, to, getMemoryIterator(from, to)); - } finally { - lock.readLock().unlock(); + public Iterator> getOnlyFromDisk(MemorySegment from, MemorySegment to) { + return combinedIterator(Objects.requireNonNullElse(from, MemorySegment.NULL), to, false); + } + + private Iterator> combinedIterator( + MemorySegment from, + MemorySegment to, + boolean includeMemoryAndFlushing + ) { + State currenState = accessStateAndCloseCheck(); + if (includeMemoryAndFlushing) { + Iterator> memoryIterator = currenState.memory.get(from, to); + Iterator> flushingIterator = currenState.flushing.get(from, to); + return currenState.storage.getIterator(from, to, memoryIterator, flushingIterator); } + return currenState.storage.getIterator(from, to, null, null); } @Override public Entry get(MemorySegment key) { - lock.readLock().lock(); - try { - Iterator> iterator = get(key, null); - if (!iterator.hasNext()) { - return null; - } - Entry next = iterator.next(); - if (MemorySegmentComparator.compare(key, next.key()) == 0) { - return next; - } + Iterator> iterator = get(key, null); + if (!iterator.hasNext()) { return null; - } finally { - lock.readLock().unlock(); } + Entry next = iterator.next(); + if (MemorySegmentComparator.compare(key, next.key()) == 0) { + return next; + } + return null; + } @Override public void upsert(Entry entry) { - lock.readLock().lock(); - try { - map.put(entry.key(), entry); - } finally { - lock.readLock().unlock(); + State currenState = accessStateAndCloseCheck(); + boolean flush = currenState.memory.put(entry.key(), entry); + if (flush) { + flushInBackground(false); } } - private Iterator> getMemoryIterator(MemorySegment from, MemorySegment to) { - lock.readLock().lock(); - try { - if (from == null && to == null) { - return map.values().iterator(); - } else if (to == null) { - return map.tailMap(from).values().iterator(); - } else if (from == null) { - return map.headMap(to).values().iterator(); + private void flushInBackground(boolean canBeParallel) { + + executorService.execute(() -> { + lock.writeLock().lock(); + try { + State currenState = accessStateAndCloseCheck(); + if (currenState.isFlushing()) { + if (canBeParallel) { + return; + } + throw new AlreadyFlushingInBg(); + } + currenState = currenState.prepareForFlush(); + this.state = currenState; + } finally { + lock.writeLock().unlock(); } - return map.subMap(from, to).values().iterator(); - } finally { - lock.readLock().unlock(); - } + try { + State currenState = accessStateAndCloseCheck(); + + Storage.save(config, currenState.flushing.values()); + Storage newStorage = Storage.load(config); + + lock.writeLock().lock(); + try { + this.state = currenState.afterFlush(newStorage); + } finally { + lock.writeLock().unlock(); + } + } catch (IOException e) { + throw new IllegalStateException(e); + } + }); + } @Override public void flush() throws IOException { + boolean flush = false; lock.writeLock().lock(); try { - storage.close(); - Storage.save(config, map.values(), storage); - map = new ConcurrentSkipListMap<>(MemorySegmentComparator::compare); - storage = Storage.load(config); + flush = state.memory.overflow(); } finally { lock.writeLock().unlock(); } + + if (flush) { + flushInBackground(true); + } } + // only single thread can call this method @Override - public void close() throws IOException { - lock.writeLock().lock(); + public synchronized void close() throws IOException { + State currenState = this.state; + if (currenState.closed) { + return; + } + executorService.shutdown(); + // await for all tasks to complete + // it can take a lot of time depending on the size of the database try { - storage.close(); - Storage.save(config, map.values(), storage); - } finally { - lock.writeLock().unlock(); + while (!executorService.awaitTermination(12, TimeUnit.HOURS)) { + wait(10); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + currenState.storage.close(); + this.state = currenState.afterClose(); + if (currenState.memory.isEmpty()) { + return; } + Storage.save(config, currenState.memory.values()); } @Override - public void compact() throws IOException { - lock.writeLock().lock(); - try { - if (map.isEmpty() && storage.isCompacted()) { + public void compact() { + State currenState = accessStateAndCloseCheck(); + + if (currenState.memory.isEmpty() && currenState.storage.isCompacted()) { + return; + } + + executorService.execute(() -> { + State stateCompaction = accessStateAndCloseCheck(); + + if (stateCompaction.memory.isEmpty() && stateCompaction.storage.isCompacted()) { return; } - Storage.compact(config, this::all); - storage.close(); - map = new ConcurrentSkipListMap<>(MemorySegmentComparator::compare); - } finally { - lock.writeLock().unlock(); + + // compact only ssTables + Storage storage = null; + try { + Storage.compact(config, + () -> new MergeSkipNullValuesIterator( + List.of( + new OrderedPeekIteratorImpl(0, + getOnlyFromDisk(null, null) + ) + ) + ) + ); + storage = Storage.load(config); + } catch (IOException e) { + throw new IllegalStateException(e); + } + + lock.writeLock().lock(); + try { + this.state = stateCompaction.afterCompaction(storage); + } finally { + lock.writeLock().unlock(); + } + }); + } + + private State accessStateAndCloseCheck() { + State currenState = this.state; + if (currenState.closed) { + throw new IllegalStateException("DAO is Already closed"); } + return currenState; } } diff --git a/src/main/java/ru/vk/itmo/prokopyevnikita/Memory.java b/src/main/java/ru/vk/itmo/prokopyevnikita/Memory.java new file mode 100644 index 000000000..1b6aca22c --- /dev/null +++ b/src/main/java/ru/vk/itmo/prokopyevnikita/Memory.java @@ -0,0 +1,74 @@ +package ru.vk.itmo.prokopyevnikita; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Collection; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +class Memory { + // Placeholder for not presented memory + static final Memory NOT_PRESENTED = new Memory(-1); + private final long threshold; + private final AtomicLong size = new AtomicLong(); + private final AtomicBoolean isOversized = new AtomicBoolean(); + + private final NavigableMap> delegate = + new ConcurrentSkipListMap<>(MemorySegmentComparator::compare); + + public Memory(long threshold) { + this.threshold = threshold; + } + + public Collection> values() { + return delegate.values(); + } + + public boolean overflow() { + return !isOversized.getAndSet(true); + } + + public boolean put(MemorySegment key, Entry entry) { + if (threshold == -1) { + throw new UnsupportedOperationException("Write not supported"); + } + // put and get previous value + Entry segmentEntry = delegate.put(key, entry); + // calculate size of new entry + long newEntrySize = Storage.getSizeOnDisk(entry); + // if previous value exists, subtract its size + if (segmentEntry != null) { + newEntrySize -= Storage.getSizeOnDisk(segmentEntry); + } + // add new entry size to total size + long newSize = size.addAndGet(newEntrySize); + // if total size is greater than threshold, return true (make autoFlush) + if (newSize > threshold) { + return overflow(); + } + return false; + } + + public Entry get(MemorySegment key) { + return delegate.get(key); + } + + public Iterator> get(MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return delegate.values().iterator(); + } else if (to == null) { + return delegate.tailMap(from).values().iterator(); + } else if (from == null) { + return delegate.headMap(to).values().iterator(); + } + return delegate.subMap(from, to).values().iterator(); + } + + public boolean isEmpty() { + return delegate.isEmpty(); + } +} diff --git a/src/main/java/ru/vk/itmo/prokopyevnikita/State.java b/src/main/java/ru/vk/itmo/prokopyevnikita/State.java new file mode 100644 index 000000000..8b361533c --- /dev/null +++ b/src/main/java/ru/vk/itmo/prokopyevnikita/State.java @@ -0,0 +1,93 @@ +package ru.vk.itmo.prokopyevnikita; + +import ru.vk.itmo.Config; + +class State { + final Config config; + final Memory memory; + final Memory flushing; + final boolean closed; + Storage storage; + + // State with opened storage + State(Config config, Memory memory, Memory flushing, Storage storage) { + this.config = config; + this.memory = memory; + this.flushing = flushing; + this.storage = storage; + this.closed = false; + } + + // State with closed storage + State(Config config, Storage storage) { + this.config = config; + this.memory = Memory.NOT_PRESENTED; + this.flushing = Memory.NOT_PRESENTED; + this.storage = storage; + this.closed = true; + } + + // initial State + static State initState(Config config, Storage storage) { + return new State( + config, + new Memory(config.flushThresholdBytes()), + Memory.NOT_PRESENTED, + storage + ); + } + + public State prepareForFlush() { + isClosedCheck(); + if (isFlushing()) { + throw new IllegalStateException("Already flushing"); + } + return new State( + config, + new Memory(config.flushThresholdBytes()), + memory, + storage + ); + } + + public State afterFlush(Storage storage) { + isClosedCheck(); + if (!isFlushing()) { + throw new IllegalStateException("Wasn't flushing"); + } + return new State( + config, + memory, + Memory.NOT_PRESENTED, + storage + ); + } + + public State afterCompaction(Storage storage) { + isClosedCheck(); + return new State( + config, + memory, + flushing, + storage + ); + } + + public State afterClose() { + isClosedCheck(); + if (!storage.isClosed()) { + throw new IllegalStateException("Storage should be closed early"); + } + return new State(config, storage); + } + + public void isClosedCheck() { + if (closed) { + throw new IllegalStateException("Already closed"); + } + } + + public boolean isFlushing() { + return this.flushing != Memory.NOT_PRESENTED; + } +} diff --git a/src/main/java/ru/vk/itmo/prokopyevnikita/Storage.java b/src/main/java/ru/vk/itmo/prokopyevnikita/Storage.java index 0dbbe2c36..5b89c9ccf 100644 --- a/src/main/java/ru/vk/itmo/prokopyevnikita/Storage.java +++ b/src/main/java/ru/vk/itmo/prokopyevnikita/Storage.java @@ -47,6 +47,10 @@ private Storage(Arena arena, List ssTables, boolean isCompacted) public static Storage load(Config config) throws IOException { Path path = config.basePath(); + Path compactedFile = path.resolve("compacted" + DB_EXTENSION); + if (Files.exists(compactedFile)) { + throw new IllegalStateException("compaction not finished!"); + } List ssTables = new ArrayList<>(); Arena arena = Arena.ofShared(); @@ -77,11 +81,8 @@ public static Storage load(Config config) throws IOException { return new Storage(arena, ssTables, isCompacted); } - public static void save(Config config, Collection> entries, Storage storage) + public static void save(Config config, Collection> entries) throws IOException { - if (storage.arena.scope().isAlive()) { - throw new IllegalStateException("Previous arena is alive"); - } if (entries.isEmpty()) { return; @@ -140,7 +141,9 @@ private static void saveByPath(Path path, IterableData entries) throws IOExcepti path, StandardOpenOption.READ, StandardOpenOption.WRITE, - StandardOpenOption.CREATE) + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ) ) { MemorySegment newSSTable = channel.map(FileChannel.MapMode.READ_WRITE, 0, sizeOfNewSSTable, arenaSave); @@ -155,7 +158,6 @@ private static void saveByPath(Path path, IterableData entries) throws IOExcepti offsetData = saveEntrySegment(newSSTable, entry, offsetData); } - } } @@ -198,6 +200,18 @@ public static void compact(Config config, IterableData entries) throws IOExcepti } } + private static long getSize(Entry entry) { + if (entry.value() == null) { + return Long.BYTES + entry.key().byteSize() + Long.BYTES; + } else { + return Long.BYTES + entry.value().byteSize() + entry.key().byteSize() + Long.BYTES; + } + } + + public static long getSizeOnDisk(Entry entry) { + return getSize(entry) + Long.BYTES; + } + public Iterator> iterateThroughSSTable( MemorySegment ssTable, MemorySegment from, @@ -227,22 +241,36 @@ public Entry next() { public Iterator> getIterator( MemorySegment from, MemorySegment to, - Iterator> memoryIterator + Iterator> memoryIterator, + Iterator> flushingIterator ) { - List>> peekIterators = new ArrayList<>(); - peekIterators.add(new OrderedPeekIteratorImpl(0, memoryIterator)); - int order = 1; - for (MemorySegment sstable : ssTables) { - Iterator> iterator = iterateThroughSSTable(sstable, from, to); - peekIterators.add(new OrderedPeekIteratorImpl(order, iterator)); - order++; + try { + List>> peekIterators = new ArrayList<>(); + if (memoryIterator != null) { + peekIterators.add(new OrderedPeekIteratorImpl(0, memoryIterator)); + } + if (flushingIterator != null) { + peekIterators.add(new OrderedPeekIteratorImpl(1, flushingIterator)); + } + int order = 2; + for (MemorySegment sstable : ssTables) { + Iterator> iterator = iterateThroughSSTable(sstable, from, to); + peekIterators.add(new OrderedPeekIteratorImpl(order, iterator)); + order++; + } + return new MergeSkipNullValuesIterator(peekIterators); + } catch (IllegalStateException e) { + if (isClosed()) { + throw new StorageClosedException(e); + } else { + throw e; + } } - return new MergeSkipNullValuesIterator(peekIterators); } @Override - public void close() { - if (arena.scope().isAlive()) { + public void close() throws IOException { + if (!isClosed()) { arena.close(); } } @@ -251,6 +279,10 @@ public boolean isCompacted() { return isCompacted; } + public boolean isClosed() { + return !arena.scope().isAlive(); + } + @FunctionalInterface public interface IterableData extends Iterable> { @Override diff --git a/src/main/java/ru/vk/itmo/prokopyevnikita/StorageAdditionalFunctionality.java b/src/main/java/ru/vk/itmo/prokopyevnikita/StorageAdditionalFunctionality.java index 045221852..fb06e1dd8 100644 --- a/src/main/java/ru/vk/itmo/prokopyevnikita/StorageAdditionalFunctionality.java +++ b/src/main/java/ru/vk/itmo/prokopyevnikita/StorageAdditionalFunctionality.java @@ -32,7 +32,7 @@ public static long saveEntrySegment(MemorySegment newSSTable, Entry Integer.MAX_VALUE) { + throw new IllegalArgumentException("Too big!"); + } + + final int capacity = (int) size; + if (array.length >= capacity) { + return; + } + + // Grow to the nearest bigger power of 2 + final int newSize = Integer.highestOneBit(capacity) << 1; + array = new byte[newSize]; + segment = MemorySegment.ofArray(array); + } + + interface ArrayConsumer { + void process(byte[] array) throws IOException; + } +} diff --git a/src/main/java/ru/vk/itmo/reference/LiveFilteringIterator.java b/src/main/java/ru/vk/itmo/reference/LiveFilteringIterator.java new file mode 100644 index 000000000..676bc0e37 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/LiveFilteringIterator.java @@ -0,0 +1,52 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Filters non tombstone {@link Entry}s. + * + * @author incubos + */ +final class LiveFilteringIterator implements Iterator> { + private final Iterator> delegate; + private Entry next; + + LiveFilteringIterator(final Iterator> delegate) { + this.delegate = delegate; + skipTombstones(); + } + + private void skipTombstones() { + while (delegate.hasNext()) { + final Entry entry = delegate.next(); + if (entry.value() != null) { + this.next = entry; + break; + } + } + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + // Consume + final Entry result = next; + next = null; + + skipTombstones(); + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/reference/MemTable.java b/src/main/java/ru/vk/itmo/reference/MemTable.java new file mode 100644 index 000000000..96843c04b --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/MemTable.java @@ -0,0 +1,49 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +/** + * Memory table. + * + * @author incubos + */ +final class MemTable { + private final NavigableMap> map = + new ConcurrentSkipListMap<>( + MemorySegmentComparator.INSTANCE); + + boolean isEmpty() { + return map.isEmpty(); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + if (from == null && to == null) { + // All + return map.values().iterator(); + } else if (from == null) { + // Head + return map.headMap(to).values().iterator(); + } else if (to == null) { + // Tail + return map.tailMap(from).values().iterator(); + } else { + // Slice + return map.subMap(from, to).values().iterator(); + } + } + + Entry get(final MemorySegment key) { + return map.get(key); + } + + Entry upsert(final Entry entry) { + return map.put(entry.key(), entry); + } +} diff --git a/src/main/java/ru/vk/itmo/reference/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/reference/MemorySegmentComparator.java new file mode 100644 index 000000000..33f2936f1 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/MemorySegmentComparator.java @@ -0,0 +1,89 @@ +package ru.vk.itmo.reference; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Comparator; + +/** + * Compares {@link MemorySegment}s. + * + * @author incubos + */ +final class MemorySegmentComparator implements Comparator { + static final Comparator INSTANCE = + new MemorySegmentComparator(); + + private MemorySegmentComparator() { + // Singleton + } + + @Override + public int compare( + final MemorySegment left, + final MemorySegment right) { + final long mismatch = left.mismatch(right); + if (mismatch == -1L) { + // No mismatch + return 0; + } + + if (mismatch == left.byteSize()) { + // left is prefix of right, so left is smaller + return -1; + } + + if (mismatch == right.byteSize()) { + // right is prefix of left, so left is greater + return 1; + } + + // Compare mismatched bytes as unsigned + return Byte.compareUnsigned( + left.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + mismatch), + right.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + mismatch)); + } + + static int compare( + final MemorySegment srcSegment, + final long srcFromOffset, + final long srcLength, + final MemorySegment dstSegment, + final long dstFromOffset, + final long dstLength) { + final long mismatch = + MemorySegment.mismatch( + srcSegment, + srcFromOffset, + srcFromOffset + srcLength, + dstSegment, + dstFromOffset, + dstFromOffset + dstLength); + if (mismatch == -1L) { + // No mismatch + return 0; + } + + if (mismatch == srcLength) { + // left is prefix of right, so left is smaller + return -1; + } + + if (mismatch == dstLength) { + // right is prefix of left, so left is greater + return 1; + } + + // Compare mismatched bytes as unsigned + return Byte.compareUnsigned( + srcSegment.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + srcFromOffset + mismatch), + dstSegment.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + dstFromOffset + mismatch)); + } +} diff --git a/src/main/java/ru/vk/itmo/reference/MergingEntryIterator.java b/src/main/java/ru/vk/itmo/reference/MergingEntryIterator.java new file mode 100644 index 000000000..8130da2cb --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/MergingEntryIterator.java @@ -0,0 +1,72 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.Queue; + +/** + * Merges entry {@link Iterator}s. + * + * @author incubos + */ +final class MergingEntryIterator implements Iterator> { + private final Queue iterators; + + MergingEntryIterator(final List iterators) { + assert iterators.stream().allMatch(WeightedPeekingEntryIterator::hasNext); + + this.iterators = new PriorityQueue<>(iterators); + } + + @Override + public boolean hasNext() { + return !iterators.isEmpty(); + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + final WeightedPeekingEntryIterator top = iterators.remove(); + final Entry result = top.next(); + + if (top.hasNext()) { + // Not exhausted + iterators.add(top); + } + + // Remove older versions of the key + while (true) { + final WeightedPeekingEntryIterator iterator = iterators.peek(); + if (iterator == null) { + // Nothing left + break; + } + + // Skip entries with the same key + final Entry entry = iterator.peek(); + if (MemorySegmentComparator.INSTANCE.compare(result.key(), entry.key()) != 0) { + // Reached another key + break; + } + + // Drop + iterators.remove(); + // Skip + iterator.next(); + if (iterator.hasNext()) { + // Not exhausted + iterators.add(iterator); + } + } + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/reference/ReferenceDao.java b/src/main/java/ru/vk/itmo/reference/ReferenceDao.java new file mode 100644 index 000000000..1531f78e2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/ReferenceDao.java @@ -0,0 +1,292 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Reference implementation of {@link Dao}. + * + * @author incubos + */ +public class ReferenceDao implements Dao> { + private final Config config; + private final Arena arena; + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + // Guarded by lock + private volatile TableSet tableSet; + + private final ExecutorService flusher = + Executors.newSingleThreadExecutor(r -> { + final Thread result = new Thread(r); + result.setName("flusher"); + return result; + }); + private final ExecutorService compactor = + Executors.newSingleThreadExecutor(r -> { + final Thread result = new Thread(r); + result.setName("compactor"); + return result; + }); + + private final AtomicBoolean closed = new AtomicBoolean(); + + public ReferenceDao(final Config config) throws IOException { + this.config = config; + this.arena = Arena.ofShared(); + + // First complete promotion of compacted SSTables + SSTables.promote( + config.basePath(), + 0, + 1); + + this.tableSet = + TableSet.from( + SSTables.discover( + arena, + config.basePath())); + } + + @Override + public Iterator> get( + final MemorySegment from, + final MemorySegment to) { + return new LiveFilteringIterator( + tableSet.get( + from, + to)); + } + + @Override + public Entry get(final MemorySegment key) { + // Without lock, just snapshot of table set + return tableSet.get(key); + } + + @Override + public void upsert(final Entry entry) { + final boolean autoFlush; + lock.readLock().lock(); + try { + if (tableSet.memTableSize.get() > config.flushThresholdBytes() + && tableSet.flushingTable != null) { + throw new IllegalStateException("Can't keep up with flushing!"); + } + + // Upsert + final Entry previous = tableSet.upsert(entry); + + // Update size estimate + final long size = tableSet.memTableSize.addAndGet(sizeOf(entry) - sizeOf(previous)); + autoFlush = size > config.flushThresholdBytes(); + } finally { + lock.readLock().unlock(); + } + + if (autoFlush) { + initiateFlush(true); + } + } + + private static long sizeOf(final Entry entry) { + if (entry == null) { + return 0L; + } + + if (entry.value() == null) { + return entry.key().byteSize(); + } + + return entry.key().byteSize() + entry.value().byteSize(); + } + + private void initiateFlush(final boolean auto) { + flusher.submit(() -> { + final TableSet currentTableSet; + lock.writeLock().lock(); + try { + if (this.tableSet.memTable.isEmpty()) { + // Nothing to flush + return; + } + + if (auto && this.tableSet.memTableSize.get() < config.flushThresholdBytes()) { + // Not enough data to flush + return; + } + + // Switch memTable to flushing + currentTableSet = this.tableSet.flushing(); + this.tableSet = currentTableSet; + } finally { + lock.writeLock().unlock(); + } + + // Write + final int sequence = currentTableSet.nextSequence(); + try { + new SSTableWriter() + .write( + config.basePath(), + sequence, + currentTableSet.flushingTable.get(null, null)); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-1); + return; + } + + // Open + final SSTable flushed; + try { + flushed = SSTables.open( + arena, + config.basePath(), + sequence); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-2); + return; + } + + // Switch + lock.writeLock().lock(); + try { + this.tableSet = this.tableSet.flushed(flushed); + } finally { + lock.writeLock().unlock(); + } + }).state(); + } + + @Override + public void flush() throws IOException { + initiateFlush(false); + } + + @Override + public void compact() throws IOException { + compactor.submit(() -> { + final TableSet currentTableSet; + lock.writeLock().lock(); + try { + currentTableSet = this.tableSet; + if (currentTableSet.ssTables.size() < 2) { + // Nothing to compact + return; + } + } finally { + lock.writeLock().unlock(); + } + + // Compact to 0 + try { + new SSTableWriter() + .write( + config.basePath(), + 0, + new LiveFilteringIterator( + currentTableSet.allSSTableEntries())); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-3); + } + + // Open 0 + final SSTable compacted; + try { + compacted = + SSTables.open( + arena, + config.basePath(), + 0); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-4); + return; + } + + // Replace old SSTables with compacted one to + // keep serving requests + final Set replaced = new HashSet<>(currentTableSet.ssTables); + lock.writeLock().lock(); + try { + this.tableSet = + this.tableSet.compacted( + replaced, + compacted); + } finally { + lock.writeLock().unlock(); + } + + // Remove compacted SSTables starting from the oldest ones. + // If we crash, 0 contains all the data, and + // it will be promoted on reopen. + for (final SSTable ssTable : currentTableSet.ssTables.reversed()) { + try { + SSTables.remove( + config.basePath(), + ssTable.sequence); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-5); + } + } + + // Promote zero to one (possibly replacing) + try { + SSTables.promote( + config.basePath(), + 0, + 1); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-6); + } + + // Replace promoted SSTable + lock.writeLock().lock(); + try { + this.tableSet = + this.tableSet.compacted( + Collections.singleton(compacted), + compacted.withSequence(1)); + } finally { + lock.writeLock().unlock(); + } + }).state(); + } + + @Override + public void close() throws IOException { + if (closed.getAndSet(true)) { + // Already closed + return; + } + + // Maybe flush + flush(); + + // Stop all the threads + flusher.close(); + compactor.close(); + + // Close arena + arena.close(); + } +} diff --git a/src/main/java/ru/vk/itmo/reference/SSTable.java b/src/main/java/ru/vk/itmo/reference/SSTable.java new file mode 100644 index 000000000..02217ab65 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/SSTable.java @@ -0,0 +1,204 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Collections; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Persistent SSTable in data file and index file. + * + * @author incubos + * @see SSTables + */ +final class SSTable { + final int sequence; + + private final MemorySegment index; + private final MemorySegment data; + private final long size; + + SSTable( + final int sequence, + final MemorySegment index, + final MemorySegment data) { + this.sequence = sequence; + this.index = index; + this.data = data; + this.size = index.byteSize() / Long.BYTES; + } + + SSTable withSequence(final int sequence) { + return new SSTable( + sequence, + index, + data); + } + + /** + * Returns index of the entry if found; otherwise, (-(insertion point) - 1). + * The insertion point is defined as the point at which the key would be inserted: + * the index of the first element greater than the key, + * or size if all keys are less than the specified key. + * Note that this guarantees that the return value will be >= 0 + * if and only if the key is found. + */ + private long entryBinarySearch(final MemorySegment key) { + long low = 0L; + long high = size - 1; + + while (low <= high) { + final long mid = (low + high) >>> 1; + final long midEntryOffset = entryOffset(mid); + final long midKeyLength = getLength(midEntryOffset); + final int compare = + MemorySegmentComparator.compare( + data, + midEntryOffset + Long.BYTES, // Position at key + midKeyLength, + key, + 0L, + key.byteSize()); + + if (compare < 0) { + low = mid + 1; + } else if (compare > 0) { + high = mid - 1; + } else { + return mid; + } + } + + return -(low + 1); + } + + private long entryOffset(final long entry) { + return index.get( + ValueLayout.OfLong.JAVA_LONG, + entry * Long.BYTES); + } + + private long getLength(final long offset) { + return data.get( + ValueLayout.OfLong.JAVA_LONG_UNALIGNED, + offset); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + assert from == null || to == null || MemorySegmentComparator.INSTANCE.compare(from, to) <= 0; + + // Slice of SSTable in absolute offsets + final long fromOffset; + final long toOffset; + + // Left offset bound + if (from == null) { + // Start from the beginning + fromOffset = 0L; + } else { + final long fromEntry = entryBinarySearch(from); + if (fromEntry >= 0L) { + fromOffset = entryOffset(fromEntry); + } else if (-fromEntry - 1 == size) { + // No relevant data + return Collections.emptyIterator(); + } else { + // Greater but existing key found + fromOffset = entryOffset(-fromEntry - 1); + } + } + + // Right offset bound + if (to == null) { + // Up to the end + toOffset = data.byteSize(); + } else { + final long toEntry = entryBinarySearch(to); + if (toEntry >= 0L) { + toOffset = entryOffset(toEntry); + } else if (-toEntry - 1 == size) { + // Up to the end + toOffset = data.byteSize(); + } else { + // Greater but existing key found + toOffset = entryOffset(-toEntry - 1); + } + } + + return new SliceIterator(fromOffset, toOffset); + } + + Entry get(final MemorySegment key) { + final long entry = entryBinarySearch(key); + if (entry < 0) { + return null; + } + + // Skip key (will reuse the argument) + long offset = entryOffset(entry); + offset += Long.BYTES + key.byteSize(); + // Extract value length + final long valueLength = getLength(offset); + if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { + // Tombstone encountered + return new BaseEntry<>(key, null); + } else { + // Get value + offset += Long.BYTES; + final MemorySegment value = data.asSlice(offset, valueLength); + return new BaseEntry<>(key, value); + } + } + + private final class SliceIterator implements Iterator> { + private long offset; + private final long toOffset; + + private SliceIterator( + final long offset, + final long toOffset) { + this.offset = offset; + this.toOffset = toOffset; + } + + @Override + public boolean hasNext() { + return offset < toOffset; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + // Read key length + final long keyLength = getLength(offset); + offset += Long.BYTES; + + // Read key + final MemorySegment key = data.asSlice(offset, keyLength); + offset += keyLength; + + // Read value length + final long valueLength = getLength(offset); + offset += Long.BYTES; + + // Read value + if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { + // Tombstone encountered + return new BaseEntry<>(key, null); + } else { + final MemorySegment value = data.asSlice(offset, valueLength); + offset += valueLength; + return new BaseEntry<>(key, value); + } + } + } +} diff --git a/src/main/java/ru/vk/itmo/reference/SSTableWriter.java b/src/main/java/ru/vk/itmo/reference/SSTableWriter.java new file mode 100644 index 000000000..fa8b6612e --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/SSTableWriter.java @@ -0,0 +1,166 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Entry; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Iterator; + +/** + * Writes {@link Entry} {@link Iterator} to SSTable on disk. + * + *

Index file {@code .index} contains {@code long} offsets to entries in data file: + * {@code [offset0, offset1, ...]} + * + *

Data file {@code .data} contains serialized entries: + * {@code } + * + *

Tombstones are encoded as {@code valueLength} {@code -1} and no subsequent value. + * + * @author incubos + */ +final class SSTableWriter { + private static final int BUFFER_SIZE = 64 * 1024; + + // Reusable buffers to eliminate allocations. + // But excessive memory copying is still there :( + // Long cell + private final ByteArraySegment longBuffer = new ByteArraySegment(Long.BYTES); + // Growable blob cell + private final ByteArraySegment blobBuffer = new ByteArraySegment(512); + + void write( + final Path baseDir, + final int sequence, + final Iterator> entries) throws IOException { + // Write to temporary files + final Path tempIndexName = SSTables.tempIndexName(baseDir, sequence); + final Path tempDataName = SSTables.tempDataName(baseDir, sequence); + + // Delete temporary files to eliminate tails + Files.deleteIfExists(tempIndexName); + Files.deleteIfExists(tempDataName); + + // Iterate in a single pass! + // Will write through FileChannel despite extra memory copying and + // no buffering (which may be implemented later). + // Looking forward to MemorySegment facilities in FileChannel! + try (OutputStream index = + new BufferedOutputStream( + new FileOutputStream( + tempIndexName.toFile()), + BUFFER_SIZE); + OutputStream data = + new BufferedOutputStream( + new FileOutputStream( + tempDataName.toFile()), + BUFFER_SIZE)) { + long entryOffset = 0L; + + // Iterate and serialize + while (entries.hasNext()) { + // First write offset to the entry + writeLong(entryOffset, index); + + // Then write the entry + final Entry entry = entries.next(); + entryOffset += writeEntry(entry, data); + } + } + + // Publish files atomically + // FIRST index, LAST data + final Path indexName = + SSTables.indexName( + baseDir, + sequence); + Files.move( + tempIndexName, + indexName, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + final Path dataName = + SSTables.dataName( + baseDir, + sequence); + Files.move( + tempDataName, + dataName, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + + private void writeLong( + final long value, + final OutputStream os) throws IOException { + longBuffer.segment().set( + ValueLayout.OfLong.JAVA_LONG_UNALIGNED, + 0, + value); + longBuffer.withArray(os::write); + } + + private void writeSegment( + final MemorySegment value, + final OutputStream os) throws IOException { + final long size = value.byteSize(); + blobBuffer.ensureCapacity(size); + MemorySegment.copy( + value, + 0L, + blobBuffer.segment(), + 0L, + size); + blobBuffer.withArray(array -> + os.write( + array, + 0, + (int) size)); + } + + /** + * Writes {@link Entry} to {@link FileChannel}. + * + * @return written bytes + */ + private long writeEntry( + final Entry entry, + final OutputStream os) throws IOException { + final MemorySegment key = entry.key(); + final MemorySegment value = entry.value(); + long result = 0L; + + // Key size + writeLong(key.byteSize(), os); + result += Long.BYTES; + + // Key + writeSegment(key, os); + result += key.byteSize(); + + // Value size and possibly value + if (value == null) { + // Tombstone + writeLong(SSTables.TOMBSTONE_VALUE_LENGTH, os); + result += Long.BYTES; + } else { + // Value length + writeLong(value.byteSize(), os); + result += Long.BYTES; + + // Value + writeSegment(value, os); + result += value.byteSize(); + } + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/reference/SSTables.java b/src/main/java/ru/vk/itmo/reference/SSTables.java new file mode 100644 index 000000000..b154c6b34 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/SSTables.java @@ -0,0 +1,162 @@ +package ru.vk.itmo.reference; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +/** + * Provides {@link SSTable} management facilities: dumping and discovery. + * + * @author incubos + */ +final class SSTables { + public static final String INDEX_SUFFIX = ".index"; + public static final String DATA_SUFFIX = ".data"; + public static final long TOMBSTONE_VALUE_LENGTH = -1L; + + private static final String TEMP_SUFFIX = ".tmp"; + + /** + * Can't instantiate. + */ + private SSTables() { + // Only static methods + } + + static Path indexName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + INDEX_SUFFIX); + } + + static Path dataName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + DATA_SUFFIX); + } + + static Path tempIndexName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + INDEX_SUFFIX + TEMP_SUFFIX); + } + + static Path tempDataName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + DATA_SUFFIX + TEMP_SUFFIX); + } + + /** + * Returns {@link List} of {@link SSTable}s from freshest to oldest. + */ + static List discover( + final Arena arena, + final Path baseDir) throws IOException { + if (!Files.exists(baseDir)) { + return Collections.emptyList(); + } + + final List result = new ArrayList<>(); + try (Stream files = Files.list(baseDir)) { + files.forEach(file -> { + final String fileName = file.getFileName().toString(); + if (!fileName.endsWith(DATA_SUFFIX)) { + // Skip non data + return; + } + + final int sequence = + // .data -> N + Integer.parseInt( + fileName.substring( + 0, + fileName.length() - DATA_SUFFIX.length())); + + try { + result.add(open(arena, baseDir, sequence)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + // Sort from freshest to oldest + result.sort((o1, o2) -> Integer.compare(o2.sequence, o1.sequence)); + + return Collections.unmodifiableList(result); + } + + static SSTable open( + final Arena arena, + final Path baseDir, + final int sequence) throws IOException { + final MemorySegment index = + mapReadOnly( + arena, + indexName(baseDir, sequence)); + final MemorySegment data = + mapReadOnly( + arena, + dataName(baseDir, sequence)); + + return new SSTable( + sequence, + index, + data); + } + + private static MemorySegment mapReadOnly( + final Arena arena, + final Path file) throws IOException { + try (FileChannel channel = + FileChannel.open( + file, + StandardOpenOption.READ)) { + return channel.map( + FileChannel.MapMode.READ_ONLY, + 0L, + Files.size(file), + arena); + } + } + + static void remove( + final Path baseDir, + final int sequence) throws IOException { + // First delete data file to make SSTable invisible + Files.delete(dataName(baseDir, sequence)); + Files.delete(indexName(baseDir, sequence)); + } + + static void promote( + final Path baseDir, + final int from, + final int to) throws IOException { + // Build to progress to the same outcome + if (Files.exists(indexName(baseDir, from))) { + Files.move( + indexName(baseDir, from), + indexName(baseDir, to), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + if (Files.exists(dataName(baseDir, from))) { + Files.move( + dataName(baseDir, from), + dataName(baseDir, to), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + } +} diff --git a/src/main/java/ru/vk/itmo/reference/TableSet.java b/src/main/java/ru/vk/itmo/reference/TableSet.java new file mode 100644 index 000000000..ab249e8bd --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/TableSet.java @@ -0,0 +1,205 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Data set in various tables. + * + * @author incubos + */ +final class TableSet { + final MemTable memTable; + final AtomicLong memTableSize; + // null or read-only + final MemTable flushingTable; + // From freshest to oldest + final List ssTables; + + private TableSet( + final MemTable memTable, + final AtomicLong memTableSize, + final MemTable flushingTable, + final List ssTables) { + this.memTable = memTable; + this.memTableSize = memTableSize; + this.flushingTable = flushingTable; + this.ssTables = ssTables; + } + + static TableSet from(final List ssTables) { + return new TableSet( + new MemTable(), + new AtomicLong(), + null, + ssTables); + } + + int nextSequence() { + return ssTables.stream() + .mapToInt(t -> t.sequence) + .max() + .orElse(0) + 1; + } + + TableSet flushing() { + if (memTable.isEmpty()) { + throw new IllegalStateException("Nothing to flush"); + } + + if (flushingTable != null) { + throw new IllegalStateException("Already flushing"); + } + + return new TableSet( + new MemTable(), + new AtomicLong(), + memTable, + ssTables); + } + + TableSet flushed(final SSTable flushed) { + final List newSSTables = new ArrayList<>(ssTables.size() + 1); + newSSTables.add(flushed); + newSSTables.addAll(ssTables); + return new TableSet( + memTable, + memTableSize, + null, + newSSTables); + } + + TableSet compacted( + final Set replaced, + final SSTable with) { + final List newSsTables = new ArrayList<>(this.ssTables.size() + 1); + + // Keep not replaced SSTables + for (final SSTable ssTable : this.ssTables) { + if (!replaced.contains(ssTable)) { + newSsTables.add(ssTable); + } + } + + // Logically the oldest one + newSsTables.add(with); + + return new TableSet( + memTable, + memTableSize, + flushingTable, + newSsTables); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + final List iterators = + new ArrayList<>(2 + ssTables.size()); + + // MemTable goes first + final Iterator> memTableIterator = + memTable.get(from, to); + if (memTableIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + Integer.MIN_VALUE, + memTableIterator)); + } + + // Then goes flushing + if (flushingTable != null) { + final Iterator> flushingIterator = + flushingTable.get(from, to); + if (flushingIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + Integer.MIN_VALUE + 1, + flushingIterator)); + } + } + + // Then go all the SSTables + for (int i = 0; i < ssTables.size(); i++) { + final SSTable ssTable = ssTables.get(i); + final Iterator> ssTableIterator = + ssTable.get(from, to); + if (ssTableIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + i, + ssTableIterator)); + } + } + + return switch (iterators.size()) { + case 0 -> Collections.emptyIterator(); + case 1 -> iterators.get(0); + default -> new MergingEntryIterator(iterators); + }; + } + + Entry get(final MemorySegment key) { + // Slightly optimized version not to pollute the heap + + // First check MemTable + Entry result = memTable.get(key); + if (result != null) { + // Transform tombstone + return swallowTombstone(result); + } + + // Then check flushing + if (flushingTable != null) { + result = flushingTable.get(key); + if (result != null) { + // Transform tombstone + return swallowTombstone(result); + } + } + + // At last check SSTables from freshest to oldest + for (final SSTable ssTable : ssTables) { + result = ssTable.get(key); + if (result != null) { + // Transform tombstone + return swallowTombstone(result); + } + } + + // Nothing found + return null; + } + + private static Entry swallowTombstone(final Entry entry) { + return entry.value() == null ? null : entry; + } + + Entry upsert(final Entry entry) { + return memTable.upsert(entry); + } + + Iterator> allSSTableEntries() { + final List iterators = + new ArrayList<>(ssTables.size()); + + for (int i = 0; i < ssTables.size(); i++) { + final SSTable ssTable = ssTables.get(i); + final Iterator> ssTableIterator = + ssTable.get(null, null); + iterators.add( + new WeightedPeekingEntryIterator( + i, + ssTableIterator)); + } + + return new MergingEntryIterator(iterators); + } +} diff --git a/src/main/java/ru/vk/itmo/reference/WeightedPeekingEntryIterator.java b/src/main/java/ru/vk/itmo/reference/WeightedPeekingEntryIterator.java new file mode 100644 index 000000000..683bd1179 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reference/WeightedPeekingEntryIterator.java @@ -0,0 +1,67 @@ +package ru.vk.itmo.reference; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Peeking {@link Iterator} wrapper. + * + * @author incubos + */ +final class WeightedPeekingEntryIterator + implements Iterator>, + Comparable { + private final int weight; + private final Iterator> delegate; + private Entry next; + + WeightedPeekingEntryIterator( + final int weight, + final Iterator> delegate) { + this.weight = weight; + this.delegate = delegate; + this.next = delegate.hasNext() ? delegate.next() : null; + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + final Entry result = next; + next = delegate.hasNext() ? delegate.next() : null; + return result; + } + + Entry peek() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + return next; + } + + @Override + public int compareTo(final WeightedPeekingEntryIterator other) { + // First compare keys + int result = + MemorySegmentComparator.INSTANCE.compare( + peek().key(), + other.peek().key()); + if (result != 0) { + return result; + } + + // Then compare weights if keys are equal + return Integer.compare(weight, other.weight); + } +} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/DaoImpl.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/DaoImpl.java new file mode 100644 index 000000000..fa65481d7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/DaoImpl.java @@ -0,0 +1,97 @@ +package ru.vk.itmo.reshetnikovaleksei; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; +import ru.vk.itmo.reshetnikovaleksei.iterators.MergeIterator; +import ru.vk.itmo.reshetnikovaleksei.iterators.PeekingIterator; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +public class DaoImpl implements Dao> { + private final ConcurrentNavigableMap> memoryTable; + private final SSTableManager ssTableManager; + + public DaoImpl(Config config) throws IOException { + this.memoryTable = new ConcurrentSkipListMap<>(MemorySegmentComparator.getInstance()); + this.ssTableManager = new SSTableManager(config); + } + + @Override + public Entry get(MemorySegment key) { + Iterator> iterator = allIterators(key, null); + + if (iterator.hasNext()) { + Entry result = iterator.next(); + if (MemorySegmentComparator.getInstance().compare(key, result.key()) == 0) { + return result.value() == null ? null : result; + } + } + + return null; + } + + @Override + public Iterator> get(MemorySegment from, MemorySegment to) { + return allIterators(from, to); + } + + @Override + public void upsert(Entry entry) { + memoryTable.put(entry.key(), entry); + } + + @Override + public void close() throws IOException { + if (!memoryTable.isEmpty()) { + ssTableManager.save(memoryTable.values()); + memoryTable.clear(); + } + + ssTableManager.close(); + } + + @Override + public void compact() throws IOException { + ssTableManager.compact(() -> get(null, null)); + memoryTable.clear(); + } + + private Iterator> allIterators(MemorySegment from, MemorySegment to) { + List iterators = new ArrayList<>(); + + Iterator> memoryIterator = memoryIterator(from, to); + Iterator> filesIterator = ssTableManager.get(from, to); + + iterators.add(new PeekingIterator(memoryIterator, 1)); + iterators.add(new PeekingIterator(filesIterator, 0)); + + return new PeekingIterator( + MergeIterator.merge( + iterators, + MemorySegmentComparator.getInstance() + ) + ); + } + + private Iterator> memoryIterator(MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return memoryTable.values().iterator(); + } + + if (from == null) { + return memoryTable.headMap(to).values().iterator(); + } + if (to == null) { + return memoryTable.tailMap(from).values().iterator(); + } + + return memoryTable.subMap(from, to).values().iterator(); + } +} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/InMemoryDao.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/InMemoryDao.java deleted file mode 100644 index d393b0ea8..000000000 --- a/src/main/java/ru/vk/itmo/reshetnikovaleksei/InMemoryDao.java +++ /dev/null @@ -1,58 +0,0 @@ -package ru.vk.itmo.reshetnikovaleksei; - -import ru.vk.itmo.Dao; -import ru.vk.itmo.Entry; - -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.util.Iterator; -import java.util.concurrent.ConcurrentSkipListMap; - -public class InMemoryDao implements Dao> { - private final ConcurrentSkipListMap> map; - - public InMemoryDao() { - this.map = new ConcurrentSkipListMap<>((a, b) -> { - var offset = a.mismatch(b); - - if (offset == -1) { - return 0; - } else if (offset == a.byteSize()) { - return -1; - } else if (offset == b.byteSize()) { - return 1; - } else { - return Byte.compare( - a.get(ValueLayout.JAVA_BYTE, offset), - b.get(ValueLayout.JAVA_BYTE, offset) - ); - } - }); - } - - @Override - public Iterator> get(MemorySegment from, MemorySegment to) { - if (from == null && to == null) { - return map.values().iterator(); - } - - if (from == null) { - return map.headMap(to).values().iterator(); - } - if (to == null) { - return map.tailMap(from).values().iterator(); - } - - return map.subMap(from, to).values().iterator(); - } - - @Override - public Entry get(MemorySegment key) { - return map.get(key); - } - - @Override - public void upsert(Entry entry) { - map.put(entry.key(), entry); - } -} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/MemorySegmentComparator.java new file mode 100644 index 000000000..30a2b08a3 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/MemorySegmentComparator.java @@ -0,0 +1,58 @@ +package ru.vk.itmo.reshetnikovaleksei; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Comparator; + +public final class MemorySegmentComparator implements Comparator { + private static MemorySegmentComparator instance; + + private MemorySegmentComparator() { + } + + public static synchronized MemorySegmentComparator getInstance() { + if (instance == null) { + instance = new MemorySegmentComparator(); + } + + return instance; + } + + @Override + public int compare(MemorySegment a, MemorySegment b) { + var offset = a.mismatch(b); + + if (offset == -1) { + return 0; + } else if (offset == a.byteSize()) { + return -1; + } else if (offset == b.byteSize()) { + return 1; + } else { + return Byte.compare( + a.get(ValueLayout.JAVA_BYTE, offset), + b.get(ValueLayout.JAVA_BYTE, offset) + ); + } + } + + public int compare(MemorySegment a, MemorySegment b, long fromOffset, long toOffset) { + long mismatch = MemorySegment.mismatch( + b, fromOffset, toOffset, + a, 0, a.byteSize() + ); + + if (mismatch == -1) { + return 0; + } else if (mismatch == a.byteSize()) { + return -1; + } else if (mismatch == b.byteSize()) { + return 1; + } + + return Byte.compare( + a.get(ValueLayout.JAVA_BYTE, mismatch), + b.get(ValueLayout.JAVA_BYTE, mismatch + fromOffset) + ); + } +} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTable.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTable.java new file mode 100644 index 000000000..4d4ec2d03 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTable.java @@ -0,0 +1,81 @@ +package ru.vk.itmo.reshetnikovaleksei; + +import ru.vk.itmo.Entry; +import ru.vk.itmo.reshetnikovaleksei.iterators.SSTableIterator; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Iterator; + +public class SSTable { + public static final String DATA_PREFIX = "data-"; + public static final String DATA_TMP = "data.tmp"; + public static final String INDEX_PREFIX = "index-"; + public static final String INDEX_TMP = "index.tmp"; + + private final MemorySegment dataSegment; + private final MemorySegment indexSegment; + private final Path dataPath; + private final Path indexPath; + + public SSTable(Path basePath, Arena arena, long idx) throws IOException { + this.dataPath = basePath.resolve(DATA_PREFIX + idx); + this.indexPath = basePath.resolve(INDEX_PREFIX + idx); + + try (FileChannel dataChannel = FileChannel.open(dataPath, StandardOpenOption.READ)) { + this.dataSegment = dataChannel.map(FileChannel.MapMode.READ_ONLY, 0, dataChannel.size(), arena); + } + try (FileChannel indexChannel = FileChannel.open(indexPath, StandardOpenOption.READ)) { + this.indexSegment = indexChannel.map(FileChannel.MapMode.READ_ONLY, 0, indexChannel.size(), arena); + } + } + + public Iterator> iterator(MemorySegment from, MemorySegment to) { + long indexFrom; + + if (from == null) { + indexFrom = 0; + } else { + indexFrom = getIndexOffsetByKey(from); + } + + return new SSTableIterator(indexFrom, to, dataSegment, indexSegment); + } + + public void deleteFiles() throws IOException { + Files.deleteIfExists(dataPath); + Files.deleteIfExists(indexPath); + } + + private long getIndexOffsetByKey(MemorySegment key) { + long l = 0; + long r = indexSegment.byteSize() / Long.BYTES - 1; + + while (l <= r) { + long m = l + (r - l) / 2; + + long dataOffset = indexSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, m * Long.BYTES); + long currKeySize = dataSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset); + dataOffset += Long.BYTES; + + long comparing = MemorySegmentComparator.getInstance().compare( + key, dataSegment, dataOffset, dataOffset + currKeySize); + if (comparing > 0) { + l = m + 1; + } else if (comparing < 0) { + r = m - 1; + } else { + return m * Long.BYTES; + } + + } + + return l * Long.BYTES; + } +} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTableManager.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTableManager.java new file mode 100644 index 000000000..c3fc3519a --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/SSTableManager.java @@ -0,0 +1,182 @@ +package ru.vk.itmo.reshetnikovaleksei; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Entry; +import ru.vk.itmo.reshetnikovaleksei.iterators.MergeIterator; +import ru.vk.itmo.reshetnikovaleksei.iterators.PeekingIterator; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Stream; + +import static ru.vk.itmo.reshetnikovaleksei.SSTable.DATA_PREFIX; +import static ru.vk.itmo.reshetnikovaleksei.SSTable.DATA_TMP; +import static ru.vk.itmo.reshetnikovaleksei.SSTable.INDEX_PREFIX; +import static ru.vk.itmo.reshetnikovaleksei.SSTable.INDEX_TMP; + +public class SSTableManager implements AutoCloseable { + + private final Arena arena; + private final Path basePath; + private final List ssTables; + + private int lastIdx; + private boolean isClosed; + + public SSTableManager(Config config) throws IOException { + this.arena = Arena.ofShared(); + this.basePath = config.basePath(); + this.ssTables = new ArrayList<>(); + + this.lastIdx = 0; + this.isClosed = false; + + if (!Files.exists(basePath)) { + return; + } + + long filesCount; + try (Stream filesStream = Files.list(basePath)) { + filesCount = filesStream.count(); + } + + for (int i = 0; i < filesCount; i++) { + try { + ssTables.add(new SSTable(basePath, arena, i)); + } catch (IOException e) { + lastIdx = i; + } + } + } + + public Iterator> get(MemorySegment key) { + return get(key, null); + } + + public Iterator> get(MemorySegment from, MemorySegment to) { + List iterators = new ArrayList<>(); + + int priority = 1; + for (SSTable ssTable : ssTables) { + iterators.add(new PeekingIterator(ssTable.iterator(from, to), priority)); + priority++; + } + + return MergeIterator.merge(iterators, MemorySegmentComparator.getInstance()); + } + + public void save(Iterable> entries) throws IOException { + Path tmpDataPath = basePath.resolve(DATA_TMP); + Path tmpIndexPath = basePath.resolve(INDEX_TMP); + + Path dataPath = basePath.resolve(DATA_PREFIX + lastIdx); + Path indexPath = basePath.resolve(INDEX_PREFIX + lastIdx); + + try ( + FileChannel dataChannel = FileChannel.open( + tmpDataPath, + StandardOpenOption.READ, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE + ); + FileChannel indexChannel = FileChannel.open( + tmpIndexPath, + StandardOpenOption.READ, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE + ); + Arena writeDataArena = Arena.ofConfined() + ) { + long dataSize = Long.BYTES; + long indexSize = 0; + for (Entry entry : entries) { + dataSize += entry.key().byteSize() + + (entry.value() == null ? 0 : entry.value().byteSize()) + + 2 * Long.BYTES; + indexSize += Long.BYTES; + } + + long dataOffset = 0; + long indexOffset = 0; + + MemorySegment dataSegment = dataChannel.map( + FileChannel.MapMode.READ_WRITE, dataOffset, dataSize, writeDataArena); + dataSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset, dataSize - Long.BYTES); + dataOffset += Long.BYTES; + + MemorySegment indexSegment = indexChannel.map( + FileChannel.MapMode.READ_WRITE, indexOffset, indexSize, writeDataArena); + + for (Entry entry : entries) { + indexSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + indexOffset += Long.BYTES; + + dataSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset, entry.key().byteSize()); + dataOffset += Long.BYTES; + MemorySegment.copy(entry.key(), 0, dataSegment, dataOffset, entry.key().byteSize()); + dataOffset += entry.key().byteSize(); + + if (entry.value() == null) { + dataSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset, -1); + dataOffset += Long.BYTES; + } else { + dataSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset, entry.value().byteSize()); + dataOffset += Long.BYTES; + MemorySegment.copy(entry.value(), 0, dataSegment, dataOffset, entry.value().byteSize()); + dataOffset += entry.value().byteSize(); + } + } + } + + moveDataFromTmpToReal(tmpDataPath, dataPath); + moveDataFromTmpToReal(tmpIndexPath, indexPath); + } + + public void compact(Iterable> entries) throws IOException { + save(entries); + + Path dataPath = basePath.resolve(DATA_PREFIX + lastIdx); + Path indexPath = basePath.resolve(INDEX_PREFIX + lastIdx); + deleteAllFiles(); + + moveDataFromTmpToReal(dataPath, basePath.resolve(DATA_PREFIX + lastIdx)); + moveDataFromTmpToReal(indexPath, basePath.resolve(INDEX_PREFIX + lastIdx)); + } + + @Override + public void close() { + if (!isClosed) { + arena.close(); + isClosed = true; + } + } + + private void moveDataFromTmpToReal(Path tmpFilePath, Path realFilePath) throws IOException { + try { + Files.createFile(realFilePath); + } catch (FileAlreadyExistsException ignored) { + // do nothing + } + + Files.move(tmpFilePath, realFilePath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } + + private void deleteAllFiles() throws IOException { + for (SSTable ssTable : ssTables) { + ssTable.deleteFiles(); + } + + lastIdx = 0; + } +} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/MergeIterator.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/MergeIterator.java new file mode 100644 index 000000000..a2cedd7e2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/MergeIterator.java @@ -0,0 +1,84 @@ +package ru.vk.itmo.reshetnikovaleksei.iterators; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.Queue; + +public final class MergeIterator implements Iterator> { + private final Queue queue; + + private MergeIterator(Queue queue) { + this.queue = queue; + } + + public static Iterator> merge( + List iterators, Comparator comparator) { + if (iterators.isEmpty()) { + return Collections.emptyIterator(); + } + + Queue queue = new PriorityQueue<>( + iterators.size(), + Comparator.comparing((PeekingIterator iter) -> iter.peek().key(), comparator) + .thenComparing(PeekingIterator::priority, Comparator.reverseOrder()) + ); + + for (PeekingIterator iterator : iterators) { + if (iterator.hasNext()) { + queue.add(iterator); + } + } + + return new MergeIterator(queue); + } + + @Override + public boolean hasNext() { + skipDeletedEntry(); + return !queue.isEmpty(); + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException("no next element"); + } + + PeekingIterator currIterator = queue.remove(); + Entry currEntry = currIterator.next(); + removeOldEntryByKey(currEntry.key()); + if (currIterator.hasNext()) { + queue.add(currIterator); + } + + return currEntry; + } + + private void skipDeletedEntry() { + while (!queue.isEmpty() && queue.peek().peek().value() == null) { + PeekingIterator currentItr = queue.remove(); + Entry current = currentItr.next(); + removeOldEntryByKey(current.key()); + if (currentItr.hasNext()) { + queue.add(currentItr); + } + } + } + + private void removeOldEntryByKey(MemorySegment key) { + while (!queue.isEmpty() && queue.peek().peek().key().mismatch(key) == -1) { + PeekingIterator currentItr = queue.remove(); + currentItr.next(); + if (currentItr.hasNext()) { + queue.add(currentItr); + } + } + } +} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/PeekingIterator.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/PeekingIterator.java new file mode 100644 index 000000000..a69e29d1f --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/PeekingIterator.java @@ -0,0 +1,51 @@ +package ru.vk.itmo.reshetnikovaleksei.iterators; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; + +public class PeekingIterator implements Iterator> { + private final Iterator> iterator; + private final int priority; + + private Entry next; + + public PeekingIterator(Iterator> iterator) { + this(iterator, 0); + } + + public PeekingIterator(Iterator> iterator, int priority) { + this.iterator = iterator; + this.priority = priority; + + if (iterator.hasNext()) { + next = iterator.next(); + } + } + + @Override + public boolean hasNext() { + return next != null || iterator.hasNext(); + } + + @Override + public Entry next() { + Entry toReturn = peek(); + next = null; + + return toReturn; + } + + public int priority() { + return priority; + } + + public Entry peek() { + if (next == null) { + next = iterator.next(); + } + + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/SSTableIterator.java b/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/SSTableIterator.java new file mode 100644 index 000000000..505e9a5fe --- /dev/null +++ b/src/main/java/ru/vk/itmo/reshetnikovaleksei/iterators/SSTableIterator.java @@ -0,0 +1,79 @@ +package ru.vk.itmo.reshetnikovaleksei.iterators; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; +import ru.vk.itmo.reshetnikovaleksei.MemorySegmentComparator; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class SSTableIterator implements Iterator> { + + private final MemorySegment dataSegment; + private final MemorySegment indexSegment; + private final MemorySegment to; + + private long indexOffset; + private long currentKeyOffset; + private long currentKeySize; + + public SSTableIterator(long indexOffset, MemorySegment to, MemorySegment dataSegment, MemorySegment indexSegment) { + this.indexOffset = indexOffset; + this.to = to; + this.dataSegment = dataSegment; + this.indexSegment = indexSegment; + + this.currentKeyOffset = -1; + this.currentKeySize = -1; + } + + @Override + public boolean hasNext() { + if (indexOffset == indexSegment.byteSize()) { + return false; + } + + if (to == null) { + return true; + } + currentKeyOffset = indexSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset); + currentKeySize = dataSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, currentKeyOffset); + long fromOffset = currentKeyOffset + Long.BYTES; + + return MemorySegmentComparator.getInstance() + .compare(to, dataSegment, fromOffset, fromOffset + currentKeySize) > 0; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException("No next element"); + } + + long keyOffset; + long keySize; + if (currentKeyOffset == -1 || currentKeySize == -1) { + keyOffset = indexSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset); + keySize = dataSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, keyOffset); + } else { + keyOffset = currentKeyOffset; + keySize = currentKeySize; + } + indexOffset += Long.BYTES; + keyOffset += Long.BYTES; + MemorySegment key = dataSegment.asSlice(keyOffset, keySize); + keyOffset += keySize; + + long valueSize = dataSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, keyOffset); + MemorySegment value; + if (valueSize == -1) { + value = null; + } else { + value = dataSegment.asSlice(keyOffset + Long.BYTES, valueSize); + } + + return new BaseEntry<>(key, value); + } +} diff --git a/src/main/java/ru/vk/itmo/shemetovalexey/DiskStorage.java b/src/main/java/ru/vk/itmo/shemetovalexey/DiskStorage.java deleted file mode 100644 index d28c2b7d6..000000000 --- a/src/main/java/ru/vk/itmo/shemetovalexey/DiskStorage.java +++ /dev/null @@ -1,40 +0,0 @@ -package ru.vk.itmo.shemetovalexey; - -import ru.vk.itmo.Entry; - -import java.lang.foreign.MemorySegment; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; - -public class DiskStorage { - - private final List segmentList; - - public DiskStorage(List segmentList) { - this.segmentList = segmentList; - } - - public Iterator> range( - Iterator> firstIterator, - MemorySegment from, - MemorySegment to) { - List>> iterators = new ArrayList<>(segmentList.size() + 1); - for (MemorySegment memorySegment : segmentList) { - iterators.add(StorageUtils.iterator(memorySegment, from, to)); - } - iterators.add(firstIterator); - - return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, InMemoryDao::compare)) { - @Override - protected boolean skip(Entry memorySegmentEntry) { - return memorySegmentEntry.value() == null; - } - }; - } - - public int getTotalFiles() { - return segmentList.size(); - } -} diff --git a/src/main/java/ru/vk/itmo/shemetovalexey/InMemoryDao.java b/src/main/java/ru/vk/itmo/shemetovalexey/InMemoryDao.java index 1f27e1d79..fc7d70445 100644 --- a/src/main/java/ru/vk/itmo/shemetovalexey/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/shemetovalexey/InMemoryDao.java @@ -3,60 +3,70 @@ import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; +import ru.vk.itmo.shemetovalexey.sstable.SSTable; +import ru.vk.itmo.shemetovalexey.sstable.SSTableIterator; +import ru.vk.itmo.shemetovalexey.sstable.SSTableStates; import java.io.IOException; +import java.io.InterruptedIOException; +import java.io.UncheckedIOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Collections; -import java.util.Comparator; +import java.util.Collection; import java.util.Iterator; -import java.util.NavigableMap; +import java.util.List; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class InMemoryDao implements Dao> { - - private final Comparator comparator = InMemoryDao::compare; - private final NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); + private static final String DATA = "data"; + private final ExecutorService bgExecutor = Executors.newSingleThreadExecutor(); + private final AtomicReference sstableState; private final Arena arena; - private final DiskStorage diskStorage; private final Path path; + private final AtomicLong size = new AtomicLong(); + private final long maxSize; + private final AtomicBoolean closed = new AtomicBoolean(); + private final ReadWriteLock upsertLock = new ReentrantReadWriteLock(); public InMemoryDao(Config config) throws IOException { - this.path = config.basePath().resolve("data"); - Files.createDirectories(path); - - arena = Arena.ofShared(); - - this.diskStorage = new DiskStorage(StorageUtils.loadOrRecover(path, arena)); - } + this.maxSize = config.flushThresholdBytes() == 0 ? config.flushThresholdBytes() : Long.MAX_VALUE / 2; - static int compare(MemorySegment memorySegment1, MemorySegment memorySegment2) { - long mismatch = memorySegment1.mismatch(memorySegment2); - if (mismatch == -1) { - return 0; - } + this.path = config.basePath().resolve(DATA); + Files.createDirectories(path); - if (mismatch == memorySegment1.byteSize()) { - return -1; - } + this.arena = Arena.ofShared(); - if (mismatch == memorySegment2.byteSize()) { - return 1; - } - byte b1 = memorySegment1.get(ValueLayout.JAVA_BYTE, mismatch); - byte b2 = memorySegment2.get(ValueLayout.JAVA_BYTE, mismatch); - return Byte.compare(b1, b2); + List segments = SSTable.loadOrRecover(path, arena); + this.sstableState = new AtomicReference<>(SSTableStates.create(segments)); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { - return diskStorage.range(getInMemory(from, to), from, to); + SSTableStates state = this.sstableState.get(); + return SSTableIterator.get( + getInMemory(state.getReadStorage(), from, to), + getInMemory(state.getWriteStorage(), from, to), + state.getDiskSegmentList(), + from, + to + ); } - private Iterator> getInMemory(MemorySegment from, MemorySegment to) { + private Iterator> getInMemory( + ConcurrentSkipListMap> storage, + MemorySegment from, + MemorySegment to + ) { if (from == null && to == null) { return storage.values().iterator(); } @@ -71,12 +81,25 @@ private Iterator> getInMemory(MemorySegment from, MemorySeg @Override public void upsert(Entry entry) { - storage.put(entry.key(), entry); + upsertLock.readLock().lock(); + try { + sstableState.get().getWriteStorage().put(entry.key(), entry); + long keySize = entry.key().byteSize(); + long valueSize = entry.value() == null ? 0 : entry.value().byteSize(); + if (size.addAndGet(keySize + valueSize) >= maxSize) { + flush(); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + upsertLock.readLock().unlock(); + } } @Override public Entry get(MemorySegment key) { - Entry entry = storage.get(key); + SSTableStates state = this.sstableState.get(); + Entry entry = state.getWriteStorage().get(key); if (entry != null) { if (entry.value() == null) { return null; @@ -84,36 +107,103 @@ public Entry get(MemorySegment key) { return entry; } - Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + entry = state.getReadStorage().get(key); + if (entry != null) { + if (entry.value() == null) { + return null; + } + return entry; + } + Iterator> iterator = SSTableIterator.get(state.getDiskSegmentList(), key); if (!iterator.hasNext()) { return null; } Entry next = iterator.next(); - if (compare(next.key(), key) == 0) { + if (MemorySegmentComparator.compare(next.key(), key) == 0) { return next; } return null; } @Override - public void compact() throws IOException { - if (storage.isEmpty() && diskStorage.getTotalFiles() <= 1) { - return; - } - StorageUtils.compact(path, this::all); + public void compact() { + bgExecutor.execute(() -> { + try { + SSTableStates state = this.sstableState.get(); + MemorySegment newPage = SSTable.compact( + arena, + path, + () -> SSTableIterator.get(state.getDiskSegmentList()) + ); + this.sstableState.set(state.compact(newPage)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + @Override + public void flush() throws IOException { + bgExecutor.execute(() -> { + ConcurrentSkipListMap> writeStorage = + sstableState.get().getWriteStorage(); + if (writeStorage.isEmpty()) { + return; + } + + SSTableStates nextState; + upsertLock.writeLock().lock(); + try { + nextState = sstableState.get().beforeFlush(); + sstableState.set(nextState); + } finally { + upsertLock.writeLock().unlock(); + } + + Collection> entries; + entries = nextState.getReadStorage().values(); + MemorySegment newPage; + try { + newPage = SSTable.saveNextSSTable(arena, path, entries); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + upsertLock.writeLock().lock(); + try { + sstableState.set(nextState.afterFlush(newPage)); + } finally { + upsertLock.writeLock().unlock(); + } + size.set(0); + }); } @Override public void close() throws IOException { - if (!arena.scope().isAlive()) { + if (closed.getAndSet(true)) { + waitForClose(); return; } - arena.close(); + flush(); + bgExecutor.execute(arena::close); + bgExecutor.shutdown(); + waitForClose(); + } - if (!storage.isEmpty()) { - StorageUtils.save(path, storage.values()); + private void waitForClose() throws InterruptedIOException { + try { + if (!bgExecutor.awaitTermination(1, TimeUnit.MINUTES)) { + throw new InterruptedIOException(); + } + } catch (InterruptedException e) { + try { + Thread.currentThread().join(); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } } } } diff --git a/src/main/java/ru/vk/itmo/shemetovalexey/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/shemetovalexey/MemorySegmentComparator.java new file mode 100644 index 000000000..12f98cf44 --- /dev/null +++ b/src/main/java/ru/vk/itmo/shemetovalexey/MemorySegmentComparator.java @@ -0,0 +1,27 @@ +package ru.vk.itmo.shemetovalexey; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +public final class MemorySegmentComparator { + private MemorySegmentComparator() { + } + + public static int compare(MemorySegment memorySegment1, MemorySegment memorySegment2) { + long mismatch = memorySegment1.mismatch(memorySegment2); + if (mismatch == -1) { + return 0; + } + + if (mismatch == memorySegment1.byteSize()) { + return -1; + } + + if (mismatch == memorySegment2.byteSize()) { + return 1; + } + byte b1 = memorySegment1.get(ValueLayout.JAVA_BYTE, mismatch); + byte b2 = memorySegment2.get(ValueLayout.JAVA_BYTE, mismatch); + return Byte.compare(b1, b2); + } +} diff --git a/src/main/java/ru/vk/itmo/shemetovalexey/MergeIterator.java b/src/main/java/ru/vk/itmo/shemetovalexey/MergeIterator.java index 4d221c0a7..1002fe67e 100644 --- a/src/main/java/ru/vk/itmo/shemetovalexey/MergeIterator.java +++ b/src/main/java/ru/vk/itmo/shemetovalexey/MergeIterator.java @@ -10,13 +10,13 @@ public class MergeIterator implements Iterator { private final PriorityQueue> priorityQueue; private final Comparator comparator; - PeekIterator peekIterator; + PeekIterator nextPeekIterator; private static class PeekIterator implements Iterator { public final int id; private final Iterator delegate; - private T peeked; + private T last; private PeekIterator(int id, Iterator delegate) { this.id = id; @@ -25,7 +25,7 @@ private PeekIterator(int id, Iterator delegate) { @Override public boolean hasNext() { - if (peeked == null) { + if (last == null) { return delegate.hasNext(); } return true; @@ -37,18 +37,18 @@ public T next() { throw new NoSuchElementException(); } T peek = peek(); - this.peeked = null; + this.last = null; return peek; } private T peek() { - if (peeked == null) { + if (last == null) { if (!delegate.hasNext()) { return null; } - peeked = delegate.next(); + last = delegate.next(); } - return peeked; + return last; } } @@ -56,8 +56,8 @@ public MergeIterator(Collection> iterators, Comparator comparator this.comparator = comparator; Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); priorityQueue = new PriorityQueue<>( - iterators.size(), - peekComp.thenComparing(o -> -o.id) + iterators.size(), + peekComp.thenComparing(o -> -o.id) ); int id = 0; @@ -68,56 +68,64 @@ public MergeIterator(Collection> iterators, Comparator comparator } } - private void task() { + private PeekIterator peek() { + while (nextPeekIterator == null) { + nextPeekIterator = priorityQueue.poll(); + if (nextPeekIterator == null) { + return null; + } + + skipIteratorsWithSameKey(); + + if (nextPeekIterator.peek() == null) { + nextPeekIterator = null; + continue; + } + + if (shouldSkip(nextPeekIterator.peek())) { + moveNextAndPutBack(nextPeekIterator); + nextPeekIterator = null; + } + } + + return nextPeekIterator; + } + + private void skipIteratorsWithSameKey() { while (true) { PeekIterator next = priorityQueue.peek(); if (next == null) { break; } - int compare = comparator.compare(peekIterator.peek(), next.peek()); - if (compare == 0) { - PeekIterator poll = priorityQueue.poll(); - if (poll != null) { - poll.next(); - if (poll.hasNext()) { - priorityQueue.add(poll); - } - } - } else { + if (!skipTheSameKey(next)) { break; } } } - private PeekIterator peek() { - while (peekIterator == null) { - peekIterator = priorityQueue.poll(); - if (peekIterator == null) { - return null; - } - - task(); - - if (peekIterator.peek() == null) { - peekIterator = null; - continue; - } + private boolean skipTheSameKey(PeekIterator next) { + int compare = comparator.compare(nextPeekIterator.peek(), next.peek()); + if (compare != 0) { + return false; + } - if (skip(peekIterator.peek())) { - peekIterator.next(); - if (peekIterator.hasNext()) { - priorityQueue.add(peekIterator); - } - peekIterator = null; - } + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + moveNextAndPutBack(poll); } + return true; + } - return peekIterator; + private void moveNextAndPutBack(PeekIterator poll) { + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } } - protected boolean skip(T t) { - return t == null; + protected boolean shouldSkip(T t) { + return new Object().equals(t); } @Override @@ -127,15 +135,15 @@ public boolean hasNext() { @Override public T next() { - PeekIterator peek = peek(); - if (peek == null) { + PeekIterator nextIterator = peek(); + if (nextIterator == null) { throw new NoSuchElementException(); } - T next = peek.next(); - this.peekIterator = null; - if (peek.hasNext()) { - priorityQueue.add(peek); + T nextValue = nextIterator.next(); + this.nextPeekIterator = null; + if (nextIterator.hasNext()) { + priorityQueue.add(nextIterator); } - return next; + return nextValue; } } diff --git a/src/main/java/ru/vk/itmo/shemetovalexey/sstable/SSTable.java b/src/main/java/ru/vk/itmo/shemetovalexey/sstable/SSTable.java new file mode 100644 index 000000000..0e41d4a2d --- /dev/null +++ b/src/main/java/ru/vk/itmo/shemetovalexey/sstable/SSTable.java @@ -0,0 +1,232 @@ +package ru.vk.itmo.shemetovalexey.sstable; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +public final class SSTable { + public static final String PREFIX = "data_"; + private static final String TMP_FILE = "index.tmp"; + private static final String IDX_FILE = "index.idx"; + private static final String COMPACTION_TMP = "compaction.tmp"; + private static final String COMPACTION = "compaction"; + private static final int KEY_VALUE_SIZE_OFFSET = 2 * Long.BYTES; + + private SSTable() { + } + + private static MemorySegment save( + Arena arena, + FileChannel fileChannel, + Iterable> iterable + ) throws IOException { + long dataSize = 0; + long count = 0; + for (Entry entry : iterable) { + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + if (value != null) { + dataSize += value.byteSize(); + } + count++; + } + long indexSize = count * KEY_VALUE_SIZE_OFFSET; + + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + indexSize + dataSize, + arena + ); + + long dataOffset = indexSize; + int indexOffset = 0; + for (Entry entry : iterable) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += entry.key().byteSize(); + indexOffset += Long.BYTES; + + MemorySegment value = entry.value(); + if (value == null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, SSTableUtils.tombstone(dataOffset)); + } else { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + dataOffset += value.byteSize(); + } + indexOffset += Long.BYTES; + } + + dataOffset = indexSize; + for (Entry entry : iterable) { + MemorySegment key = entry.key(); + MemorySegment.copy(key, 0, fileSegment, dataOffset, key.byteSize()); + dataOffset += key.byteSize(); + + MemorySegment value = entry.value(); + if (value != null) { + MemorySegment.copy(value, 0, fileSegment, dataOffset, value.byteSize()); + dataOffset += value.byteSize(); + } + } + return fileSegment; + } + + public static MemorySegment saveNextSSTable( + Arena arena, + Path storagePath, + Iterable> iterable + ) throws IOException { + final Path indexTmp = storagePath.resolve(TMP_FILE); + final Path indexFile = storagePath.resolve(IDX_FILE); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // ignore + } + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + + String newFileName = PREFIX + existedFiles.size(); + + MemorySegment fileSegment; + try (FileChannel fileChannel = FileChannel.open( + storagePath.resolve(newFileName), + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE + )) { + fileSegment = save(arena, fileChannel, iterable); + } + + List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + list.add(newFileName); + Files.write( + indexTmp, + list, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + return fileSegment; + } + + public static MemorySegment compact( + Arena arena, + Path storagePath, + Iterable> iterable + ) throws IOException { + Path compactionTmpFile = storagePath.resolve(COMPACTION_TMP); + + MemorySegment fileSegment; + try (FileChannel fileChannel = FileChannel.open( + compactionTmpFile, + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE + )) { + fileSegment = save(arena, fileChannel, iterable); + } + + Files.move( + compactionTmpFile, + storagePath.resolve(COMPACTION), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + + finalizeCompaction(storagePath); + return fileSegment; + } + + private static void finalizeCompaction(Path storagePath) throws IOException { + try (Stream stream = Files.find( + storagePath, + 1, + (path, attrs) -> path.getFileName().toString().startsWith(PREFIX) + )) { + stream.forEach(p -> { + try { + Files.delete(p); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + Path indexTmp = storagePath.resolve(TMP_FILE); + Path indexFile = storagePath.resolve(IDX_FILE); + + Files.deleteIfExists(indexFile); + Files.deleteIfExists(indexTmp); + + Path compactionFile = SSTableUtils.compactionFile(storagePath); + boolean noData = Files.size(compactionFile) == 0; + + Files.write( + indexTmp, + noData ? Collections.emptyList() : Collections.singleton(PREFIX + "0"), + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE); + if (noData) { + Files.delete(compactionFile); + } else { + Files.move(compactionFile, storagePath.resolve(PREFIX + "0"), StandardCopyOption.ATOMIC_MOVE); + } + } + + public static List loadOrRecover(Path storagePath, Arena arena) throws IOException { + if (Files.exists(SSTableUtils.compactionFile(storagePath))) { + finalizeCompaction(storagePath); + } + + Path indexTmp = storagePath.resolve(TMP_FILE); + Path indexFile = storagePath.resolve(IDX_FILE); + + if (!Files.exists(indexFile)) { + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + Files.createFile(indexFile); + } + } + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + List result = new ArrayList<>(existedFiles.size()); + for (String fileName : existedFiles) { + Path file = storagePath.resolve(fileName); + try (FileChannel fileChannel = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(file), + arena + ); + result.add(fileSegment); + } + } + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/shemetovalexey/sstable/SSTableIterator.java b/src/main/java/ru/vk/itmo/shemetovalexey/sstable/SSTableIterator.java new file mode 100644 index 000000000..2f59ba442 --- /dev/null +++ b/src/main/java/ru/vk/itmo/shemetovalexey/sstable/SSTableIterator.java @@ -0,0 +1,91 @@ +package ru.vk.itmo.shemetovalexey.sstable; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; +import ru.vk.itmo.shemetovalexey.MemorySegmentComparator; +import ru.vk.itmo.shemetovalexey.MergeIterator; + +import java.lang.foreign.MemorySegment; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +public final class SSTableIterator { + private SSTableIterator() { + } + + public static Iterator> get(List segmentList) { + return get(segmentList, null); + } + + public static Iterator> get(List segmentList, MemorySegment from) { + return get( + Collections.emptyIterator(), + Collections.emptyIterator(), + segmentList, + from, + null + ); + } + + public static Iterator> get( + Iterator> firstIterator, + Iterator> secondIterator, + List segmentList, + MemorySegment from, + MemorySegment to + ) { + List>> iterators = new ArrayList<>(segmentList.size() + 1); + for (MemorySegment memorySegment : segmentList) { + iterators.add(iterator(memorySegment, from, to)); + } + iterators.add(firstIterator); + iterators.add(secondIterator); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, MemorySegmentComparator::compare)) { + @Override + protected boolean shouldSkip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + static Iterator> iterator(MemorySegment page, MemorySegment from, MemorySegment to) { + long recordIndexFrom = from == null ? 0 : SSTableUtils.normalize(SSTableUtils.indexOf(page, from)); + long recordIndexTo = to == null ? SSTableUtils.recordsCount(page) : SSTableUtils.normalize( + SSTableUtils.indexOf(page, to) + ); + long recordsCount = SSTableUtils.recordsCount(page); + + return new Iterator<>() { + long index = recordIndexFrom; + + @Override + public boolean hasNext() { + return index < recordIndexTo; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + MemorySegment key = SSTableUtils.slice( + page, + SSTableUtils.startOfKey(page, index), + SSTableUtils.endOfKey(page, index) + ); + long startOfValue = SSTableUtils.startOfValue(page, index); + MemorySegment value = + startOfValue < 0 + ? null + : SSTableUtils.slice(page, startOfValue, SSTableUtils.endOfValue(page, index, recordsCount)); + index++; + return new BaseEntry<>(key, value); + } + }; + } +} diff --git a/src/main/java/ru/vk/itmo/shemetovalexey/sstable/SSTableStates.java b/src/main/java/ru/vk/itmo/shemetovalexey/sstable/SSTableStates.java new file mode 100644 index 000000000..dc5b1cd5e --- /dev/null +++ b/src/main/java/ru/vk/itmo/shemetovalexey/sstable/SSTableStates.java @@ -0,0 +1,65 @@ +package ru.vk.itmo.shemetovalexey.sstable; + +import ru.vk.itmo.Entry; +import ru.vk.itmo.shemetovalexey.MemorySegmentComparator; + +import java.lang.foreign.MemorySegment; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentSkipListMap; + +public final class SSTableStates { + private final ConcurrentSkipListMap> readStorage; + private final ConcurrentSkipListMap> writeStorage; + private final List diskSegmentList; + + private SSTableStates( + ConcurrentSkipListMap> readStorage, + ConcurrentSkipListMap> writeStorage, + List diskSegmentList + ) { + this.readStorage = readStorage; + this.writeStorage = writeStorage; + this.diskSegmentList = diskSegmentList; + } + + private static ConcurrentSkipListMap> createMap() { + return new ConcurrentSkipListMap<>(MemorySegmentComparator::compare); + } + + public static SSTableStates create(List segments) { + return new SSTableStates( + createMap(), + createMap(), + segments + ); + } + + public SSTableStates compact(MemorySegment compacted) { + return new SSTableStates(readStorage, writeStorage, Collections.singletonList(compacted)); + } + + public SSTableStates beforeFlush() { + return new SSTableStates(writeStorage, createMap(), diskSegmentList); + } + + public SSTableStates afterFlush(MemorySegment newPage) { + List segments = new ArrayList<>(diskSegmentList.size() + 1); + segments.addAll(diskSegmentList); + segments.add(newPage); + return new SSTableStates(createMap(), writeStorage, segments); + } + + public ConcurrentSkipListMap> getReadStorage() { + return readStorage; + } + + public ConcurrentSkipListMap> getWriteStorage() { + return writeStorage; + } + + public List getDiskSegmentList() { + return diskSegmentList; + } +} diff --git a/src/main/java/ru/vk/itmo/bandurinvladislav/StorageUtil.java b/src/main/java/ru/vk/itmo/shemetovalexey/sstable/SSTableUtils.java similarity index 64% rename from src/main/java/ru/vk/itmo/bandurinvladislav/StorageUtil.java rename to src/main/java/ru/vk/itmo/shemetovalexey/sstable/SSTableUtils.java index fe679dbd0..5008d7902 100644 --- a/src/main/java/ru/vk/itmo/bandurinvladislav/StorageUtil.java +++ b/src/main/java/ru/vk/itmo/shemetovalexey/sstable/SSTableUtils.java @@ -1,13 +1,20 @@ -package ru.vk.itmo.bandurinvladislav; +package ru.vk.itmo.shemetovalexey.sstable; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; +import java.nio.file.Path; -public final class StorageUtil { - private StorageUtil() { +final class SSTableUtils { + private static final String COMPACTION = "compaction"; + + private SSTableUtils() { + } + + static Path compactionFile(Path storagePath) { + return storagePath.resolve(COMPACTION); } - public static long indexOf(MemorySegment segment, MemorySegment key) { + static long indexOf(MemorySegment segment, MemorySegment key) { long recordsCount = recordsCount(segment); long left = 0; @@ -44,47 +51,48 @@ public static long indexOf(MemorySegment segment, MemorySegment key) { return tombstone(left); } - public static long recordsCount(MemorySegment segment) { + static long recordsCount(MemorySegment segment) { long indexSize = indexSize(segment); return indexSize / Long.BYTES / 2; } - public static long indexSize(MemorySegment segment) { + static long indexSize(MemorySegment segment) { return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); } - public static MemorySegment slice(MemorySegment page, long start, long end) { + static MemorySegment slice(MemorySegment page, long start, long end) { return page.asSlice(start, end - start); } - public static long startOfKey(MemorySegment segment, long recordIndex) { + static long startOfKey(MemorySegment segment, long recordIndex) { return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES); } - public static long endOfKey(MemorySegment segment, long recordIndex) { + static long endOfKey(MemorySegment segment, long recordIndex) { return normalizedStartOfValue(segment, recordIndex); } - public static long normalizedStartOfValue(MemorySegment segment, long recordIndex) { + static long normalizedStartOfValue(MemorySegment segment, long recordIndex) { return normalize(startOfValue(segment, recordIndex)); } - public static long startOfValue(MemorySegment segment, long recordIndex) { + static long startOfValue(MemorySegment segment, long recordIndex) { return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES + Long.BYTES); } - public static long endOfValue(MemorySegment segment, long recordIndex, long recordsCount) { + static long endOfValue(MemorySegment segment, long recordIndex, long recordsCount) { if (recordIndex < recordsCount - 1) { return startOfKey(segment, recordIndex + 1); } return segment.byteSize(); } - public static long tombstone(long offset) { + static long tombstone(long offset) { + // set first bit for tombstone offset (equals: -offset - 1) return 1L << 63 | offset; } - public static long normalize(long value) { + static long normalize(long value) { return value & ~(1L << 63); } } diff --git a/src/main/java/ru/vk/itmo/solnyshkoksenia/DaoException.java b/src/main/java/ru/vk/itmo/solnyshkoksenia/DaoException.java new file mode 100644 index 000000000..b8ac14feb --- /dev/null +++ b/src/main/java/ru/vk/itmo/solnyshkoksenia/DaoException.java @@ -0,0 +1,11 @@ +package ru.vk.itmo.solnyshkoksenia; + +public class DaoException extends RuntimeException { + public DaoException(String message) { + super(message); + } + + public DaoException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/ru/vk/itmo/solnyshkoksenia/DaoImpl.java b/src/main/java/ru/vk/itmo/solnyshkoksenia/DaoImpl.java index e30eaeb44..d299840a0 100644 --- a/src/main/java/ru/vk/itmo/solnyshkoksenia/DaoImpl.java +++ b/src/main/java/ru/vk/itmo/solnyshkoksenia/DaoImpl.java @@ -10,92 +10,182 @@ import java.lang.foreign.MemorySegment; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Collections; import java.util.Comparator; import java.util.Iterator; -import java.util.NavigableMap; +import java.util.List; import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class DaoImpl implements Dao> { private static final Comparator comparator = new MemorySegmentComparator(); - private final NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); - private Arena arena; - private DiskStorage diskStorage; - - public DaoImpl() { - // Empty constructor - } + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private final ReadWriteLock lock = new ReentrantReadWriteLock(true); + private final Arena arena; + private final Path path; + private volatile State curState; public DaoImpl(Config config) throws IOException { - Path path = config.basePath().resolve("data"); + path = config.basePath().resolve("data"); Files.createDirectories(path); arena = Arena.ofShared(); - this.diskStorage = new DiskStorage(DiskStorage.loadOrRecover(path, arena), path); + this.curState = new State(config, new DiskStorage(DiskStorage.loadOrRecover(path, arena), path)); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { - return diskStorage.range(getInMemory(from, to), from, to); + State state = this.curState.checkAndGet(); + List>> iterators = List.of( + state.getInMemory(state.flushingStorage, from, to), + state.getInMemory(state.storage, from, to) + ); + + Iterator> iterator = new MergeIterator<>(iterators, + (e1, e2) -> comparator.compare(e1.key(), e2.key())); + Iterator> innerIterator = state.diskStorage.range(iterator, from, to); + + return new Iterator<>() { + @Override + public boolean hasNext() { + return innerIterator.hasNext(); + } + + @Override + public Entry next() { + return innerIterator.next(); + } + }; } - private Iterator> getInMemory(MemorySegment from, MemorySegment to) { - if (from == null && to == null) { - return storage.values().iterator(); - } - if (from == null) { - return storage.headMap(to).values().iterator(); + public void upsert(Entry entry, Long ttl) { + State state = this.curState.checkAndGet(); + + lock.readLock().lock(); + try { + state.putInMemory(entry, ttl); + } finally { + lock.readLock().unlock(); } - if (to == null) { - return storage.tailMap(from).values().iterator(); + + if (state.isOverflowed()) { + try { + autoFlush(); + } catch (IOException e) { + throw new DaoException("Memory storage overflowed. Cannot flush", e); + } } - return storage.subMap(from, to).values().iterator(); } @Override public void upsert(Entry entry) { - storage.put(entry.key(), entry); + upsert(entry, null); } @Override public Entry get(MemorySegment key) { - Entry entry = storage.get(key); - if (entry != null) { - if (entry.value() == null) { - return null; + State state = this.curState.checkAndGet(); + return state.get(key, comparator); + } + + @Override + public void flush() throws IOException { + State state = this.curState.checkAndGet(); + if (state.storage.isEmpty() || state.isFlushing()) { + return; + } + autoFlush(); + } + + private void autoFlush() throws IOException { + State state = this.curState.checkAndGet(); + lock.writeLock().lock(); + try { + if (state.isFlushing()) { + if (state.isOverflowed()) { + throw new IOException(); + } else { + return; + } } - return entry; + this.curState = state.moveStorage(); + } finally { + lock.writeLock().unlock(); } - Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + executor.execute(this::tryFlush); + } - if (!iterator.hasNext()) { - return null; + private void tryFlush() { + State state = this.curState.checkAndGet(); + try { + state.flush(); + } catch (IOException e) { + throw new DaoException("Flush failed", e); } - Entry next = iterator.next(); - if (comparator.compare(next.key(), key) == 0) { - return next; + + lock.writeLock().lock(); + try { + this.curState = new State(state.config, state.storage, new ConcurrentSkipListMap<>(comparator), + new DiskStorage(DiskStorage.loadOrRecover(path, arena), path)); + } catch (IOException e) { + throw new DaoException("Cannot recover storage on disk", e); + } finally { + lock.writeLock().unlock(); } - return null; } @Override public void compact() throws IOException { - diskStorage.compact(storage.values()); - storage.clear(); + try { + executor.submit(this::tryCompact).get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new DaoException("Compaction failed. Thread interrupted", e); + } catch (ExecutionException e) { + throw new DaoException("Compaction failed", e); + } + } + + private Object tryCompact() { + State state = this.curState.checkAndGet(); + try { + state.diskStorage.compact(); + } catch (IOException e) { + throw new DaoException("Cannot compact", e); + } + + lock.writeLock().lock(); + try { + this.curState = new State(state.config, state.storage, state.flushingStorage, + new DiskStorage(DiskStorage.loadOrRecover(path, arena), path)); + } catch (IOException e) { + throw new DaoException("Cannot recover storage on disk after compaction", e); + } finally { + lock.writeLock().unlock(); + } + + return null; } @Override - public void close() throws IOException { - if (!arena.scope().isAlive()) { + public synchronized void close() throws IOException { + State state = this.curState; + if (state.isClosed() || !arena.scope().isAlive()) { return; } + if (!state.storage.isEmpty()) { + state.save(); + } + + executor.close(); arena.close(); - if (!storage.isEmpty()) { - diskStorage.save(storage.values()); - } + this.curState = state.close(); } } diff --git a/src/main/java/ru/vk/itmo/solnyshkoksenia/EntryExtended.java b/src/main/java/ru/vk/itmo/solnyshkoksenia/EntryExtended.java new file mode 100644 index 000000000..6aa739b5d --- /dev/null +++ b/src/main/java/ru/vk/itmo/solnyshkoksenia/EntryExtended.java @@ -0,0 +1,10 @@ +package ru.vk.itmo.solnyshkoksenia; + +import ru.vk.itmo.Entry; + +public record EntryExtended(Data key, Data value, Data expiration) implements Entry { + @Override + public String toString() { + return "{" + key + ":" + value + ":" + expiration + "}"; + } +} diff --git a/src/main/java/ru/vk/itmo/solnyshkoksenia/State.java b/src/main/java/ru/vk/itmo/solnyshkoksenia/State.java new file mode 100644 index 000000000..476df582f --- /dev/null +++ b/src/main/java/ru/vk/itmo/solnyshkoksenia/State.java @@ -0,0 +1,148 @@ +package ru.vk.itmo.solnyshkoksenia; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Entry; +import ru.vk.itmo.solnyshkoksenia.storage.DiskStorage; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +public class State { + private static final Comparator comparator = new MemorySegmentComparator(); + protected final Config config; + protected final NavigableMap> storage; + protected final NavigableMap> flushingStorage; + protected final DiskStorage diskStorage; + private final AtomicLong storageByteSize = new AtomicLong(); + private final AtomicBoolean isClosed = new AtomicBoolean(); + private final AtomicBoolean overflow = new AtomicBoolean(); + + public State(Config config, + NavigableMap> storage, + NavigableMap> flushingStorage, + DiskStorage diskStorage) { + this.config = config; + this.storage = storage; + this.flushingStorage = flushingStorage; + this.diskStorage = diskStorage; + } + + public State(Config config, + DiskStorage diskStorage) { + this.config = config; + this.storage = new ConcurrentSkipListMap<>(comparator); + this.flushingStorage = new ConcurrentSkipListMap<>(comparator); + this.diskStorage = diskStorage; + } + + public void putInMemory(Entry entry, Long ttl) { + MemorySegment expiration = null; + if (ttl != null) { + long[] ar = {System.currentTimeMillis() + ttl}; + expiration = MemorySegment.ofArray(ar); + } + EntryExtended entryExtended = new EntryExtended<>(entry.key(), entry.value(), expiration); + EntryExtended previousEntry = storage.put(entryExtended.key(), entryExtended); + + if (previousEntry != null) { + storageByteSize.addAndGet(-getSize(previousEntry)); + } + + if (storageByteSize.addAndGet(getSize(entryExtended)) > config.flushThresholdBytes()) { + overflow.set(true); + } + } + + public void save() throws IOException { + diskStorage.save(storage.values()); + } + + private static long getSize(EntryExtended entry) { + long valueSize = entry.value() == null ? 0 : entry.value().byteSize(); + long expirationSize = entry.expiration() == null ? 0 : entry.expiration().byteSize(); + return Long.BYTES + entry.key().byteSize() + Long.BYTES + valueSize + Long.BYTES + expirationSize; + } + + public State checkAndGet() { + if (isClosed.get()) { + throw new DaoException("Dao is already closed"); + } + return this; + } + + public boolean isClosed() { + return isClosed.get(); + } + + public boolean isOverflowed() { + return overflow.get(); + } + + public boolean isFlushing() { + return !flushingStorage.isEmpty(); + } + + public State moveStorage() { + return new State(config, new ConcurrentSkipListMap<>(comparator), storage, diskStorage); + } + + public void flush() throws IOException { + diskStorage.save(flushingStorage.values()); + } + + public State close() { + isClosed.set(true); + return this; + } + + public Entry get(MemorySegment key, Comparator comparator) { + EntryExtended entry = storage.get(key); + if (isValidEntry(entry)) { + return entry.value() == null ? null : entry; + } + + entry = flushingStorage.get(key); + if (isValidEntry(entry)) { + return entry.value() == null ? null : entry; + } + + Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + + if (!iterator.hasNext()) { + return null; + } + EntryExtended next = iterator.next(); + if (comparator.compare(next.key(), key) == 0 && isValidEntry(next)) { + return next; + } + return null; + } + + private boolean isValidEntry(EntryExtended entry) { + return entry != null && (entry.expiration() == null + || entry.expiration().toArray(ValueLayout.JAVA_LONG_UNALIGNED)[0] > System.currentTimeMillis()); + } + + protected Iterator> getInMemory( + NavigableMap> memory, + MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return memory.values().iterator(); + } + if (from == null) { + return memory.headMap(to).values().iterator(); + } + if (to == null) { + return memory.tailMap(from).values().iterator(); + } + return memory.subMap(from, to).values().iterator(); + } +} diff --git a/src/main/java/ru/vk/itmo/solnyshkoksenia/storage/DiskStorage.java b/src/main/java/ru/vk/itmo/solnyshkoksenia/storage/DiskStorage.java index 4faa321b4..e97ba26af 100644 --- a/src/main/java/ru/vk/itmo/solnyshkoksenia/storage/DiskStorage.java +++ b/src/main/java/ru/vk/itmo/solnyshkoksenia/storage/DiskStorage.java @@ -2,13 +2,13 @@ import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Entry; +import ru.vk.itmo.solnyshkoksenia.EntryExtended; import ru.vk.itmo.solnyshkoksenia.MemorySegmentComparator; import ru.vk.itmo.solnyshkoksenia.MergeIterator; import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.nio.file.FileAlreadyExistsException; @@ -17,6 +17,7 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -34,11 +35,11 @@ public DiskStorage(List segmentList, Path storagePath) { this.storagePath = storagePath; } - public Iterator> range( - Iterator> firstIterator, + public Iterator> range( + Iterator> firstIterator, MemorySegment from, MemorySegment to) { - List>> iterators = new ArrayList<>(segmentList.size() + 1); + List>> iterators = new ArrayList<>(segmentList.size() + 1); for (MemorySegment memorySegment : segmentList) { iterators.add(iterator(memorySegment, from, to)); } @@ -46,13 +47,17 @@ public Iterator> range( return new MergeIterator<>(iterators, (e1, e2) -> comparator.compare(e1.key(), e2.key())) { @Override - protected boolean skip(Entry memorySegmentEntry) { + protected boolean skip(EntryExtended memorySegmentEntry) { + if (memorySegmentEntry.expiration() != null) { + return memorySegmentEntry.value() == null + || !utils.checkTTL(memorySegmentEntry.expiration(), System.currentTimeMillis()); + } return memorySegmentEntry.value() == null; } }; } - public void save(Iterable> iterable) + public void save(Iterable> iterable) throws IOException { final Path indexTmp = storagePath.resolve("index.tmp"); final Path indexFile = storagePath.resolve(INDEX_FILE_NAME); @@ -66,17 +71,22 @@ public void save(Iterable> iterable) String newFileName = String.valueOf(existedFiles.size()); - long dataSize = 0; - long count = 0; - for (Entry entry : iterable) { - dataSize += entry.key().byteSize(); - MemorySegment value = entry.value(); - if (value != null) { - dataSize += value.byteSize(); + final long currentTime = System.currentTimeMillis(); + + Entry sizes = new BaseEntry<>(0L, 0L); + for (EntryExtended entry : iterable) { + MemorySegment expiration = entry.expiration(); + if (expiration == null || utils.checkTTL(expiration, currentTime)) { + sizes = utils.countEntrySize(entry, sizes); } - count++; } - long indexSize = count * 2 * Long.BYTES; + long dataSize = sizes.key(); + long count = sizes.value(); + + if (count == 0) { + return; + } + long indexSize = count * 3 * Long.BYTES; try ( FileChannel fileChannel = FileChannel.open( @@ -90,14 +100,17 @@ public void save(Iterable> iterable) MemorySegment fileSegment = utils.mapFile(fileChannel, indexSize + dataSize, writeArena); // index: - // |key0_Start|value0_Start|key1_Start|value1_Start|key2_Start|value2_Start|... + // |key0_Start|value0_Start|expiration0_Start|key1_Start|value1_Start|expiration1_Start|key2_Start|... // key0_Start = data start = end of index // data: - // |key0|value0|key1|value1|... + // |key0|value0|expiration0|key1|value1|expiration1|... Entry offsets = new BaseEntry<>(indexSize, 0L); - for (Entry entry : iterable) { - offsets = utils.putEntry(fileSegment, offsets, entry); + for (EntryExtended entry : iterable) { + MemorySegment expiration = entry.expiration(); + if (expiration == null || utils.checkTTL(expiration, currentTime)) { + offsets = utils.putEntry(fileSegment, offsets, entry); + } } } @@ -117,7 +130,7 @@ public void save(Iterable> iterable) Files.delete(indexTmp); } - public void compact(Iterable> iterable) throws IOException { + public void compact() throws IOException { final Path tmpFile = storagePath.resolve("tmp"); final Path indexFile = storagePath.resolve(INDEX_FILE_NAME); @@ -129,23 +142,27 @@ public void compact(Iterable> iterable) throws IOException List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); - if (existedFiles.isEmpty() && !iterable.iterator().hasNext()) { + if (existedFiles.isEmpty()) { return; // nothing to compact } - Iterator> iterator = range(iterable.iterator(), null, null); - Iterator> iterator1 = range(iterable.iterator(), null, null); + Iterator> iterator = range(Collections.emptyIterator(), null, null); + Iterator> iterator1 = range(Collections.emptyIterator(), null, null); long dataSize = 0; long indexSize = 0; while (iterator.hasNext()) { - indexSize += Long.BYTES * 2; - Entry entry = iterator.next(); + indexSize += Long.BYTES * 3; + EntryExtended entry = iterator.next(); dataSize += entry.key().byteSize(); MemorySegment value = entry.value(); if (value != null) { dataSize += value.byteSize(); } + MemorySegment expiration = entry.expiration(); + if (expiration != null) { + dataSize += expiration.byteSize(); + } } try ( @@ -169,16 +186,26 @@ public void compact(Iterable> iterable) throws IOException Files.delete(storagePath.resolve(file)); } - Files.move(tmpFile, storagePath.resolve("0"), StandardCopyOption.ATOMIC_MOVE, - StandardCopyOption.REPLACE_EXISTING); + final Path indexTmp = storagePath.resolve("indexTmp"); + Files.deleteIfExists(indexTmp); + + boolean noData = Files.size(tmpFile) == 0; Files.write( - indexFile, - List.of("0"), + indexTmp, + noData ? Collections.emptyList() : List.of("0"), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING ); + + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE); + if (noData) { + Files.delete(tmpFile); + } else { + Files.move(tmpFile, storagePath.resolve("0"), StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } } public static List loadOrRecover(Path storagePath, Arena arena) throws IOException { @@ -208,46 +235,10 @@ public static List loadOrRecover(Path storagePath, Arena arena) t return result; } - private static long indexOf(MemorySegment segment, MemorySegment key) { - long recordsCount = utils.recordsCount(segment); - - long left = 0; - long right = recordsCount - 1; - while (left <= right) { - long mid = (left + right) >>> 1; - - long startOfKey = utils.startOfKey(segment, mid); - long endOfKey = utils.endOfKey(segment, mid); - long mismatch = MemorySegment.mismatch(segment, startOfKey, endOfKey, key, 0, key.byteSize()); - if (mismatch == -1) { - return mid; - } - - if (mismatch == key.byteSize()) { - right = mid - 1; - continue; - } - - if (mismatch == endOfKey - startOfKey) { - left = mid + 1; - continue; - } - - int b1 = Byte.toUnsignedInt(segment.get(ValueLayout.JAVA_BYTE, startOfKey + mismatch)); - int b2 = Byte.toUnsignedInt(key.get(ValueLayout.JAVA_BYTE, mismatch)); - if (b1 > b2) { - right = mid - 1; - } else { - left = mid + 1; - } - } - - return utils.tombstone(left); - } - - private static Iterator> iterator(MemorySegment page, MemorySegment from, MemorySegment to) { - long recordIndexFrom = from == null ? 0 : utils.normalize(indexOf(page, from)); - long recordIndexTo = to == null ? utils.recordsCount(page) : utils.normalize(indexOf(page, to)); + private static Iterator> iterator(MemorySegment page, + MemorySegment from, MemorySegment to) { + long recordIndexFrom = from == null ? 0 : utils.normalize(utils.indexOf(page, from)); + long recordIndexTo = to == null ? utils.recordsCount(page) : utils.normalize(utils.indexOf(page, to)); long recordsCount = utils.recordsCount(page); return new Iterator<>() { @@ -259,7 +250,7 @@ public boolean hasNext() { } @Override - public Entry next() { + public EntryExtended next() { if (!hasNext()) { throw new NoSuchElementException(); } @@ -268,9 +259,14 @@ public Entry next() { MemorySegment value = startOfValue < 0 ? null - : utils.slice(page, startOfValue, utils.endOfValue(page, index, recordsCount)); + : utils.slice(page, startOfValue, utils.endOfValue(page, index)); + long startOfExp = utils.startOfExpiration(page, index); + MemorySegment expiration = + startOfExp < 0 + ? null + : utils.slice(page, startOfExp, utils.endOfExpiration(page, index, recordsCount)); index++; - return new BaseEntry<>(key, value); + return new EntryExtended<>(key, value, expiration); } }; } diff --git a/src/main/java/ru/vk/itmo/solnyshkoksenia/storage/StorageUtils.java b/src/main/java/ru/vk/itmo/solnyshkoksenia/storage/StorageUtils.java index c1c9e579a..3100a9304 100644 --- a/src/main/java/ru/vk/itmo/solnyshkoksenia/storage/StorageUtils.java +++ b/src/main/java/ru/vk/itmo/solnyshkoksenia/storage/StorageUtils.java @@ -2,6 +2,7 @@ import ru.vk.itmo.BaseEntry; import ru.vk.itmo.Entry; +import ru.vk.itmo.solnyshkoksenia.EntryExtended; import java.io.IOException; import java.lang.foreign.Arena; @@ -15,7 +16,7 @@ protected MemorySegment slice(MemorySegment page, long start, long end) { } protected long startOfKey(MemorySegment segment, long recordIndex) { - return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES); + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 3 * Long.BYTES); } protected long endOfKey(MemorySegment segment, long recordIndex) { @@ -23,10 +24,18 @@ protected long endOfKey(MemorySegment segment, long recordIndex) { } protected long startOfValue(MemorySegment segment, long recordIndex) { - return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 2 * Long.BYTES + Long.BYTES); + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 3 * Long.BYTES + Long.BYTES); } - protected long endOfValue(MemorySegment segment, long recordIndex, long recordsCount) { + protected long endOfValue(MemorySegment segment, long recordIndex) { + return normalizedStartOfExpiration(segment, recordIndex); + } + + protected long startOfExpiration(MemorySegment segment, long recordIndex) { + return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, recordIndex * 3 * Long.BYTES + Long.BYTES * 2); + } + + protected long endOfExpiration(MemorySegment segment, long recordIndex, long recordsCount) { if (recordIndex < recordsCount - 1) { return startOfKey(segment, recordIndex + 1); } @@ -43,7 +52,7 @@ protected long normalize(long value) { protected long recordsCount(MemorySegment segment) { long indexSize = indexSize(segment); - return indexSize / Long.BYTES / 2; + return indexSize / Long.BYTES / 3; } protected MemorySegment mapFile(FileChannel fileChannel, long size, Arena arena) throws IOException { @@ -55,17 +64,31 @@ protected MemorySegment mapFile(FileChannel fileChannel, long size, Arena arena) ); } - protected Entry putEntry(MemorySegment fileSegment, Entry offsets, Entry entry) { + protected Entry countEntrySize(EntryExtended entry, Entry sizes) { + long dataSize = sizes.key(); + dataSize += entry.key().byteSize(); + MemorySegment value = entry.value(); + if (value != null) { + dataSize += value.byteSize(); + } + MemorySegment expiration = entry.expiration(); + if (expiration != null) { + dataSize += expiration.byteSize(); + } + return new BaseEntry<>(dataSize, sizes.value() + 1); + } + + protected Entry putEntry(MemorySegment fileSegment, Entry offsets, EntryExtended entry) { long dataOffset = offsets.key(); long indexOffset = offsets.value(); fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); indexOffset += Long.BYTES; MemorySegment key = entry.key(); - MemorySegment value = entry.value(); MemorySegment.copy(key, 0, fileSegment, dataOffset, key.byteSize()); dataOffset += key.byteSize(); + MemorySegment value = entry.value(); if (value == null) { fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, tombstone(dataOffset)); } else { @@ -75,6 +98,16 @@ protected Entry putEntry(MemorySegment fileSegment, Entry offsets, E } indexOffset += Long.BYTES; + MemorySegment expiration = entry.expiration(); + if (expiration == null) { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, tombstone(dataOffset)); + } else { + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); + MemorySegment.copy(expiration, 0, fileSegment, dataOffset, expiration.byteSize()); + dataOffset += expiration.byteSize(); + } + indexOffset += Long.BYTES; + return new BaseEntry<>(dataOffset, indexOffset); } @@ -82,7 +115,52 @@ private long normalizedStartOfValue(MemorySegment segment, long recordIndex) { return normalize(startOfValue(segment, recordIndex)); } + private long normalizedStartOfExpiration(MemorySegment segment, long recordIndex) { + return normalize(startOfExpiration(segment, recordIndex)); + } + private static long indexSize(MemorySegment segment) { return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); } + + protected long indexOf(MemorySegment segment, MemorySegment key) { + long recordsCount = recordsCount(segment); + + long left = 0; + long right = recordsCount - 1; + while (left <= right) { + long mid = (left + right) >>> 1; + + long startOfKey = startOfKey(segment, mid); + long endOfKey = endOfKey(segment, mid); + long mismatch = MemorySegment.mismatch(segment, startOfKey, endOfKey, key, 0, key.byteSize()); + if (mismatch == -1) { + return mid; + } + + if (mismatch == key.byteSize()) { + right = mid - 1; + continue; + } + + if (mismatch == endOfKey - startOfKey) { + left = mid + 1; + continue; + } + + int b1 = Byte.toUnsignedInt(segment.get(ValueLayout.JAVA_BYTE, startOfKey + mismatch)); + int b2 = Byte.toUnsignedInt(key.get(ValueLayout.JAVA_BYTE, mismatch)); + if (b1 > b2) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + return tombstone(left); + } + + protected boolean checkTTL(MemorySegment expiration, long time) { + return expiration.get(ValueLayout.JAVA_LONG_UNALIGNED, 0) > time; + } } diff --git a/src/main/java/ru/vk/itmo/svistukhinandrey/DiskStorage.java b/src/main/java/ru/vk/itmo/svistukhinandrey/DiskStorage.java index c5b6fcbf6..19c776b90 100644 --- a/src/main/java/ru/vk/itmo/svistukhinandrey/DiskStorage.java +++ b/src/main/java/ru/vk/itmo/svistukhinandrey/DiskStorage.java @@ -15,44 +15,83 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.ArrayList; -import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; +import java.util.concurrent.CopyOnWriteArrayList; public class DiskStorage { + private static final String DATA_FILE_AFTER_COMPACTION = "0"; private final List segmentList; public DiskStorage(List segmentList) { - this.segmentList = segmentList; + this.segmentList = new CopyOnWriteArrayList<>(); + this.segmentList.addAll(segmentList); } - public Iterator> all(Collection> storage) { - return range(storage.iterator(), null, null); + public void mapSSTableAfterFlush(Path storagePath, String fileName, Arena arena) throws IOException { + Path file = storagePath.resolve(fileName); + try (FileChannel fileChannel = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE)) { + MemorySegment fileSegment = fileChannel.map( + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(file), + arena + ); + this.segmentList.add(fileSegment); + } + } + + public void mapSSTableAfterCompaction(Path storagePath, Arena arena) throws IOException { + this.segmentList.clear(); + mapSSTableAfterFlush(storagePath, DATA_FILE_AFTER_COMPACTION, arena); } public Iterator> range( - Iterator> firstIterator, + StorageState storageState, + MemorySegment from, + MemorySegment to + ) { + List>> iterators = new ArrayList<>(segmentList.size() + 1); + for (MemorySegment memorySegment : segmentList) { + iterators.add(iterator(memorySegment, from, to)); + } + + if (storageState.getFlushingSSTable() != null) { + iterators.add(storageState.getFlushingSSTable().get(from, to)); + } + iterators.add(storageState.getActiveSSTable().get(from, to)); + + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, MemorySegmentUtils::compare)) { + @Override + protected boolean shouldSkip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } + + public Iterator> rangeFromDisk( MemorySegment from, MemorySegment to ) { List>> iterators = new ArrayList<>(segmentList.size() + 1); + iterators.add(Collections.emptyIterator()); for (MemorySegment memorySegment : segmentList) { iterators.add(iterator(memorySegment, from, to)); } - iterators.add(firstIterator); - return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, PersistentDao::compare)) { + return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, MemorySegmentUtils::compare)) { @Override - protected boolean skip(Entry memorySegmentEntry) { + protected boolean shouldSkip(Entry memorySegmentEntry) { return memorySegmentEntry.value() == null; } }; } - public static void save( + public static String save( Path storagePath, Iterable> iterable ) throws IOException { @@ -150,6 +189,7 @@ public static void save( ); Files.delete(indexTmp); + return newFileName; } public static List loadOrRecover(Path storagePath, Arena arena) throws IOException { diff --git a/src/main/java/ru/vk/itmo/svistukhinandrey/MemorySegmentUtils.java b/src/main/java/ru/vk/itmo/svistukhinandrey/MemorySegmentUtils.java index 29b82be38..1224e61d6 100644 --- a/src/main/java/ru/vk/itmo/svistukhinandrey/MemorySegmentUtils.java +++ b/src/main/java/ru/vk/itmo/svistukhinandrey/MemorySegmentUtils.java @@ -86,6 +86,29 @@ public static long tombstone(long offset) { return 1L << 63 | offset; } + public static int compare(MemorySegment memorySegment1, MemorySegment memorySegment2) { + long mismatch = memorySegment1.mismatch(memorySegment2); + if (mismatch == -1) { + return 0; + } + + if (mismatch == memorySegment1.byteSize()) { + return -1; + } + + if (mismatch == memorySegment2.byteSize()) { + return 1; + } + byte b1 = memorySegment1.get(ValueLayout.JAVA_BYTE, mismatch); + byte b2 = memorySegment2.get(ValueLayout.JAVA_BYTE, mismatch); + return Byte.compare(b1, b2); + } + + public static boolean isSameKey(MemorySegment memorySegment1, MemorySegment memorySegment2) { + long mismatch = memorySegment1.mismatch(memorySegment2); + return mismatch == -1; + } + private static long indexSize(MemorySegment segment) { return segment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); } diff --git a/src/main/java/ru/vk/itmo/svistukhinandrey/MergeIterator.java b/src/main/java/ru/vk/itmo/svistukhinandrey/MergeIterator.java index f247492f9..68e3943bc 100644 --- a/src/main/java/ru/vk/itmo/svistukhinandrey/MergeIterator.java +++ b/src/main/java/ru/vk/itmo/svistukhinandrey/MergeIterator.java @@ -10,8 +10,7 @@ public class MergeIterator implements Iterator { private final PriorityQueue> priorityQueue; private final Comparator comparator; - - private PeekIterator peek; + private PeekIterator nextIterator; private static class PeekIterator implements Iterator { @@ -70,32 +69,62 @@ public MergeIterator(Collection> iterators, Comparator comparator } private PeekIterator peek() { - while (peek == null) { - peek = priorityQueue.poll(); - if (peek == null) { + while (nextIterator == null) { + nextIterator = priorityQueue.poll(); + if (nextIterator == null) { return null; } - peekFromPriorityQueue(); + skipIteratorsWithSameKey(); - if (peek.peek() == null) { - peek = null; + if (nextIterator.peek() == null) { + nextIterator = null; continue; } - if (skip(peek.peek())) { - peek.next(); - if (peek.hasNext()) { - priorityQueue.add(peek); - } - peek = null; + if (shouldSkip(nextIterator.peek())) { + moveNextAndPutBack(nextIterator); + nextIterator = null; } } - return peek; + return nextIterator; + } + + private void skipIteratorsWithSameKey() { + while (true) { + PeekIterator next = priorityQueue.peek(); + if (next == null) { + break; + } + + if (!skipTheSameKey(next)) { + break; + } + } + } + + private boolean skipTheSameKey(PeekIterator next) { + int compare = comparator.compare(nextIterator.peek(), next.peek()); + if (compare != 0) { + return false; + } + + PeekIterator poll = priorityQueue.poll(); + if (poll != null) { + moveNextAndPutBack(poll); + } + return true; + } + + private void moveNextAndPutBack(PeekIterator poll) { + poll.next(); + if (poll.hasNext()) { + priorityQueue.add(poll); + } } - protected boolean skip(T t) { + protected boolean shouldSkip(T t) { return t == null; } @@ -110,34 +139,11 @@ public T next() { if (peeked == null) { throw new NoSuchElementException(); } - T next = peeked.next(); - this.peek = null; + T nextValue = peeked.next(); + this.nextIterator = null; if (peeked.hasNext()) { priorityQueue.add(peeked); } - return next; - } - - private void peekFromPriorityQueue() { - while (true) { - PeekIterator next = priorityQueue.peek(); - if (next == null) { - break; - } - - int compare = comparator.compare(peek.peek(), next.peek()); - if (compare == 0) { - PeekIterator poll = priorityQueue.poll(); - if (poll != null) { - poll.next(); - boolean hasNext = poll.hasNext(); - if (hasNext) { - priorityQueue.add(poll); - } - } - } else { - break; - } - } + return nextValue; } } diff --git a/src/main/java/ru/vk/itmo/svistukhinandrey/PersistentDao.java b/src/main/java/ru/vk/itmo/svistukhinandrey/PersistentDao.java index d650ece55..af1b6c275 100644 --- a/src/main/java/ru/vk/itmo/svistukhinandrey/PersistentDao.java +++ b/src/main/java/ru/vk/itmo/svistukhinandrey/PersistentDao.java @@ -5,82 +5,78 @@ import ru.vk.itmo.Entry; import java.io.IOException; +import java.io.UncheckedIOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.util.Collections; -import java.util.Comparator; import java.util.Iterator; -import java.util.NavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class PersistentDao implements Dao> { - private final Comparator comparator = PersistentDao::compare; - private final NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); + private final StorageState storageState; private final Arena arena; private final DiskStorage diskStorage; + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private Future compactionTask; + private Future flushTask; private final Path dataPath; - private final Path compactTempPath; + private final Path compactPath; + private final long flushThresholdBytes; public PersistentDao(Config config) throws IOException { - this.dataPath = config.basePath().resolve("data"); - this.compactTempPath = config.basePath().resolve("compact_temp"); + dataPath = config.basePath().resolve("data"); + compactPath = config.basePath().resolve("compact_temp"); + flushThresholdBytes = config.flushThresholdBytes(); + Files.createDirectories(dataPath); - Files.deleteIfExists(compactTempPath); + Files.deleteIfExists(compactPath); arena = Arena.ofShared(); this.diskStorage = new DiskStorage(DiskStorage.loadOrRecover(dataPath, arena)); - } - - static int compare(MemorySegment memorySegment1, MemorySegment memorySegment2) { - long mismatch = memorySegment1.mismatch(memorySegment2); - if (mismatch == -1) { - return 0; - } - - if (mismatch == memorySegment1.byteSize()) { - return -1; - } - - if (mismatch == memorySegment2.byteSize()) { - return 1; - } - byte b1 = memorySegment1.get(ValueLayout.JAVA_BYTE, mismatch); - byte b2 = memorySegment2.get(ValueLayout.JAVA_BYTE, mismatch); - return Byte.compare(b1, b2); + this.storageState = StorageState.initStorageState(); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { - return diskStorage.range(getInMemory(from, to), from, to); + return diskStorage.range(storageState, from, to); } - private Iterator> getInMemory(MemorySegment from, MemorySegment to) { - if (from == null && to == null) { - return storage.values().iterator(); - } - if (from == null) { - return storage.headMap(to).values().iterator(); + @Override + public void upsert(Entry entry) { + if (storageState.getActiveSSTable().getStorageSize() >= flushThresholdBytes + && storageState.getFlushingSSTable() != null + ) { + throw new IllegalStateException("SSTable is full. Wait until flush."); } - if (to == null) { - return storage.tailMap(from).values().iterator(); + + lock.writeLock().lock(); + try { + storageState.getActiveSSTable().upsert(entry); + } finally { + lock.writeLock().unlock(); } - return storage.subMap(from, to).values().iterator(); - } - @Override - public void upsert(Entry entry) { - storage.put(entry.key(), entry); + try { + if (storageState.getActiveSSTable().getStorageSize() >= flushThresholdBytes) { + flush(); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } } @Override public Entry get(MemorySegment key) { - Entry entry = storage.get(key); + Entry entry = storageState.getActiveSSTable().get(key); if (entry != null) { if (entry.value() == null) { return null; @@ -88,30 +84,56 @@ public Entry get(MemorySegment key) { return entry; } - Iterator> iterator = diskStorage.range(Collections.emptyIterator(), key, null); + Iterator> iterator = diskStorage.rangeFromDisk(key, null); if (!iterator.hasNext()) { return null; } Entry next = iterator.next(); - if (compare(next.key(), key) == 0) { + if (MemorySegmentUtils.isSameKey(next.key(), key)) { return next; } + return null; } @Override - public void compact() throws IOException { - Iterable> compactValues = () -> diskStorage.all(storage.values()); - if (compactValues.iterator().hasNext()) { - Files.createDirectories(compactTempPath); + public synchronized void compact() { + if (compactionTask != null && !compactionTask.isDone()) { + return; + } - DiskStorage.save(compactTempPath, compactValues); - DiskStorage.deleteObsoleteData(dataPath); - Files.move(compactTempPath, dataPath, StandardCopyOption.ATOMIC_MOVE); + compactionTask = executor.submit(() -> { + try { + Iterable> compactValues = () -> diskStorage.rangeFromDisk(null, null); + if (compactValues.iterator().hasNext()) { + Files.createDirectories(compactPath); + DiskStorage.save(compactPath, compactValues); + DiskStorage.deleteObsoleteData(dataPath); + Files.move(compactPath, dataPath, StandardCopyOption.ATOMIC_MOVE); + diskStorage.mapSSTableAfterCompaction(dataPath, arena); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } - storage.clear(); + @Override + public synchronized void flush() throws IOException { + if ((flushTask != null && flushTask.isDone()) || !storageState.isReadyForFlush()) { + return; } + + storageState.prepareStorageForFlush(); + flushTask = executor.submit(() -> { + lock.writeLock().lock(); + try { + flushOnDisk(); + } finally { + lock.writeLock().unlock(); + } + }); } @Override @@ -120,10 +142,30 @@ public void close() throws IOException { return; } - arena.close(); + try { + executor.shutdown(); + try { + executor.awaitTermination(5, TimeUnit.MINUTES); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } finally { + if (storageState.isReadyForFlush()) { + storageState.prepareStorageForFlush(); + flushOnDisk(); + } + arena.close(); + } + } - if (!storage.isEmpty()) { - DiskStorage.save(dataPath, storage.values()); + private void flushOnDisk() { + try { + Iterable> valuesToFlush = () -> storageState.getFlushingSSTable().getAll(); + String filename = DiskStorage.save(dataPath, valuesToFlush); + storageState.removeFlushingSSTable(); + diskStorage.mapSSTableAfterFlush(dataPath, filename, arena); + } catch (IOException e) { + throw new UncheckedIOException(e); } } } diff --git a/src/main/java/ru/vk/itmo/svistukhinandrey/SSTable.java b/src/main/java/ru/vk/itmo/svistukhinandrey/SSTable.java new file mode 100644 index 000000000..58629ef95 --- /dev/null +++ b/src/main/java/ru/vk/itmo/svistukhinandrey/SSTable.java @@ -0,0 +1,60 @@ +package ru.vk.itmo.svistukhinandrey; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicLong; + +public class SSTable { + private final Comparator comparator = MemorySegmentUtils::compare; + private final NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); + private final AtomicLong sizeInBytes = new AtomicLong(0); + + public Iterator> getAll() { + return storage.values().iterator(); + } + + public Iterator> get(MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return storage.values().iterator(); + } + if (from == null) { + return storage.headMap(to).values().iterator(); + } + if (to == null) { + return storage.tailMap(from).values().iterator(); + } + + return storage.subMap(from, to).values().iterator(); + } + + public Entry get(MemorySegment key) { + return storage.get(key); + } + + public void upsert(Entry entry) { + Entry oldValue = storage.get(entry.key()); + storage.put(entry.key(), entry); + if (oldValue != null) { + sizeInBytes.addAndGet(-byteSizeOfEntry(oldValue)); + } + sizeInBytes.addAndGet(byteSizeOfEntry(entry)); + } + + public NavigableMap> getStorage() { + return storage; + } + + public long getStorageSize() { + return sizeInBytes.get(); + } + + public static long byteSizeOfEntry(Entry entry) { + long valueSize = entry.value() == null ? 0L : entry.value().byteSize(); + return entry.key().byteSize() + valueSize; + } +} diff --git a/src/main/java/ru/vk/itmo/svistukhinandrey/StorageState.java b/src/main/java/ru/vk/itmo/svistukhinandrey/StorageState.java new file mode 100644 index 000000000..00d27b9ec --- /dev/null +++ b/src/main/java/ru/vk/itmo/svistukhinandrey/StorageState.java @@ -0,0 +1,39 @@ +package ru.vk.itmo.svistukhinandrey; + +public class StorageState { + private SSTable activeSSTable; + private SSTable flushingSSTable; + + public StorageState(SSTable activeSSTable, SSTable flushingSSTable) { + this.activeSSTable = activeSSTable; + this.flushingSSTable = flushingSSTable; + } + + public SSTable getActiveSSTable() { + return activeSSTable; + } + + public SSTable getFlushingSSTable() { + return flushingSSTable; + } + + public boolean isReadyForFlush() { + return flushingSSTable == null && !activeSSTable.getStorage().isEmpty(); + } + + public void prepareStorageForFlush() { + this.flushingSSTable = activeSSTable; + this.activeSSTable = new SSTable(); + } + + public void removeFlushingSSTable() { + this.flushingSSTable = null; + } + + public static StorageState initStorageState() { + return new StorageState( + new SSTable(), + null + ); + } +} diff --git a/src/main/java/ru/vk/itmo/test/TestDao.java b/src/main/java/ru/vk/itmo/test/TestDao.java index 54c731a7d..66a4ebc35 100644 --- a/src/main/java/ru/vk/itmo/test/TestDao.java +++ b/src/main/java/ru/vk/itmo/test/TestDao.java @@ -29,7 +29,8 @@ class TestDao> implements Dao> } public Dao> reopen() throws IOException { - return new TestDao<>(factory, config); + delegate = factory.createDao(config); + return this; } @Override diff --git a/src/main/java/ru/vk/itmo/test/abramovilya/DaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/abramovilya/DaoFactoryImpl.java index 17e21aa3d..08025974b 100644 --- a/src/main/java/ru/vk/itmo/test/abramovilya/DaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/abramovilya/DaoFactoryImpl.java @@ -10,7 +10,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 3) +@DaoFactory(stage = 4) public class DaoFactoryImpl implements DaoFactory.Factory> { @Override public ru.vk.itmo.Dao> createDao(Config config) throws IOException { diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/DaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/DaoFactoryImpl.java deleted file mode 100644 index 6c4f1525a..000000000 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/DaoFactoryImpl.java +++ /dev/null @@ -1,42 +0,0 @@ -package ru.vk.itmo.test.bandurinvladislav; - -import ru.vk.itmo.Config; -import ru.vk.itmo.Dao; -import ru.vk.itmo.Entry; -import ru.vk.itmo.bandurinvladislav.PersistentDao; -import ru.vk.itmo.test.DaoFactory; - -import java.io.IOException; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.charset.StandardCharsets; - -@DaoFactory(stage = 4) -public class DaoFactoryImpl implements DaoFactory.Factory> { - - @Override - public Dao> createDao(Config config) throws IOException { - return new PersistentDao(config); - } - - @Override - public String toString(MemorySegment memorySegment) { - return memorySegment == null ? null : - new String(memorySegment.toArray(ValueLayout.JAVA_BYTE), StandardCharsets.UTF_8); - } - - public static String toStringS(MemorySegment memorySegment) { - return memorySegment == null ? null : - new String(memorySegment.toArray(ValueLayout.JAVA_BYTE), StandardCharsets.UTF_8); - } - - @Override - public MemorySegment fromString(String data) { - return data == null ? null : MemorySegment.ofArray(data.getBytes(StandardCharsets.UTF_8)); - } - - @Override - public Entry fromBaseEntry(Entry baseEntry) { - return baseEntry; - } -} diff --git a/src/main/java/ru/vk/itmo/test/bazhenovkirill/DaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/bazhenovkirill/DaoFactoryImpl.java index 150fb53b7..943d6082b 100644 --- a/src/main/java/ru/vk/itmo/test/bazhenovkirill/DaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/bazhenovkirill/DaoFactoryImpl.java @@ -11,7 +11,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 2) +@DaoFactory(stage = 4) public class DaoFactoryImpl implements DaoFactory.Factory> { @Override diff --git a/src/main/java/ru/vk/itmo/test/belonogovnikolay/InMemoryDaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/belonogovnikolay/InMemoryDaoFactoryImpl.java index 1daaef62d..7f892292e 100644 --- a/src/main/java/ru/vk/itmo/test/belonogovnikolay/InMemoryDaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/belonogovnikolay/InMemoryDaoFactoryImpl.java @@ -6,12 +6,13 @@ import ru.vk.itmo.belonogovnikolay.InMemoryTreeDao; import ru.vk.itmo.test.DaoFactory; +import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; import java.util.Objects; -@DaoFactory(stage = 2) +@DaoFactory(stage = 4) public class InMemoryDaoFactoryImpl implements DaoFactory.Factory> { /** * Creates new instance of Dao. @@ -19,13 +20,8 @@ public class InMemoryDaoFactoryImpl implements DaoFactory.Factory> createDao() { - return InMemoryTreeDao.newInstance(); - } - - @Override - public Dao> createDao(Config config) { - return InMemoryTreeDao.newInstance(config); + public Dao> createDao(Config config) throws IOException { + return new InMemoryTreeDao(config); } diff --git a/src/main/java/ru/vk/itmo/test/chernyshevyaroslav/InMemoryDaoFactory.java b/src/main/java/ru/vk/itmo/test/chernyshevyaroslav/ChernyshevDaoFactory.java similarity index 63% rename from src/main/java/ru/vk/itmo/test/chernyshevyaroslav/InMemoryDaoFactory.java rename to src/main/java/ru/vk/itmo/test/chernyshevyaroslav/ChernyshevDaoFactory.java index 7ba91e6a4..7055ff62f 100644 --- a/src/main/java/ru/vk/itmo/test/chernyshevyaroslav/InMemoryDaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/chernyshevyaroslav/ChernyshevDaoFactory.java @@ -1,22 +1,28 @@ package ru.vk.itmo.test.chernyshevyaroslav; +import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import ru.vk.itmo.chernyshevyaroslav.InMemoryDao; +import ru.vk.itmo.chernyshevyaroslav.ChernyshevDao; import ru.vk.itmo.test.DaoFactory; + +import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory -public class InMemoryDaoFactory implements DaoFactory.Factory> { +@DaoFactory(stage = 4, week = 6) +public class ChernyshevDaoFactory implements DaoFactory.Factory> { @Override - public Dao> createDao() { - return new InMemoryDao(); + public Dao> createDao(Config config) throws IOException { + return new ChernyshevDao(config); } @Override public String toString(MemorySegment memorySegment) { + if (memorySegment == null) { + return null; + } return new String(memorySegment.toArray(ValueLayout.JAVA_BYTE), StandardCharsets.UTF_8); } diff --git a/src/main/java/ru/vk/itmo/test/cheshevandrey/InMemoryFactory.java b/src/main/java/ru/vk/itmo/test/cheshevandrey/InMemoryFactory.java index 2b61e0b4e..9732661ff 100644 --- a/src/main/java/ru/vk/itmo/test/cheshevandrey/InMemoryFactory.java +++ b/src/main/java/ru/vk/itmo/test/cheshevandrey/InMemoryFactory.java @@ -3,7 +3,7 @@ import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import ru.vk.itmo.cheshevandrey.InMemoryDao; +import ru.vk.itmo.cheshevandrey.PersistentDao; import ru.vk.itmo.test.DaoFactory; import java.io.IOException; @@ -12,12 +12,12 @@ import static java.nio.charset.StandardCharsets.UTF_8; -@DaoFactory(stage = 2) +@DaoFactory(stage = 4) public class InMemoryFactory implements DaoFactory.Factory> { @Override public Dao> createDao(Config config) throws IOException { - return new InMemoryDao(config); + return new PersistentDao(config); } @Override diff --git a/src/main/java/ru/vk/itmo/test/danilinandrew/MyFactory.java b/src/main/java/ru/vk/itmo/test/danilinandrew/MyFactory.java index 0125a828c..9a48a2a56 100644 --- a/src/main/java/ru/vk/itmo/test/danilinandrew/MyFactory.java +++ b/src/main/java/ru/vk/itmo/test/danilinandrew/MyFactory.java @@ -3,14 +3,15 @@ import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import ru.vk.itmo.danilinandrew.InMemoryDao; +import ru.vk.itmo.danilinandrew.StorageDao; import ru.vk.itmo.test.DaoFactory; +import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 2) +@DaoFactory(stage = 4) public class MyFactory implements DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { @@ -23,8 +24,8 @@ public MemorySegment fromString(String data) { } @Override - public Dao> createDao(Config config) { - return new InMemoryDao(config); + public Dao> createDao(Config config) throws IOException { + return new StorageDao(config); } @Override diff --git a/src/main/java/ru/vk/itmo/test/dyagayalexandra/MemorySegmentDaoFactory.java b/src/main/java/ru/vk/itmo/test/dyagayalexandra/MemorySegmentDaoFactory.java index b2fdb9e4c..65e6a1bea 100644 --- a/src/main/java/ru/vk/itmo/test/dyagayalexandra/MemorySegmentDaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/dyagayalexandra/MemorySegmentDaoFactory.java @@ -10,7 +10,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 3) +@DaoFactory(stage = 5) public class MemorySegmentDaoFactory implements DaoFactory.Factory> { @Override diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDao.java index a943c5a9a..7f39643db 100644 --- a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDao.java @@ -109,14 +109,48 @@ public void upsert(Entry entry) { public void flush() throws IOException { long currentTimeMillis = System.currentTimeMillis(); long nanoTime = System.nanoTime(); - Path filePath = sstablesPath.resolve( + Path filePath = getFilePath(currentTimeMillis, nanoTime); + dumpToFile(filePath, currentTimeMillis, nanoTime); + } + + private Path getFilePath(long curTime, long nanoTime) { + return sstablesPath.resolve( Path.of( - Long.toString(currentTimeMillis, Character.MAX_RADIX) - + Long.toString(nanoTime, Character.MAX_RADIX) - + SSTABLE_SUFFIX + Long.toString(curTime, Character.MAX_RADIX) + + Long.toString(nanoTime, Character.MAX_RADIX) + + SSTABLE_SUFFIX ) ); - dumpToFile(filePath, currentTimeMillis, nanoTime); + } + + @Override + public void compact() throws IOException { + long fileSize = 2 * Long.BYTES + Integer.BYTES; + int countOfKeys = 0; + Iterator> it = all(); + while (it.hasNext()) { + Entry entry = it.next(); + fileSize += entry.key().byteSize() + entry.value().byteSize(); + countOfKeys++; + } + long curTimeMillis = System.currentTimeMillis(); + long nanoTime = System.nanoTime(); + + fileSize += 2L * countOfKeys * Long.BYTES; + Path filePath = getFilePath(curTimeMillis, nanoTime); + Set openOptions = + Set.of( + StandardOpenOption.CREATE, + StandardOpenOption.READ, + StandardOpenOption.WRITE + ); + try (FileChannel fc = FileChannel.open(filePath, openOptions); Arena writeArena = Arena.ofConfined()) { + MemorySegment mapped = fc.map(READ_WRITE, 0, fileSize, writeArena); + dumpToMemSegment(mapped, all(), countOfKeys, curTimeMillis, nanoTime); + } + for (Path file: filesSet) { + Files.delete(file); + } filesSet.add(filePath); } @@ -168,36 +202,42 @@ private void dumpToFile(Path path, long currentTimeMillis, long nanoTime) throws size += Integer.BYTES + (2L * mappings.size() + 2) * Long.BYTES; try (FileChannel fc = FileChannel.open(path, OPEN_OPTIONS); Arena writeArena = Arena.ofConfined()) { MemorySegment mapped = fc.map(READ_WRITE, 0, size, writeArena); - long offset = 0; - mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, currentTimeMillis); - offset += Long.BYTES; - mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, nanoTime); + dumpToMemSegment(mapped, getFromMemory(null, null), mappings.size(), currentTimeMillis, nanoTime); + } + } + + private void dumpToMemSegment(MemorySegment mapped, Iterator> iterator, + int numOfKeys, long curTime, long nanoTime) { + long offset = 0; + mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, curTime); + offset += Long.BYTES; + mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, nanoTime); + offset += Long.BYTES; + mapped.set(ValueLayout.JAVA_INT_UNALIGNED, offset, numOfKeys); + offset += Integer.BYTES; + long offsetToWrite = Integer.BYTES + (2L * numOfKeys + 2) * Long.BYTES; + while (iterator.hasNext()) { + Entry entry = iterator.next(); + mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, offsetToWrite); offset += Long.BYTES; - mapped.set(ValueLayout.JAVA_INT_UNALIGNED, offset, mappings.size()); - offset += Integer.BYTES; - long offsetToWrite = Integer.BYTES + (2L * mappings.size() + 2) * Long.BYTES; - for (Entry entry: mappings.values()) { + MemorySegment.copy( + entry.key(), 0, + mapped, offsetToWrite, + entry.key().byteSize() + ); + offsetToWrite += entry.key().byteSize(); + if (entry.value() == null) { + mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, -1); + } else { mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, offsetToWrite); - offset += Long.BYTES; MemorySegment.copy( - entry.key(), 0, + entry.value(), 0, mapped, offsetToWrite, - entry.key().byteSize() + entry.value().byteSize() ); - offsetToWrite += entry.key().byteSize(); - if (entry.value() == null) { - mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, -1); - } else { - mapped.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, offsetToWrite); - MemorySegment.copy( - entry.value(), 0, - mapped, offsetToWrite, - entry.value().byteSize() - ); - offsetToWrite += entry.value().byteSize(); - } - offset += Long.BYTES; + offsetToWrite += entry.value().byteSize(); } + offset += Long.BYTES; } } diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDaoFactory.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDaoFactory.java index 8dd186d93..8066f180d 100644 --- a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/InMemoryDaoFactory.java @@ -10,7 +10,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 3) +@DaoFactory(stage = 4) public class InMemoryDaoFactory implements DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { diff --git a/src/main/java/ru/vk/itmo/test/grunskiialexey/MemorySegmentDaoFactory.java b/src/main/java/ru/vk/itmo/test/grunskiialexey/MemorySegmentDaoFactory.java index 4164dbcba..fb1241488 100644 --- a/src/main/java/ru/vk/itmo/test/grunskiialexey/MemorySegmentDaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/grunskiialexey/MemorySegmentDaoFactory.java @@ -11,7 +11,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 2) +@DaoFactory(stage = 4) public class MemorySegmentDaoFactory implements DaoFactory.Factory> { @Override diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java index 740ee6d7a..bba99fa88 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoFactory.java @@ -9,7 +9,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@ru.vk.itmo.test.DaoFactory(stage = 3, week = 100) +@ru.vk.itmo.test.DaoFactory(stage = 4, week = 1111) public class DaoFactory implements ru.vk.itmo.test.DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { @@ -34,11 +34,11 @@ public Entry fromBaseEntry(Entry baseEntry) { @Override public Dao> createDao() { - return new InMemoryDao(); + return new DaoWithCompaction(); } @Override public Dao> createDao(Config config) throws IOException { - return new InMemoryDao(config); + return new DaoWithCompaction(config); } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java new file mode 100644 index 000000000..b18717438 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/DaoWithCompaction.java @@ -0,0 +1,21 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.Config; +import java.io.IOException; + +public class DaoWithCompaction extends InMemoryDao { + public DaoWithCompaction() { + super(); + } + + public DaoWithCompaction(Config conf) { + super(conf); + } + + @Override + public void compact() throws IOException { + controller.dumpIterator(new SSTableIterable(getMemTable().values(), controller, null, null)); + closeMemTable(); + controller.deleteAllOldFiles(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index 030f3ab6b..516bb75d5 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -17,7 +17,7 @@ public class InMemoryDao implements Dao> { private final SortedMap> mp = new ConcurrentSkipListMap<>(memorySegmentComparatorImpl); - private final SSTablesController controller; + protected final SSTablesController controller; public InMemoryDao() { this.controller = new SSTablesController(new MemSegComparatorNull()); @@ -53,6 +53,7 @@ public Entry get(MemorySegment key) { if (value != null) { return value.value() == null ? null : value; } + var res = controller.getRow(controller.searchInSStables(key)); if (res == null) { return null; @@ -67,7 +68,19 @@ public void upsert(Entry entry) { @Override public void close() throws IOException { - controller.dumpMemTableToSStable(mp); + try { + controller.dumpIterator(mp.values()); + } finally { + mp.clear(); + } + } + + protected void closeMemTable() { mp.clear(); } + + protected SortedMap> getMemTable() { + return mp; + } + } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java b/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java index a7e1e6b6f..eaff57f3c 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/MemSegComparatorNull.java @@ -5,9 +5,6 @@ public class MemSegComparatorNull extends MemorySegmentComparator { @Override public int compare(MemorySegment segment1, MemorySegment segment2) { - if (segment1 == null && segment2 == null) { - throw new IllegalArgumentException("Incomparable null and null"); - } if (segment1 == null) { return -1; } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java new file mode 100644 index 000000000..9c875c2c2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterable.java @@ -0,0 +1,28 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Collection; +import java.util.Iterator; + +public class SSTableIterable implements Iterable> { + private final Collection> memTable; + private final SSTablesController controller; + private final MemorySegment from; + private final MemorySegment to; + + public SSTableIterable(Collection> it, SSTablesController controller, + MemorySegment from, MemorySegment to) { + this.memTable = it; + this.controller = controller; + + this.from = from; + this.to = to; + } + + @Override + public Iterator> iterator() { + return new SSTableIterator(memTable.iterator(), controller, from, to); + } +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 6e096bc94..6d53ce9b3 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -23,7 +23,7 @@ public class SSTableIterator implements Iterator> { public SSTableIterator(Iterator> it, SSTablesController controller, MemorySegment from, MemorySegment to) { - this.memTableIterator = it; + memTableIterator = it; this.controller = controller; this.from = from; @@ -32,24 +32,26 @@ public SSTableIterator(Iterator> it, SSTablesController con positioningIterator(); } - private void insertNew(final SSTableRowInfo info) { - var curInfo = info; - Entry kv = controller.getRow(curInfo); + private void insertNew(SSTableRowInfo info) { + Entry kv = controller.getRow(info); - while (kv != null) { - SSTableRowInfo old = mp.putIfAbsent(kv.key(), curInfo); - if (old == null) { - return; - } + if (kv == null) { + return; + } - SSTableRowInfo oldInfo = old.ssTableInd > curInfo.ssTableInd ? curInfo : old; - SSTableRowInfo newInfo = old.ssTableInd < curInfo.ssTableInd ? curInfo : old; + if (!mp.containsKey(kv.key())) { + mp.put(kv.key(), info); + return; + } + SSTableRowInfo old = mp.get(kv.key()); - mp.put(controller.getRow(newInfo).key(), newInfo); + SSTableRowInfo oldInfo = old.ssTableInd > info.ssTableInd ? info : old; + SSTableRowInfo newInfo = old.ssTableInd < info.ssTableInd ? info : old; - curInfo = controller.getNextInfo(oldInfo, to); - kv = controller.getRow(curInfo); - } + mp.put(controller.getRow(newInfo).key(), newInfo); + + // tail recursion + insertNew(controller.getNextInfo(oldInfo, to)); } private void positioningIterator() { diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java index 29716eb73..309a26400 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java @@ -1,12 +1,12 @@ package ru.vk.itmo.test.kachmareugene; public class SSTableRowInfo { - final long keyOffset; - final long valueOffset; - final long keySize; - final long rowShift; + long keyOffset; + long valueOffset; + long keySize; + long rowShift; private final long valueSize; - final int ssTableInd; + int ssTableInd; public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, long valueSize, int ssTableInd, long rowShift) { @@ -25,4 +25,8 @@ public boolean isDeletedData() { public long getValueSize() { return valueSize; } + + public long totalShift() { + return keyOffset + keySize + valueSize; + } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 7dfec8800..7341db9e3 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -8,38 +8,39 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; +import java.nio.file.OpenOption; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; +import java.util.Iterator; import java.util.List; -import java.util.SortedMap; +import java.util.Set; import java.util.stream.Stream; +import static ru.vk.itmo.test.kachmareugene.Utils.getValueOrNull; + public class SSTablesController { private final Path ssTablesDir; - private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss.SSSSS"); private final List ssTables = new ArrayList<>(); - private final List ssTablesIndexes = new ArrayList<>(); + private final List ssTablesPaths = new ArrayList<>(); private static final String SS_TABLE_COMMON_PREF = "ssTable"; - - // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen + // index format: (long) keyOffset, (long) keyLen, (long) valueOffset, (long) valueLen private static final long ONE_LINE_SIZE = 4 * Long.BYTES; - private static final String INDEX_COMMON_PREF = "index"; + private static final Set options = Set.of(StandardOpenOption.WRITE, StandardOpenOption.READ, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); private final Arena arenaForReading = Arena.ofShared(); + private boolean isClosedArena; private final Comparator segComp; public SSTablesController(Path dir, Comparator com) { this.ssTablesDir = dir; this.segComp = com; - openFiles(dir, SS_TABLE_COMMON_PREF, ssTables); - openFiles(dir, INDEX_COMMON_PREF, ssTablesIndexes); + ssTablesPaths.addAll(openFiles(dir, SS_TABLE_COMMON_PREF, ssTables)); } public SSTablesController(Comparator com) { @@ -47,7 +48,7 @@ public SSTablesController(Comparator com) { this.segComp = com; } - private void openFiles(Path dir, String fileNamePref, List storage) { + private List openFiles(Path dir, String fileNamePref, List storage) { try { Files.createDirectories(dir); } catch (IOException e) { @@ -56,7 +57,7 @@ private void openFiles(Path dir, String fileNamePref, List storag try (Stream tabels = Files.find(dir, 1, (path, ignore) -> path.getFileName().toString().startsWith(fileNamePref))) { final List list = new ArrayList<>(tabels.toList()); - Collections.sort(list); + Utils.sortByNames(list, fileNamePref); list.forEach(t -> { try (FileChannel channel = FileChannel.open(t, StandardOpenOption.READ)) { storage.add(channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size(), arenaForReading)); @@ -64,33 +65,33 @@ private void openFiles(Path dir, String fileNamePref, List storag throw new UncheckedIOException(e); } }); + return list; } catch (IOException e) { throw new UncheckedIOException(e); } } - private boolean greaterThen(MemorySegment mappedIndex, long lineInBytesOffset, - MemorySegment mappedData, MemorySegment key) { - long offset = mappedIndex.get(ValueLayout.JAVA_LONG, lineInBytesOffset); - long size = mappedIndex.get(ValueLayout.JAVA_LONG, lineInBytesOffset + Long.BYTES); - return segComp.compare(key, mappedData.asSlice(offset, size)) > 0; + private boolean greaterThen(long keyOffset, long keySize, + MemorySegment mapped, MemorySegment key) { + + return segComp.compare(key, mapped.asSlice(keyOffset, keySize)) > 0; } //Gives offset for line in index file - private long searchKeyInFile(MemorySegment mappedIndex, MemorySegment mappedData, MemorySegment key) { + private long searchKeyInFile(int ind, MemorySegment mapped, MemorySegment key) { long l = -1; - long r = mappedIndex.byteSize() / ONE_LINE_SIZE; + long r = getNumberOfEntries(mapped); while (r - l > 1) { long mid = (l + r) / 2; - - if (greaterThen(mappedIndex, mid * ONE_LINE_SIZE, mappedData, key)) { + SSTableRowInfo info = createRowInfo(ind, mid); + if (greaterThen(info.keyOffset, info.keySize, mapped, key)) { l = mid; } else { r = mid; } } - return r == (mappedIndex.byteSize() / ONE_LINE_SIZE) ? -1 : r; + return r == getNumberOfEntries(mapped) ? -1 : r; } //return - List ordered form the latest created sstable to the first. @@ -98,7 +99,10 @@ public List firstGreaterKeys(MemorySegment key) { List ans = new ArrayList<>(); for (int i = ssTables.size() - 1; i >= 0; i--) { - long entryIndexesLine = searchKeyInFile(ssTablesIndexes.get(i), ssTables.get(i), key); + long entryIndexesLine = 0; + if (key != null) { + entryIndexesLine = searchKeyInFile(i, ssTables.get(i), key); + } if (entryIndexesLine < 0) { continue; } @@ -107,18 +111,18 @@ public List firstGreaterKeys(MemorySegment key) { return ans; } - private SSTableRowInfo createRowInfo(int ind, long rowShift) { - long start = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, rowShift * ONE_LINE_SIZE); - long size = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG, rowShift * ONE_LINE_SIZE + Long.BYTES); + private SSTableRowInfo createRowInfo(int ind, final long rowIndex) { + long start = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED, rowIndex * ONE_LINE_SIZE + Long.BYTES); + long size = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED, rowIndex * ONE_LINE_SIZE + Long.BYTES * 2); - long start1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,rowShift * ONE_LINE_SIZE + Long.BYTES * 2); - long size1 = ssTablesIndexes.get(ind).get(ValueLayout.JAVA_LONG,rowShift * ONE_LINE_SIZE + Long.BYTES * 3); - return new SSTableRowInfo(start, size, start1, size1, ind, rowShift); + long start1 = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED,rowIndex * ONE_LINE_SIZE + Long.BYTES * 3); + long size1 = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED,rowIndex * ONE_LINE_SIZE + Long.BYTES * 4); + return new SSTableRowInfo(start, size, start1, size1, ind, rowIndex); } public SSTableRowInfo searchInSStables(MemorySegment key) { - for (int i = ssTablesIndexes.size() - 1; i >= 0; i--) { - long ind = searchKeyInFile(ssTablesIndexes.get(i), ssTables.get(i), key); + for (int i = ssTables.size() - 1; i >= 0; i--) { + long ind = searchKeyInFile(i, ssTables.get(i), key); if (ind >= 0) { return createRowInfo(i, ind); } @@ -145,7 +149,7 @@ public Entry getRow(SSTableRowInfo info) { * Ignores deleted values. */ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { - for (long t = info.rowShift + 1; t < ssTablesIndexes.get(info.ssTableInd).byteSize() / ONE_LINE_SIZE; t++) { + for (long t = info.rowShift + 1; t < getNumberOfEntries(ssTables.get(info.ssTableInd)); t++) { var inf = createRowInfo(info.ssTableInd, t); Entry row = getRow(inf); @@ -153,78 +157,88 @@ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { return inf; } } - return null; } - private long dumpLong(MemorySegment mapped, long value, long offset) { - mapped.set(ValueLayout.JAVA_LONG, offset, value); - return offset + Long.BYTES; + private long getNumberOfEntries(MemorySegment memSeg) { + return memSeg.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); } - private long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) { - MemorySegment.copy(data, 0, mapped, offset, data.byteSize()); - return offset + data.byteSize(); - } - - public void dumpMemTableToSStable(SortedMap> mp) throws IOException { + public void dumpIterator(Iterable> iter) throws IOException { + Iterator> iter1 = iter.iterator(); - if (ssTablesDir == null || mp.isEmpty()) { - arenaForReading.close(); + if (ssTablesDir == null || !iter1.hasNext()) { + closeArena(); return; } - LocalDateTime time = LocalDateTime.now(ZoneId.systemDefault()); + String suff = String.valueOf(Utils.getMaxNumberOfFile(ssTablesDir, SS_TABLE_COMMON_PREF) + 1); + + final Path tmpFile = ssTablesDir.resolve("data.tmp"); + final Path targetFile = ssTablesDir.resolve(SS_TABLE_COMMON_PREF + suff); + + try { + Files.createFile(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + suff)); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state + } + try (FileChannel ssTableChannel = - FileChannel.open(ssTablesDir.resolve(SS_TABLE_COMMON_PREF + formatter.format(time)), - StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE); - FileChannel indexChannel = - FileChannel.open(ssTablesDir.resolve(INDEX_COMMON_PREF + formatter.format(time)), - StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE); + FileChannel.open(tmpFile, options); Arena saveArena = Arena.ofConfined()) { long ssTableLenght = 0L; - long indexLength = mp.size() * ONE_LINE_SIZE; + long indexLength = 0L; - for (var kv : mp.values()) { - ssTableLenght += - kv.key().byteSize() + getValueOrNull(kv).byteSize(); + while (iter1.hasNext()) { + var seg = iter1.next(); + ssTableLenght += seg.key().byteSize() + getValueOrNull(seg).byteSize(); + indexLength += ONE_LINE_SIZE; } - long currOffsetSSTable = 0L; - long currOffsetIndex = 0L; MemorySegment mappedSSTable = ssTableChannel.map( - FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght, saveArena); + FileChannel.MapMode.READ_WRITE, currOffsetSSTable, ssTableLenght + indexLength + Long.BYTES, + saveArena); - MemorySegment mappedIndex = indexChannel.map( - FileChannel.MapMode.READ_WRITE, currOffsetIndex, indexLength, saveArena); + currOffsetSSTable = Utils.dumpLong(mappedSSTable, indexLength / ONE_LINE_SIZE, currOffsetSSTable); - for (var kv : mp.values()) { - currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); - currOffsetSSTable = dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); - currOffsetIndex = dumpLong(mappedIndex, kv.key().byteSize(), currOffsetIndex); + long shiftForData = indexLength + Long.BYTES; + + for (Entry kv : iter) { + // key offset + currOffsetSSTable = Utils.dumpLong(mappedSSTable, shiftForData, currOffsetSSTable); + // key length + currOffsetSSTable = Utils.dumpLong(mappedSSTable, kv.key().byteSize(), currOffsetSSTable); + shiftForData += kv.key().byteSize(); + + // value offset + currOffsetSSTable = Utils.dumpLong(mappedSSTable, shiftForData, currOffsetSSTable); + // value length + currOffsetSSTable = Utils.dumpLong(mappedSSTable, Utils.rightByteSize(kv), currOffsetSSTable); + shiftForData += getValueOrNull(kv).byteSize(); + } - currOffsetIndex = dumpLong(mappedIndex, currOffsetSSTable, currOffsetIndex); - currOffsetSSTable = dumpSegment(mappedSSTable, getValueOrNull(kv), currOffsetSSTable); - currOffsetIndex = dumpLong(mappedIndex, rightByteSize(kv), currOffsetIndex); + for (Entry kv : iter) { + currOffsetSSTable = Utils.dumpSegment(mappedSSTable, kv.key(), currOffsetSSTable); + currOffsetSSTable = Utils.dumpSegment(mappedSSTable, getValueOrNull(kv), currOffsetSSTable); } + + Files.move(tmpFile, targetFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } finally { - arenaForReading.close(); + closeArena(); } } - private long rightByteSize(Entry memSeg) { - if (memSeg.value() == null) { - return -1; + private void closeArena() { + if (!isClosedArena) { + arenaForReading.close(); } - return memSeg.value().byteSize(); + isClosedArena = true; } - private MemorySegment getValueOrNull(Entry kv) { - MemorySegment value = kv.value(); - if (kv.value() == null) { - value = MemorySegment.NULL; - } - return value; + public void deleteAllOldFiles() throws IOException { + closeArena(); + Utils.deleteFiles(ssTablesPaths); } } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java new file mode 100644 index 000000000..77886d9d3 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java @@ -0,0 +1,75 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.Entry; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Stream; + +public final class Utils { + private Utils() { + + } + + public static long dumpLong(MemorySegment mapped, long value, long offset) { + mapped.set(ValueLayout.JAVA_LONG, offset, value); + return offset + Long.BYTES; + } + + public static long dumpSegment(MemorySegment mapped, MemorySegment data, long offset) { + MemorySegment.copy(data, 0, mapped, offset, data.byteSize()); + return offset + data.byteSize(); + } + + public static long getNumberFromFileName(Path pathToFile, String prefix) { + return Long.parseLong(pathToFile.getFileName().toString().substring(prefix.length())); + } + + public static void sortByNames(List l, String prefix) { + l.sort(Comparator.comparingLong(s -> getNumberFromFileName(s, prefix))); + } + + public static long getMaxNumberOfFile(Path dir, String prefix) { + try (Stream tabels = Files.find(dir, 1, + (path, ignore) -> path.getFileName().toString().startsWith(prefix))) { + final List list = tabels.toList(); + long maxi = 0; + for (Path p : list) { + if (getNumberFromFileName(p, prefix) > maxi) { + maxi = getNumberFromFileName(p, prefix); + } + } + return maxi; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static void deleteFiles(List files) throws IOException { + for (Path file : files) { + Files.deleteIfExists(file); + } + files.clear(); + } + + public static long rightByteSize(Entry memSeg) { + if (memSeg.value() == null) { + return -1; + } + return memSeg.value().byteSize(); + } + + public static MemorySegment getValueOrNull(Entry kv) { + MemorySegment value = kv.value(); + if (kv.value() == null) { + value = MemorySegment.NULL; + } + return value; + } +} diff --git a/src/main/java/ru/vk/itmo/test/kovalchukvladislav/MemorySegmentDaoFactory.java b/src/main/java/ru/vk/itmo/test/kovalchukvladislav/MemorySegmentDaoFactory.java index c96427e2f..7867c9fd4 100644 --- a/src/main/java/ru/vk/itmo/test/kovalchukvladislav/MemorySegmentDaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/kovalchukvladislav/MemorySegmentDaoFactory.java @@ -12,7 +12,7 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 3) +@DaoFactory(stage = 5) public class MemorySegmentDaoFactory implements DaoFactory.Factory> { private static final Charset CHARSET = StandardCharsets.UTF_8; private static final ValueLayout.OfByte VALUE_LAYOUT = ValueLayout.JAVA_BYTE; diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/DaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/kovalevigor/DaoFactoryImpl.java index f444b48ae..e346e1a2e 100644 --- a/src/main/java/ru/vk/itmo/test/kovalevigor/DaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/DaoFactoryImpl.java @@ -12,7 +12,7 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 2) +@DaoFactory(stage = 5) public class DaoFactoryImpl implements DaoFactory.Factory> { public static final Charset CHARSET = StandardCharsets.UTF_8; diff --git a/src/main/java/ru/vk/itmo/test/novichkovandrew/DaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/novichkovandrew/DaoFactoryImpl.java index 8beb254f3..fd0db9c2c 100644 --- a/src/main/java/ru/vk/itmo/test/novichkovandrew/DaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/novichkovandrew/DaoFactoryImpl.java @@ -12,8 +12,9 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 2) +@DaoFactory(stage = 3) public class DaoFactoryImpl implements DaoFactory.Factory> { + @Override public Dao> createDao() { return new InMemoryDao(); diff --git a/src/main/java/ru/vk/itmo/test/osipovdaniil/MyFactory.java b/src/main/java/ru/vk/itmo/test/osipovdaniil/MyFactory.java index 49432aca8..37b8fd6fd 100644 --- a/src/main/java/ru/vk/itmo/test/osipovdaniil/MyFactory.java +++ b/src/main/java/ru/vk/itmo/test/osipovdaniil/MyFactory.java @@ -6,20 +6,16 @@ import ru.vk.itmo.osipovdaniil.InMemoryDao; import ru.vk.itmo.test.DaoFactory; +import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 2) +@DaoFactory(stage = 4) public class MyFactory implements DaoFactory.Factory> { - - @Override - public Dao> createDao() { - return new InMemoryDao(); - } - + @Override - public Dao> createDao(final Config config) { + public Dao> createDao(final Config config) throws IOException { return new InMemoryDao(config); } diff --git a/src/main/java/ru/vk/itmo/test/osokindmitry/MyFactory.java b/src/main/java/ru/vk/itmo/test/osokindmitry/DmitFactory.java similarity index 77% rename from src/main/java/ru/vk/itmo/test/osokindmitry/MyFactory.java rename to src/main/java/ru/vk/itmo/test/osokindmitry/DmitFactory.java index 764c8c95c..65943cbc4 100644 --- a/src/main/java/ru/vk/itmo/test/osokindmitry/MyFactory.java +++ b/src/main/java/ru/vk/itmo/test/osokindmitry/DmitFactory.java @@ -3,19 +3,20 @@ import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import ru.vk.itmo.osokindmitry.InMemoryDao; +import ru.vk.itmo.osokindmitry.PersistentDao; import ru.vk.itmo.test.DaoFactory; +import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 2) -public class MyFactory implements DaoFactory.Factory> { +@DaoFactory(stage = 4) +public class DmitFactory implements DaoFactory.Factory> { @Override - public Dao> createDao(Config config) { - return new InMemoryDao(config); + public Dao> createDao(Config config) throws IOException { + return new PersistentDao(config); } @Override diff --git a/src/main/java/ru/vk/itmo/test/prokopyevnikita/FactoryImpl.java b/src/main/java/ru/vk/itmo/test/prokopyevnikita/FactoryImpl.java index e412db6fa..73258dc9f 100644 --- a/src/main/java/ru/vk/itmo/test/prokopyevnikita/FactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/prokopyevnikita/FactoryImpl.java @@ -12,7 +12,7 @@ import static java.lang.foreign.ValueLayout.JAVA_BYTE; -@DaoFactory(stage = 4) +@DaoFactory(stage = 5) public class FactoryImpl implements DaoFactory.Factory> { @Override public Dao> createDao(Config config) throws IOException { diff --git a/src/main/java/ru/vk/itmo/test/reference/ReferenceDaoFactory.java b/src/main/java/ru/vk/itmo/test/reference/ReferenceDaoFactory.java new file mode 100644 index 000000000..5b410908f --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/ReferenceDaoFactory.java @@ -0,0 +1,49 @@ +package ru.vk.itmo.test.reference; + +import ru.vk.itmo.Config; +import ru.vk.itmo.Dao; +import ru.vk.itmo.Entry; +import ru.vk.itmo.reference.ReferenceDao; +import ru.vk.itmo.test.DaoFactory; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.charset.StandardCharsets; + +/** + * Instantiates {@link ReferenceDao}. + * + * @author incubos + */ +@DaoFactory(stage = 5) +public class ReferenceDaoFactory implements DaoFactory.Factory> { + @Override + public Dao> createDao(final Config config) throws IOException { + return new ReferenceDao(config); + } + + @Override + public String toString(final MemorySegment memorySegment) { + if (memorySegment == null) { + return null; + } + + final byte[] array = memorySegment.toArray(ValueLayout.JAVA_BYTE); + return new String(array, StandardCharsets.UTF_8); + } + + @Override + public MemorySegment fromString(final String data) { + return data == null + ? null + : MemorySegment.ofArray( + data.getBytes( + StandardCharsets.UTF_8)); + } + + @Override + public Entry fromBaseEntry(final Entry baseEntry) { + return baseEntry; + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/DaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/DaoFactoryImpl.java index c9ecc4f07..8ae493ef1 100644 --- a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/DaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/DaoFactoryImpl.java @@ -1,15 +1,17 @@ package ru.vk.itmo.test.reshetnikovaleksei; +import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; -import ru.vk.itmo.reshetnikovaleksei.InMemoryDao; +import ru.vk.itmo.reshetnikovaleksei.DaoImpl; import ru.vk.itmo.test.DaoFactory; +import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory +@DaoFactory(stage = 4) public class DaoFactoryImpl implements DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { @@ -32,6 +34,11 @@ public Entry fromBaseEntry(Entry baseEntry) { @Override public Dao> createDao() { - return new InMemoryDao(); + return null; + } + + @Override + public Dao> createDao(Config config) throws IOException { + return new DaoImpl(config); } } diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDao.java index 392585fa2..74006fe9a 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDao.java @@ -3,12 +3,14 @@ import ru.vk.itmo.Config; import ru.vk.itmo.Dao; import ru.vk.itmo.Entry; +import ru.vk.itmo.test.ryabovvadim.iterators.EntrySkipNullsIterator; import ru.vk.itmo.test.ryabovvadim.iterators.FutureIterator; import ru.vk.itmo.test.ryabovvadim.iterators.GatheringIterator; import ru.vk.itmo.test.ryabovvadim.iterators.LazyIterator; import ru.vk.itmo.test.ryabovvadim.iterators.PriorityIterator; import ru.vk.itmo.test.ryabovvadim.utils.FileUtils; import ru.vk.itmo.test.ryabovvadim.utils.MemorySegmentUtils; +import ru.vk.itmo.test.ryabovvadim.utils.NumberUtils; import java.io.IOException; import java.lang.foreign.Arena; @@ -36,7 +38,10 @@ public class InMemoryDao implements Dao> { private final ConcurrentNavigableMap> memoryTable = new ConcurrentSkipListMap<>(MemorySegmentUtils::compareMemorySegments); private final Config config; - private final List ssTables = new ArrayList<>(); + + private final NavigableSet ssTables = new TreeSet<>( + Comparator.comparingLong(SSTable::getId).reversed() + ); public InMemoryDao() throws IOException { this(null); @@ -49,29 +54,9 @@ public InMemoryDao(Config config) throws IOException { } if (Files.notExists(config.basePath())) { - Files.createDirectory(config.basePath()); - } - - NavigableSet dataFileNumbers = new TreeSet<>(Comparator.reverseOrder()); - Files.walkFileTree(config.basePath(), Set.of(), 1, new SimpleFileVisitor<>() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - if (file.getFileName().toString().endsWith("." + DATA_FILE_EXT)) { - dataFileNumbers.add(Long.parseLong( - file.getFileName().toString().substring( - 0, - file.getFileName().toString().indexOf("." + DATA_FILE_EXT) - ) - )); - } - - return FileVisitResult.CONTINUE; - } - }); - - for (long number : dataFileNumbers) { - ssTables.add(new SSTable(config.basePath(), Long.toString(number), arena)); + Files.createDirectories(config.basePath()); } + updateSSTables(true); } @Override @@ -90,7 +75,7 @@ public Iterator> allFrom(MemorySegment from) { return all(); } - return makeIteratorWithSkipNulls(memoryTable.tailMap(from), from, null); + return makeIteratorWithSkipNulls(memoryTable.tailMap(from), load(from, null)); } @Override @@ -99,12 +84,12 @@ public Iterator> allTo(MemorySegment to) { return all(); } - return makeIteratorWithSkipNulls(memoryTable.headMap(to), null, to); + return makeIteratorWithSkipNulls(memoryTable.headMap(to), load(null, to)); } @Override public Iterator> all() { - return makeIteratorWithSkipNulls(memoryTable, null, null); + return makeIteratorWithSkipNulls(memoryTable, load(null, null)); } @Override @@ -116,7 +101,7 @@ public Iterator> get(MemorySegment from, MemorySegment to) return allFrom(from); } - return makeIteratorWithSkipNulls(memoryTable.subMap(from, to), from, to); + return makeIteratorWithSkipNulls(memoryTable.subMap(from, to), load(from, to)); } private Entry load(MemorySegment key) { @@ -155,22 +140,20 @@ private Entry handleDeletededEntry(Entry entry) { } private FutureIterator> makeIteratorWithSkipNulls( - Map> entries, - MemorySegment from, - MemorySegment to + Map> memoryEntries, + List>> loadedIterators ) { - List>> loadedIterators = load(from, to); - Iterator> entriesIterator = entries.values().iterator(); + Iterator> entriesIterator = memoryEntries.values().iterator(); + + if (loadedIterators.isEmpty()) { + return new EntrySkipNullsIterator(entriesIterator); + } int priority = 0; List>> priorityIterators = new ArrayList<>(); if (entriesIterator.hasNext()) { - priorityIterators.add(new PriorityIterator<>( - new LazyIterator<>(entriesIterator::next, entriesIterator::hasNext), - priority - )); - ++priority; + priorityIterators.add(new PriorityIterator<>(new LazyIterator<>(entriesIterator), priority++)); } for (FutureIterator> it : loadedIterators) { priorityIterators.add(new PriorityIterator<>(it, priority++)); @@ -179,37 +162,13 @@ private FutureIterator> makeIteratorWithSkipNulls( GatheringIterator> gatheringIterator = new GatheringIterator<>( priorityIterators, Comparator.comparing( - it -> ((PriorityIterator>) it).showNext().key(), + (PriorityIterator> it) -> it.showNext().key(), MemorySegmentUtils::compareMemorySegments - ).thenComparingInt(it -> ((PriorityIterator>) it).getPriority()), + ).thenComparingInt(PriorityIterator::getPriority), Comparator.comparing(Entry::key, MemorySegmentUtils::compareMemorySegments) ); - return new FutureIterator<>() { - @Override - public Entry showNext() { - skipNulls(); - return gatheringIterator.showNext(); - } - - @Override - public boolean hasNext() { - skipNulls(); - return gatheringIterator.hasNext(); - } - - @Override - public Entry next() { - skipNulls(); - return gatheringIterator.next(); - } - - private void skipNulls() { - while (gatheringIterator.hasNext() && gatheringIterator.showNext().value() == null) { - gatheringIterator.next(); - } - } - }; + return new EntrySkipNullsIterator(gatheringIterator); } @Override @@ -220,22 +179,57 @@ public void upsert(Entry entry) { @Override public void flush() throws IOException { if (existsPath() && !memoryTable.isEmpty()) { - String nameSavedTable = saveMemoryTable(config.basePath()); + long ssTableId = saveEntries(memoryTable.values()); memoryTable.clear(); - ssTables.add(new SSTable(config.basePath(), nameSavedTable, arena)); + ssTables.add(new SSTable(config.basePath(), ssTableId, arena)); + } + } + + @Override + public void compact() throws IOException { + if (existsPath()) { + saveEntries(this::all); + + for (SSTable ssTable : ssTables) { + ssTable.delete(); + } + ssTables.clear(); + memoryTable.clear(); + + updateSSTables(false); } } - private String saveMemoryTable(Path path) throws IOException { - FileUtils.createParentDirectories(config.basePath()); + private void updateSSTables(boolean isStartup) throws IOException { + Files.walkFileTree(config.basePath(), Set.of(), 1, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (FileUtils.hasExtension(file, DATA_FILE_EXT) && NumberUtils.isInteger( + FileUtils.extractFileName(file, DATA_FILE_EXT) + )) { + ssTables.add(new SSTable( + config.basePath(), + Long.parseLong(FileUtils.extractFileName(file, DATA_FILE_EXT)), + arena + )); + return FileVisitResult.CONTINUE; + } + if (isStartup) { + Files.deleteIfExists(file); + } + return FileVisitResult.CONTINUE; + } + }); + } + + private long saveEntries(Iterable> entries) throws IOException { long maxTableNumber = 0; for (SSTable ssTable : ssTables) { - maxTableNumber = Math.max(maxTableNumber, Long.parseLong(ssTable.getName())); + maxTableNumber = Math.max(maxTableNumber, ssTable.getId()); } - SSTable.save(path, Long.toString(maxTableNumber + 1), memoryTable.values(), arena); - - return Long.toString(maxTableNumber + 1); + SSTable.save(config.basePath(), maxTableNumber + 1, entries, arena); + return maxTableNumber + 1; } private boolean existsPath() { diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDaoFactory.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDaoFactory.java index 4d9adabea..636e6227b 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDaoFactory.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/InMemoryDaoFactory.java @@ -11,7 +11,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; -@DaoFactory(stage = 3) +@DaoFactory(stage = 4) public class InMemoryDaoFactory implements DaoFactory.Factory> { @Override public String toString(MemorySegment memorySegment) { diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/SSTable.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/SSTable.java index 7ff1f5af1..953eec515 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/SSTable.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/SSTable.java @@ -14,41 +14,31 @@ import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; +import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; +import java.nio.file.StandardCopyOption; import java.util.Iterator; -import java.util.List; +import java.util.NoSuchElementException; import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.READ; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; import static java.nio.file.StandardOpenOption.WRITE; public class SSTable { - private final String name; + private final Path parentPath; + private final long id; private final MemorySegment data; private final int countRecords; - private final List offsets; - public SSTable(Path prefix, String name, Arena arena) throws IOException { - this.name = name; - - Path dataFile = FileUtils.makePath(prefix, name, FileUtils.DATA_FILE_EXT); - Path offsetsFile = FileUtils.makePath(prefix, name, FileUtils.OFFSETS_FILE_EXT); + public SSTable(Path parentPath, long id, Arena arena) throws IOException { + this.parentPath = parentPath; + this.id = id; + Path dataFile = getDataFilePath(); try (FileChannel dataFileChannel = FileChannel.open(dataFile, READ)) { - try (FileChannel offsetsFileChannel = FileChannel.open(offsetsFile, READ)) { - this.data = dataFileChannel.map(MapMode.READ_ONLY, 0, dataFileChannel.size(), arena); - this.countRecords = (int) (offsetsFileChannel.size() / Long.BYTES); - this.offsets = new ArrayList<>(); - - MemorySegment offsetsSegment = offsetsFileChannel.map( - MapMode.READ_ONLY, 0, offsetsFileChannel.size(), arena - ); - for (int i = 0; i < countRecords; ++i) { - offsets.add(offsetsSegment.get(ValueLayout.JAVA_LONG, i * Long.BYTES)); - } - } + this.data = dataFileChannel.map(MapMode.READ_ONLY, 0, dataFileChannel.size(), arena); + this.countRecords = (int) (this.data.get(ValueLayout.JAVA_LONG, 0) / Long.BYTES); } } @@ -57,7 +47,7 @@ public Entry findEntry(MemorySegment key) { if (offsetIndex < 0) { return null; } - return new BaseEntry<>(key, readValue(getRecordInfo(offsets.get(offsetIndex)))); + return new BaseEntry<>(key, readValue(getRecordInfo(getOffset(offsetIndex)))); } public FutureIterator> findEntries(MemorySegment from, MemorySegment to) { @@ -72,7 +62,7 @@ public FutureIterator> findEntries(MemorySegment from, Memo toIndex = toOffsetIndex < 0 ? -toOffsetIndex : toOffsetIndex; } - Iterator offsetsIterator = offsets.subList(fromIndex, toIndex).iterator(); + Iterator offsetsIterator = getOffsetIterator(fromIndex, toIndex); return new LazyIterator<>( () -> { RecordInfo recordInfo = getRecordInfo(offsetsIterator.next()); @@ -82,19 +72,14 @@ public FutureIterator> findEntries(MemorySegment from, Memo ); } - public String getName() { - return name; - } - private int binSearchIndex(MemorySegment key, boolean lowerBound) { int l = -1; int r = countRecords; - while (l + 1 < r) { int mid = (l + r) / 2; - RecordInfo recordInfo = getRecordInfo(offsets.get(mid)); + RecordInfo recordInfo = getRecordInfo(getOffset(mid)); int compareResult = MemorySegmentUtils.compareMemorySegments( - data, recordInfo.getKeyOffset(), recordInfo.getValueOffset(), + data, recordInfo.keyOffset(), recordInfo.valueOffset(), key, 0, key.byteSize() ); @@ -111,150 +96,169 @@ private int binSearchIndex(MemorySegment key, boolean lowerBound) { } private RecordInfo getRecordInfo(long recordOffset) { - long curOffset = recordOffset; - ++curOffset; - - byte sizeInfo = data.get(ValueLayout.JAVA_BYTE, curOffset); - ++curOffset; + long curOffset = recordOffset + 1; + byte sizeInfo = data.get(ValueLayout.JAVA_BYTE, curOffset++); int keySizeSize = sizeInfo >> 4; int valueSizeSize = sizeInfo & 0xf; byte[] keySizeInBytes = new byte[keySizeSize]; for (int i = 0; i < keySizeSize; ++i) { - keySizeInBytes[i] = data.get(ValueLayout.JAVA_BYTE, curOffset); - ++curOffset; + keySizeInBytes[i] = data.get(ValueLayout.JAVA_BYTE, curOffset++); } byte[] valueSizeInBytes = new byte[valueSizeSize]; for (int i = 0; i < valueSizeSize; ++i) { - valueSizeInBytes[i] = data.get(ValueLayout.JAVA_BYTE, curOffset); - ++curOffset; + valueSizeInBytes[i] = data.get(ValueLayout.JAVA_BYTE, curOffset++); } long keySize = NumberUtils.fromBytes(keySizeInBytes); long valueSize = NumberUtils.fromBytes(valueSizeInBytes); byte meta = data.get(ValueLayout.JAVA_BYTE, recordOffset); - return new RecordInfo(meta, keySize, curOffset, valueSize, curOffset + keySize); } private MemorySegment readKey(RecordInfo recordInfo) { - return data.asSlice(recordInfo.getKeyOffset(), recordInfo.getKeySize()); + return data.asSlice(recordInfo.keyOffset(), recordInfo.keySize()); } private MemorySegment readValue(RecordInfo recordInfo) { - if (SSTableMeta.isRemovedValue(recordInfo.getMeta())) { + if (SSTableMeta.isRemovedValue(recordInfo.meta())) { return null; } - return data.asSlice(recordInfo.getValueOffset(), recordInfo.getValueSize()); + return data.asSlice(recordInfo.valueOffset(), recordInfo.valueSize()); + } + + private long getOffset(int index) { + return data.get(ValueLayout.JAVA_LONG, index * Long.BYTES); + } + + private Iterator getOffsetIterator(int fromIndex, int toIndex) { + return new Iterator<>() { + private int curIndex = fromIndex; + + @Override + public boolean hasNext() { + return curIndex < toIndex; + } + + @Override + public Long next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return getOffset(curIndex++); + } + }; } public static void save( Path prefix, - String name, - Collection> entries, + long id, + Iterable> entries, Arena arena ) throws IOException { - Path dataFile = FileUtils.makePath(prefix, name, FileUtils.DATA_FILE_EXT); - Path offsetsFile = FileUtils.makePath(prefix, name, FileUtils.OFFSETS_FILE_EXT); - - try (FileChannel dataFileChannel = FileChannel.open(dataFile, CREATE, WRITE, READ)) { - try (FileChannel offsetsFileChannel = FileChannel.open(offsetsFile, CREATE, WRITE, READ)) { - long dataSize = 0; - for (Entry entry : entries) { - dataSize += 2; - MemorySegment key = entry.key(); - MemorySegment value = entry.value(); - - dataSize += NumberUtils.toBytes(key.byteSize()).length + key.byteSize(); - if (value != null) { - dataSize += NumberUtils.toBytes(value.byteSize()).length + value.byteSize(); - } - } + long dataSize = 0; + int countRecords = 0; + for (Entry entry : entries) { + ++countRecords; + dataSize += 2; + MemorySegment key = entry.key(); + MemorySegment value = entry.value(); + + dataSize += NumberUtils.toBytes(key.byteSize()).length + key.byteSize(); + if (value != null) { + dataSize += NumberUtils.toBytes(value.byteSize()).length + value.byteSize(); + } + } + + if (countRecords == 0) { + return; + } + + Path tmpDataFile = FileUtils.makePath(prefix, Long.toString(id), FileUtils.TMP_FILE_EXT); + try (FileChannel dataFileChannel = FileChannel.open(tmpDataFile, CREATE, WRITE, READ, TRUNCATE_EXISTING)) { + long dataOffset = (long) countRecords * Long.BYTES; + MemorySegment dataSegment = dataFileChannel.map( + MapMode.READ_WRITE, + 0, + dataSize + dataOffset, + arena + ); - MemorySegment dataSegment = dataFileChannel.map( - MapMode.READ_WRITE, 0, dataSize, arena - ); - MemorySegment offsetsSegment = offsetsFileChannel.map( - MapMode.READ_WRITE, 0, Long.BYTES * entries.size(), arena - ); - long dataSegmentOffset = 0; - long offsetsSegmentOffset = 0; - - for (Entry entry : entries) { - MemorySegment key = entry.key(); - MemorySegment value = entry.value(); - byte[] keySizeInBytes = NumberUtils.toBytes(key.byteSize()); - byte[] valueSizeInBytes = value == null - ? new byte[0] - : NumberUtils.toBytes(value.byteSize()); - - offsetsSegment.set(ValueLayout.JAVA_LONG, offsetsSegmentOffset, dataSegmentOffset); - offsetsSegmentOffset += Long.BYTES; - - byte meta = buildMeta(entry); - byte sizeInfo = (byte) ((keySizeInBytes.length << 4) | valueSizeInBytes.length); - - dataSegment.set(ValueLayout.JAVA_BYTE, dataSegmentOffset, meta); - dataSegmentOffset += 1; - dataSegment.set(ValueLayout.JAVA_BYTE, dataSegmentOffset, sizeInfo); - dataSegmentOffset += 1; - - MemorySegment.copy( - keySizeInBytes, - 0, - dataSegment, - ValueLayout.JAVA_BYTE, - dataSegmentOffset, - keySizeInBytes.length - ); - dataSegmentOffset += keySizeInBytes.length; - MemorySegment.copy( - valueSizeInBytes, - 0, - dataSegment, - ValueLayout.JAVA_BYTE, - dataSegmentOffset, - valueSizeInBytes.length - ); - dataSegmentOffset += valueSizeInBytes.length; - MemorySegment.copy(key, 0, dataSegment, dataSegmentOffset, key.byteSize()); - dataSegmentOffset += key.byteSize(); - if (value != null) { - MemorySegment.copy( - value, 0, dataSegment, dataSegmentOffset, value.byteSize() - ); - dataSegmentOffset += value.byteSize(); - } + int curEntryNumber = 0; + for (Entry entry : entries) { + dataSegment.set(ValueLayout.JAVA_LONG, curEntryNumber * Long.BYTES, dataOffset); + + MemorySegment key = entry.key(); + MemorySegment value = entry.value(); + byte[] keySizeInBytes = NumberUtils.toBytes(key.byteSize()); + byte[] valueSizeInBytes = value == null + ? new byte[0] + : NumberUtils.toBytes(value.byteSize()); + + byte meta = SSTableMeta.buildMeta(entry); + byte sizeInfo = (byte) ((keySizeInBytes.length << 4) | valueSizeInBytes.length); + dataSegment.set(ValueLayout.JAVA_BYTE, dataOffset++, meta); + dataSegment.set(ValueLayout.JAVA_BYTE, dataOffset++, sizeInfo); + + MemorySegmentUtils.copyByteArray(keySizeInBytes, dataSegment, dataOffset); + dataOffset += keySizeInBytes.length; + MemorySegmentUtils.copyByteArray(valueSizeInBytes, dataSegment, dataOffset); + dataOffset += valueSizeInBytes.length; + MemorySegment.copy(key, 0, dataSegment, dataOffset, key.byteSize()); + dataOffset += key.byteSize(); + if (value != null) { + MemorySegment.copy(value, 0, dataSegment, dataOffset, value.byteSize()); + dataOffset += value.byteSize(); } + + ++curEntryNumber; } } + + Path dataFile = FileUtils.makePath(prefix, Long.toString(id), FileUtils.DATA_FILE_EXT); + Files.move(tmpDataFile, dataFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } - private static byte buildMeta(Entry entry) { - byte meta = 0; + public void delete() throws IOException { + Files.deleteIfExists(getDataFilePath()); + } - if (entry.value() == null) { - meta |= SSTableMeta.REMOVE_VALUE; - } - return meta; + private Path getDataFilePath() { + return FileUtils.makePath(parentPath, Long.toString(id), FileUtils.DATA_FILE_EXT); } - private static class SSTableMeta { + public long getId() { + return id; + } + + private static final class SSTableMeta { private static final byte REMOVE_VALUE = 0x1; - private static boolean isRemovedValue(byte meta) { + public static boolean isRemovedValue(byte meta) { return (meta & REMOVE_VALUE) == REMOVE_VALUE; } + + public static byte buildMeta(Entry entry) { + byte meta = 0; + + if (entry.value() == null) { + meta |= SSTableMeta.REMOVE_VALUE; + } + return meta; + } + + private SSTableMeta() { + } } - private static class RecordInfo { + private static final class RecordInfo { private final byte meta; private final long keySize; private final long keyOffset; private final long valueSize; private final long valueOffset; - public RecordInfo( + private RecordInfo( byte meta, long keySize, long keyOffset, @@ -268,23 +272,23 @@ public RecordInfo( this.valueOffset = valueOffset; } - public byte getMeta() { + public byte meta() { return meta; } - public long getKeySize() { + public long keySize() { return keySize; } - public long getKeyOffset() { + public long keyOffset() { return keyOffset; } - public long getValueSize() { + public long valueSize() { return valueSize; } - public long getValueOffset() { + public long valueOffset() { return valueOffset; } } diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/EntrySkipNullsIterator.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/EntrySkipNullsIterator.java new file mode 100644 index 000000000..c7216429d --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/EntrySkipNullsIterator.java @@ -0,0 +1,42 @@ +package ru.vk.itmo.test.ryabovvadim.iterators; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; + +public class EntrySkipNullsIterator implements FutureIterator> { + private final FutureIterator> delegate; + + public EntrySkipNullsIterator(Iterator> delegate) { + this.delegate = new LazyIterator<>(delegate); + } + + public EntrySkipNullsIterator(FutureIterator> delegate) { + this.delegate = delegate; + } + + @Override + public Entry showNext() { + skipNulls(); + return delegate.showNext(); + } + + @Override + public boolean hasNext() { + skipNulls(); + return delegate.hasNext(); + } + + @Override + public Entry next() { + skipNulls(); + return delegate.next(); + } + + private void skipNulls() { + while (delegate.hasNext() && delegate.showNext().value() == null) { + delegate.next(); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/LazyIterator.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/LazyIterator.java index b146f4be1..7a0b772a0 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/LazyIterator.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/iterators/LazyIterator.java @@ -1,5 +1,6 @@ package ru.vk.itmo.test.ryabovvadim.iterators; +import java.util.Iterator; import java.util.function.Supplier; public class LazyIterator implements FutureIterator { @@ -7,6 +8,10 @@ public class LazyIterator implements FutureIterator { private final Supplier hasNextEntry; private T next; + public LazyIterator(Iterator iterator) { + this(iterator::next, iterator::hasNext); + } + public LazyIterator(Supplier getEntry, Supplier hasNextEntry) { this.loadEntry = getEntry; this.hasNextEntry = hasNextEntry; diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/FileUtils.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/FileUtils.java index 22da19507..bb747ba87 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/FileUtils.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/FileUtils.java @@ -6,7 +6,8 @@ public final class FileUtils { public static final String DATA_FILE_EXT = "data"; - public static final String OFFSETS_FILE_EXT = "offsets"; + public static final String TMP_FILE_EXT = "tmp"; + private static final String FILE_EXTENSION_DELIMITER = "."; public static Path makePath(Path prefix, String name, String extension) { return Path.of(prefix.toString(), name + "." + extension); @@ -22,7 +23,22 @@ public static void createParentDirectories(Path path) throws IOException { Files.createDirectories(parent); } } - + + public static boolean hasExtension(Path path, String extension) { + return path.getFileName().toString().endsWith(FILE_EXTENSION_DELIMITER + extension); + } + + public static String extractFileName(Path path, String extension) { + String fullFileName = path.getFileName().toString(); + int index = fullFileName.indexOf(FILE_EXTENSION_DELIMITER + extension); + + if (index == -1) { + throw new IllegalArgumentException("File " + path + " doesn't have extension " + extension); + } + + return fullFileName.substring(0, index); + } + private FileUtils() { } } diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/MemorySegmentUtils.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/MemorySegmentUtils.java index 822cdb5c1..2cd3c4a63 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/MemorySegmentUtils.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/MemorySegmentUtils.java @@ -50,6 +50,17 @@ public static int compareMemorySegments( ); } + public static void copyByteArray(byte[] src, MemorySegment dst, long offsetDst) { + MemorySegment.copy( + src, + 0, + dst, + JAVA_BYTE, + offsetDst, + src.length + ); + } + private MemorySegmentUtils() { } } diff --git a/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/NumberUtils.java b/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/NumberUtils.java index ce1f602c4..f099c0cc2 100644 --- a/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/NumberUtils.java +++ b/src/main/java/ru/vk/itmo/test/ryabovvadim/utils/NumberUtils.java @@ -34,6 +34,16 @@ public static byte[] toBytes(long value) { return result; } + + public static boolean isInteger(String s) { + for (int i = 0; i < s.length(); ++i) { + if (!Character.isDigit(s.charAt(i))) { + return false; + } + } + + return true; + } private NumberUtils() { } diff --git a/src/main/java/ru/vk/itmo/test/shemetovalexey/DaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/shemetovalexey/DaoFactoryImpl.java index 4601c1b1c..993c83bbd 100644 --- a/src/main/java/ru/vk/itmo/test/shemetovalexey/DaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/shemetovalexey/DaoFactoryImpl.java @@ -11,7 +11,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 4) +@DaoFactory(stage = 50) public class DaoFactoryImpl implements DaoFactory.Factory> { @Override public Dao> createDao(Config config) throws IOException { diff --git a/src/main/java/ru/vk/itmo/test/solnyshkoksenia/FactoryImpl.java b/src/main/java/ru/vk/itmo/test/solnyshkoksenia/FactoryImpl.java index 7da590ad1..c3f24a1e3 100644 --- a/src/main/java/ru/vk/itmo/test/solnyshkoksenia/FactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/solnyshkoksenia/FactoryImpl.java @@ -12,15 +12,10 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 4) +@DaoFactory(stage = 5) public class FactoryImpl implements DaoFactory.Factory> { private static final Charset UTF_8 = StandardCharsets.UTF_8; - @Override - public Dao> createDao() { - return new DaoImpl(); - } - @Override public Dao> createDao(Config config) throws IOException { return new DaoImpl(config); diff --git a/src/main/java/ru/vk/itmo/test/svistukhinandrey/DaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/svistukhinandrey/DaoFactoryImpl.java index 0907f6628..f045c9629 100644 --- a/src/main/java/ru/vk/itmo/test/svistukhinandrey/DaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/svistukhinandrey/DaoFactoryImpl.java @@ -11,7 +11,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 4) +@DaoFactory(stage = 5) public class DaoFactoryImpl implements DaoFactory.Factory> { @Override diff --git a/src/main/java/ru/vk/itmo/test/tuzikovalexandr/DaoFactoryImpl.java b/src/main/java/ru/vk/itmo/test/tuzikovalexandr/DaoFactoryImpl.java index 84ce44a04..b99eeb29b 100644 --- a/src/main/java/ru/vk/itmo/test/tuzikovalexandr/DaoFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/tuzikovalexandr/DaoFactoryImpl.java @@ -11,7 +11,7 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 2) +@DaoFactory(stage = 4) public class DaoFactoryImpl implements DaoFactory.Factory> { @Override diff --git a/src/main/java/ru/vk/itmo/test/viktorkorotkikh/FactoryImpl.java b/src/main/java/ru/vk/itmo/test/viktorkorotkikh/FactoryImpl.java index ec5b3be70..414d8bad4 100644 --- a/src/main/java/ru/vk/itmo/test/viktorkorotkikh/FactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/viktorkorotkikh/FactoryImpl.java @@ -10,11 +10,11 @@ import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -@DaoFactory(stage = 4, week = 2) +@DaoFactory(stage = 5, week = 2) public class FactoryImpl implements DaoFactory.Factory> { @Override public Dao> createDao(Config config) { - return new LSMDaoImpl(config.basePath()); + return new LSMDaoImpl(config.basePath(), config.flushThresholdBytes()); } @Override diff --git a/src/main/java/ru/vk/itmo/tuzikovalexandr/InMemoryDaoImpl.java b/src/main/java/ru/vk/itmo/tuzikovalexandr/InMemoryDaoImpl.java index 41af88086..9bbb76dc3 100644 --- a/src/main/java/ru/vk/itmo/tuzikovalexandr/InMemoryDaoImpl.java +++ b/src/main/java/ru/vk/itmo/tuzikovalexandr/InMemoryDaoImpl.java @@ -5,38 +5,60 @@ import ru.vk.itmo.Entry; import java.io.IOException; +import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Iterator; import java.util.NavigableMap; import java.util.concurrent.ConcurrentSkipListMap; public class InMemoryDaoImpl implements Dao> { private final NavigableMap> memory = - new ConcurrentSkipListMap<>(new MemorySegmentComparator()); + new ConcurrentSkipListMap<>(MemorySegmentComparator::compare); private final SSTable ssTable; + private final Arena arena; + private final Path path; public InMemoryDaoImpl(Config config) throws IOException { - this.ssTable = new SSTable(config); + this.path = config.basePath().resolve("data"); + Files.createDirectories(path); + + arena = Arena.ofShared(); + + this.ssTable = new SSTable(SSTable.loadData(path, arena)); } @Override public Iterator> get(MemorySegment from, MemorySegment to) { + return ssTable.range(getInMemory(from, to), from, to); + } + public Iterator> getInMemory(MemorySegment from, MemorySegment to) { if (from == null && to == null) { return memory.values().iterator(); } else if (from == null) { return memory.headMap(to, false).values().iterator(); } else if (to == null) { - return memory.subMap(from, true, memory.lastKey(), false).values().iterator(); - } else { - return memory.subMap(from, true, to, false).values().iterator(); + return memory.tailMap(from, true).values().iterator(); } + + return memory.subMap(from, true, to, false).values().iterator(); } @Override public Entry get(MemorySegment key) { - var entry = memory.get(key); - return entry == null ? ssTable.readData(key) : entry; + Entry entry = memory.get(key); + + if (entry == null) { + entry = ssTable.readData(key); + } + + if (entry != null && entry.value() == null) { + return null; + } + + return entry; } @Override @@ -50,16 +72,21 @@ public Iterator> all() { } @Override - public void flush() throws IOException { - throw new UnsupportedOperationException(""); + public void compact() throws IOException { + ssTable.compactData(path, () -> get(null, null)); + memory.clear(); } @Override public void close() throws IOException { - if (memory.isEmpty()) { + if (!arena.scope().isAlive()) { return; } - ssTable.saveMemData(memory.values()); + arena.close(); + + if (!memory.isEmpty()) { + ssTable.saveMemData(path, memory.values()); + } } } diff --git a/src/main/java/ru/vk/itmo/tuzikovalexandr/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/tuzikovalexandr/MemorySegmentComparator.java index 3a7f22f38..59b7691b5 100644 --- a/src/main/java/ru/vk/itmo/tuzikovalexandr/MemorySegmentComparator.java +++ b/src/main/java/ru/vk/itmo/tuzikovalexandr/MemorySegmentComparator.java @@ -2,12 +2,13 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.util.Comparator; -public class MemorySegmentComparator implements Comparator { +public final class MemorySegmentComparator { - @Override - public int compare(MemorySegment o1, MemorySegment o2) { + private MemorySegmentComparator() { + } + + public static int compare(MemorySegment o1, MemorySegment o2) { long offset = o1.mismatch(o2); if (offset == -1) { return 0; diff --git a/src/main/java/ru/vk/itmo/tuzikovalexandr/PeekIterator.java b/src/main/java/ru/vk/itmo/tuzikovalexandr/PeekIterator.java new file mode 100644 index 000000000..680045853 --- /dev/null +++ b/src/main/java/ru/vk/itmo/tuzikovalexandr/PeekIterator.java @@ -0,0 +1,45 @@ +package ru.vk.itmo.tuzikovalexandr; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class PeekIterator implements Iterator { + + private final int priority; + private T currentEntry; + private final Iterator iterator; + + public PeekIterator(Iterator iterator, int priority) { + this.priority = priority; + this.iterator = iterator; + } + + @Override + public boolean hasNext() { + return currentEntry != null || iterator.hasNext(); + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + T next = peek(); + currentEntry = null; + return next; + } + + public T peek() { + if (currentEntry == null) { + if (!iterator.hasNext()) { + return null; + } + currentEntry = iterator.next(); + } + return currentEntry; + } + + public int getPriority() { + return priority; + } +} diff --git a/src/main/java/ru/vk/itmo/tuzikovalexandr/RangeIterator.java b/src/main/java/ru/vk/itmo/tuzikovalexandr/RangeIterator.java new file mode 100644 index 000000000..08c287481 --- /dev/null +++ b/src/main/java/ru/vk/itmo/tuzikovalexandr/RangeIterator.java @@ -0,0 +1,101 @@ +package ru.vk.itmo.tuzikovalexandr; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.Queue; + +public abstract class RangeIterator implements Iterator { + + private final Queue> iterators; + private final Comparator comparator; + private PeekIterator peekIterator; + + protected RangeIterator(Collection> peekIterators, Comparator comparator) { + this.comparator = comparator; + Comparator> peekComp = (o1, o2) -> comparator.compare(o1.peek(), o2.peek()); + iterators = new PriorityQueue<>(peekIterators.size(), peekComp.thenComparing(o -> -o.getPriority())); + + int priority = 0; + for (Iterator iterator : peekIterators) { + if (iterator.hasNext()) { + iterators.add(new PeekIterator<>(iterator, priority++)); + } + } + } + + private PeekIterator peek() { + while (peekIterator == null) { + peekIterator = iterators.poll(); + if (peekIterator == null) { + return null; + } + + skipOldEntries(); + + skipNullEntries(); + } + + return peekIterator; + } + + private void skipOldEntries() { + while (true) { + PeekIterator next = iterators.peek(); + if (next == null) { + break; + } + + int compare = comparator.compare(peekIterator.peek(), next.peek()); + if (compare == 0) { + PeekIterator poll = iterators.poll(); + if (poll != null) { + poll.next(); + if (poll.hasNext()) { + iterators.add(poll); + } + } + } else { + break; + } + } + } + + private void skipNullEntries() { + if (peekIterator.peek() == null) { + peekIterator = null; + return; + } + + if (skip(peekIterator.peek())) { + peekIterator.next(); + if (peekIterator.hasNext()) { + iterators.add(peekIterator); + } + peekIterator = null; + } + } + + protected abstract boolean skip(T entry); + + @Override + public boolean hasNext() { + return peek() != null; + } + + @Override + public T next() { + PeekIterator localPeekIterator = peek(); + if (localPeekIterator == null) { + throw new NoSuchElementException(); + } + T next = localPeekIterator.next(); + this.peekIterator = null; + if (localPeekIterator.hasNext()) { + iterators.add(localPeekIterator); + } + return next; + } +} diff --git a/src/main/java/ru/vk/itmo/tuzikovalexandr/SSTable.java b/src/main/java/ru/vk/itmo/tuzikovalexandr/SSTable.java index f7c4fae7d..37101f2ad 100644 --- a/src/main/java/ru/vk/itmo/tuzikovalexandr/SSTable.java +++ b/src/main/java/ru/vk/itmo/tuzikovalexandr/SSTable.java @@ -1,7 +1,6 @@ package ru.vk.itmo.tuzikovalexandr; import ru.vk.itmo.BaseEntry; -import ru.vk.itmo.Config; import ru.vk.itmo.Entry; import java.io.IOException; @@ -9,11 +8,19 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.OpenOption; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; -import java.util.Collection; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; import java.util.Set; import static java.nio.channels.FileChannel.MapMode.READ_ONLY; @@ -25,85 +32,246 @@ public class SSTable { StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING ); - private final Path filePath; - private final Arena readArena; - private final MemorySegment readSegment; - private static final String FILE_PATH = "data"; - public SSTable(Config config) throws IOException { - this.filePath = config.basePath().resolve(FILE_PATH); + private static final String FILE_PREFIX = "data_"; + private static final String OFFSET_PREFIX = "offset_"; + private static final String INDEX_FILE = "index.idx"; + private static final String INDEX_TMP = "index.tmp"; + private final List> files; - readArena = Arena.ofConfined(); + public SSTable(List> files) throws IOException { + this.files = files; + } - if (Files.notExists(filePath)) { - readSegment = null; - return; + public Iterator> range( + Iterator> firstIterator, + MemorySegment from, + MemorySegment to) { + List>> iterators = new ArrayList<>(files.size() + 1); + for (Entry entry : files) { + iterators.add(readDataFromTo(entry.key(), entry.value(), from, to)); } + iterators.add(firstIterator); + return new RangeIterator<>(iterators, Comparator.comparing(Entry::key, MemorySegmentComparator::compare)) { + @Override + protected boolean skip(Entry memorySegmentEntry) { + return memorySegmentEntry.value() == null; + } + }; + } - try (FileChannel fc = FileChannel.open(filePath, StandardOpenOption.READ)) { - readSegment = fc.map(READ_ONLY, 0, Files.size(filePath), readArena); + // storage format: offsetFile |keyOffset|valueOffset| dataFile |key|value| + public void saveMemData(Path basePath, Iterable> entries) throws IOException { + final Path indexTmp = basePath.resolve(INDEX_TMP); + final Path indexFile = basePath.resolve(INDEX_FILE); + + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state } - } + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + + String newFileName = getNewFileIndex(existedFiles); - public void saveMemData(Collection> entries) throws IOException { - if (!readArena.scope().isAlive()) { - return; + int countOffsets = 0; + long offsetData = 0; + long memorySize = 0; + for (Entry entry : entries) { + memorySize += entry.key().byteSize(); + if (entry.value() != null) { + memorySize += entry.value().byteSize(); + } + if (entry.value() == null) { + memorySize += Long.BYTES; + } + countOffsets++; } - readArena.close(); + long[] offsets = new long[countOffsets * 2]; - long offset = 0L; - long memorySize = entries.stream().mapToLong( - entry -> entry.key().byteSize() + entry.value().byteSize() - ).sum() + Long.BYTES * entries.size() * 2L; + int index = 0; - try (FileChannel fc = FileChannel.open(filePath, openOptions)) { + try (FileChannel fcData = FileChannel.open(basePath.resolve(FILE_PREFIX + newFileName), openOptions); + FileChannel fcOffset = FileChannel.open(basePath.resolve(OFFSET_PREFIX + newFileName), openOptions); + Arena writeArena = Arena.ofConfined()) { - MemorySegment writeSegment = fc.map(READ_WRITE, 0, memorySize, Arena.ofConfined()); + MemorySegment writeSegmentData = fcData.map(READ_WRITE, 0, memorySize, writeArena); + MemorySegment writeSegmentOffset = fcOffset.map( + READ_WRITE, 0, (long) offsets.length * Long.BYTES, writeArena + ); for (Entry entry : entries) { - writeSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.key().byteSize()); - offset += Long.BYTES; - writeSegment.asSlice(offset).copyFrom(entry.key()); - offset += entry.key().byteSize(); - - writeSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.value().byteSize()); - offset += Long.BYTES; - writeSegment.asSlice(offset).copyFrom(entry.value()); - offset += entry.value().byteSize(); + MemorySegment key = entry.key(); + offsets[index] = offsetData; + MemorySegment.copy(key, 0, writeSegmentData, offsetData, entry.key().byteSize()); + offsetData += key.byteSize(); + + MemorySegment value = entry.value(); + offsets[index + 1] = offsetData; + if (value != null) { + MemorySegment.copy(value, 0, writeSegmentData, offsetData, entry.value().byteSize()); + offsetData += value.byteSize(); + } + if (value == null) { + writeSegmentData.set(ValueLayout.JAVA_LONG_UNALIGNED, offsetData, -1L); + offsetData += Long.BYTES; + } + + index += 2; } + + MemorySegment.copy( + MemorySegment.ofArray(offsets), ValueLayout.JAVA_LONG, 0, + writeSegmentOffset, ValueLayout.JAVA_LONG,0, offsets.length + ); } + + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + + List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + list.add(newFileName); + Files.write( + indexFile, + list, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.delete(indexTmp); } public Entry readData(MemorySegment key) { - if (readSegment == null) { + if (files == null || key == null) { return null; } - long offset = 0L; + for (int i = files.size() - 1; i >= 0; i--) { + MemorySegment offsetSegment = files.get(i).key(); + MemorySegment dataSegment = files.get(i).value(); + + long offsetResult = Utils.binarySearch(key, offsetSegment, dataSegment); + + if (offsetResult >= 0) { + return Utils.getEntryByKeyOffset(offsetResult, offsetSegment, dataSegment); + } + } + + return null; + } + + public Iterator> readDataFromTo(MemorySegment offsetSegment, MemorySegment dataSegment, + MemorySegment from, MemorySegment to) { + long start = from == null ? 0 : Math.abs(Utils.binarySearch(from, offsetSegment, dataSegment)); + long end = to == null ? offsetSegment.byteSize() - Long.BYTES * 2 : + Math.abs(Utils.binarySearch(to, offsetSegment, dataSegment)) - Long.BYTES * 2; + + return new Iterator<>() { + long currentOffset = start; + + @Override + public boolean hasNext() { + return currentOffset <= end; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + Entry currentEntry = + Utils.getEntryByKeyOffset(currentOffset, offsetSegment, dataSegment); + + currentOffset += Long.BYTES * 2; + return currentEntry; + } + }; + } + + public void compactData(Path storagePath, Iterable> iterator) throws IOException { + saveMemData(storagePath, iterator); + + final Path indexTmp = storagePath.resolve(INDEX_TMP); + final Path indexFile = storagePath.resolve(INDEX_FILE); + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + + String lastFileIndex = existedFiles.getLast(); + + deleteOldFiles(storagePath, lastFileIndex); + + Files.writeString( + indexFile, + lastFileIndex, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); - while (offset < readSegment.byteSize()) { - long keySize = readSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); - offset += Long.BYTES; + Files.delete(indexTmp); + } + + public void deleteOldFiles(Path storagePath, String lastFileIndex) throws IOException { + String lastFileNameOffset = OFFSET_PREFIX + lastFileIndex; + String lastFileNameData = FILE_PREFIX + lastFileIndex; + + try (DirectoryStream fileStream = Files.newDirectoryStream(storagePath)) { + for (Path path : fileStream) { + String fileName = path.getFileName().toString(); + if (!fileName.equals(INDEX_FILE) && !fileName.equals(lastFileNameData) + && !fileName.equals(lastFileNameOffset) && !fileName.equals(INDEX_TMP)) { + Files.delete(path); + } + } + } + } - long valueSize = readSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, offset + keySize); + public static List> loadData(Path storagePath, Arena arena) throws IOException { + Path indexTmp = storagePath.resolve(INDEX_TMP); + Path indexFile = storagePath.resolve(INDEX_FILE); - if (keySize != key.byteSize()) { - offset += keySize + valueSize + Long.BYTES; - continue; + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state } + } + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + List> result = new ArrayList<>(existedFiles.size()); + for (String fileName : existedFiles) { + Path offsetFullPath = storagePath.resolve(OFFSET_PREFIX + fileName); + Path fileFullPath = storagePath.resolve(FILE_PREFIX + fileName); - MemorySegment keySegment = readSegment.asSlice(offset, keySize); - offset += keySize + Long.BYTES; + try (FileChannel fcOffset = FileChannel.open(offsetFullPath, StandardOpenOption.READ); + FileChannel fcData = FileChannel.open(fileFullPath, StandardOpenOption.READ)) { + MemorySegment readSegmentOffset = fcOffset.map( + READ_ONLY, 0, Files.size(offsetFullPath), arena + ); + MemorySegment readSegmentData = fcData.map( + READ_ONLY, 0, Files.size(fileFullPath), arena + ); - if (key.mismatch(keySegment) == -1) { - MemorySegment valueSegment = readSegment.asSlice(offset, valueSize); - return new BaseEntry<>(keySegment, valueSegment); + result.add(new BaseEntry<>(readSegmentOffset, readSegmentData)); } + } - offset += valueSize; + return result; + } + + private String getNewFileIndex(List existedFiles) { + if (existedFiles.isEmpty()) { + return "1"; } - return null; + int lastIndex = Integer.parseInt(existedFiles.getLast()); + return String.valueOf(lastIndex + 1); } } diff --git a/src/main/java/ru/vk/itmo/tuzikovalexandr/Utils.java b/src/main/java/ru/vk/itmo/tuzikovalexandr/Utils.java new file mode 100644 index 000000000..0c86d843d --- /dev/null +++ b/src/main/java/ru/vk/itmo/tuzikovalexandr/Utils.java @@ -0,0 +1,75 @@ +package ru.vk.itmo.tuzikovalexandr; + +import ru.vk.itmo.BaseEntry; +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +public final class Utils { + private Utils() { + } + + public static Entry getEntryByKeyOffset( + long offsetResult, MemorySegment offsetSegment, MemorySegment dataSegment) { + + long offset = offsetResult + Long.BYTES; + long valueOffset = offsetSegment.get(ValueLayout.JAVA_LONG, offset); + + MemorySegment valueSegment; + offset += Long.BYTES; + if (offset >= offsetSegment.byteSize()) { + valueSegment = dataSegment.asSlice(valueOffset); + + } else { + long valueSize = offsetSegment.get(ValueLayout.JAVA_LONG, offset) - valueOffset; + + valueSegment = dataSegment.asSlice(valueOffset, valueSize); + } + + if (valueSegment.byteSize() == Long.BYTES && valueSegment.get(ValueLayout.JAVA_LONG_UNALIGNED, 0) == -1) { + valueSegment = null; + } + + long keyOffset = offsetSegment.get(ValueLayout.JAVA_LONG, offsetResult); + long keySize = valueOffset - keyOffset; + MemorySegment keySegment = dataSegment.asSlice(keyOffset, keySize); + + return new BaseEntry<>(keySegment, valueSegment); + } + + public static long binarySearch(MemorySegment key, MemorySegment offsetSegment, + MemorySegment dataSegment) { + long left = 0; + long right = offsetSegment.byteSize() / Long.BYTES - 1; + + while (left <= right) { + + long middle = (right - left) / 2 + left; + + long offset = middle * Long.BYTES * 2; + if (offset >= offsetSegment.byteSize()) { + return -left * Long.BYTES * 2; + } + + long keyOffset = offsetSegment.get(ValueLayout.JAVA_LONG, offset); + + offset = middle * Long.BYTES * 2 + Long.BYTES; + long keySize = offsetSegment.get(ValueLayout.JAVA_LONG, offset) - keyOffset; + + MemorySegment keySegment = dataSegment.asSlice(keyOffset, keySize); + + int result = MemorySegmentComparator.compare(keySegment, key); + + if (result < 0) { + left = middle + 1; + } else if (result > 0) { + right = middle - 1; + } else { + return middle * Long.BYTES * 2; + } + } + + return -left * Long.BYTES * 2; + } +} diff --git a/src/main/java/ru/vk/itmo/viktorkorotkikh/BackgroundExecutionException.java b/src/main/java/ru/vk/itmo/viktorkorotkikh/BackgroundExecutionException.java new file mode 100644 index 000000000..45bba281c --- /dev/null +++ b/src/main/java/ru/vk/itmo/viktorkorotkikh/BackgroundExecutionException.java @@ -0,0 +1,7 @@ +package ru.vk.itmo.viktorkorotkikh; + +public class BackgroundExecutionException extends RuntimeException { + public BackgroundExecutionException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/ru/vk/itmo/viktorkorotkikh/CompactionException.java b/src/main/java/ru/vk/itmo/viktorkorotkikh/CompactionException.java new file mode 100644 index 000000000..ca9cb91c8 --- /dev/null +++ b/src/main/java/ru/vk/itmo/viktorkorotkikh/CompactionException.java @@ -0,0 +1,7 @@ +package ru.vk.itmo.viktorkorotkikh; + +public class CompactionException extends RuntimeException { + public CompactionException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/ru/vk/itmo/viktorkorotkikh/FlushingException.java b/src/main/java/ru/vk/itmo/viktorkorotkikh/FlushingException.java new file mode 100644 index 000000000..e7c430730 --- /dev/null +++ b/src/main/java/ru/vk/itmo/viktorkorotkikh/FlushingException.java @@ -0,0 +1,7 @@ +package ru.vk.itmo.viktorkorotkikh; + +public class FlushingException extends RuntimeException { + public FlushingException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/ru/vk/itmo/viktorkorotkikh/LSMDaoImpl.java b/src/main/java/ru/vk/itmo/viktorkorotkikh/LSMDaoImpl.java index 2636cbcf2..6d2485966 100644 --- a/src/main/java/ru/vk/itmo/viktorkorotkikh/LSMDaoImpl.java +++ b/src/main/java/ru/vk/itmo/viktorkorotkikh/LSMDaoImpl.java @@ -7,28 +7,43 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.NavigableMap; -import java.util.NoSuchElementException; -import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class LSMDaoImpl implements Dao> { - private NavigableMap> storage; + private volatile MemTable memTable; - private List ssTables; + private volatile MemTable flushingMemTable; + + private volatile Future flushFuture; + + private volatile Future compactionFuture; + + private volatile List ssTables; private Arena ssTablesArena; private final Path storagePath; - private static NavigableMap> createNewMemTable() { - return new ConcurrentSkipListMap<>(MemorySegmentComparator.INSTANCE); - } + private final long flushThresholdBytes; - public LSMDaoImpl(Path storagePath) { - this.storage = createNewMemTable(); + private final ExecutorService bgExecutor = Executors.newSingleThreadExecutor(); + + private final ReadWriteLock upsertLock = new ReentrantReadWriteLock(); + + private final AtomicBoolean closed = new AtomicBoolean(false); + + public LSMDaoImpl(Path storagePath, long flushThresholdBytes) { + this.memTable = new MemTable(flushThresholdBytes); + this.flushingMemTable = new MemTable(-1); try { this.ssTablesArena = Arena.ofShared(); this.ssTables = SSTable.load(ssTablesArena, storagePath); @@ -37,6 +52,7 @@ public LSMDaoImpl(Path storagePath) { throw new LSMDaoCreationException(e); } this.storagePath = storagePath; + this.flushThresholdBytes = flushThresholdBytes; } @Override @@ -45,39 +61,29 @@ public Iterator> get(MemorySegment from, MemorySegment to) } private MergeIterator.MergeIteratorWithTombstoneFilter mergeIterator(MemorySegment from, MemorySegment to) { - List ssTableIterators = - ssTables.stream().map(ssTable -> ssTable.iterator(from, to)).toList(); - return MergeIterator.create(lsmPointerIterator(from, to), ssTableIterators); - } - - private LSMPointerIterator lsmPointerIterator(MemorySegment from, MemorySegment to) { - return new MemTableIterator(iterator(from, to)); - } - - private Iterator> iterator(MemorySegment from, MemorySegment to) { - if (from == null && to == null) { - return storage.sequencedValues().iterator(); - } - - if (from == null) { - return storage.headMap(to).sequencedValues().iterator(); - } - - if (to == null) { - return storage.tailMap(from).sequencedValues().iterator(); - } - - return storage.subMap(from, to).sequencedValues().iterator(); + List ssTableIterators = SSTable.ssTableIterators(ssTables, from, to); + return MergeIterator.create( + memTable.iterator(from, to, 0), + flushingMemTable.iterator(from, to, 1), + ssTableIterators + ); } @Override public Entry get(MemorySegment key) { - Entry fromMemTable = storage.get(key); + Entry fromMemTable = memTable.get(key); if (fromMemTable != null) { return fromMemTable.value() == null ? null : fromMemTable; } - // reverse order because last sstable has the highest priority - for (int i = ssTables.size() - 1; i >= 0; i--) { + Entry fromFlushingMemTable = flushingMemTable.get(key); + if (fromFlushingMemTable != null) { + return fromFlushingMemTable.value() == null ? null : fromFlushingMemTable; + } + return getFromDisk(key); + } + + private Entry getFromDisk(MemorySegment key) { + for (int i = ssTables.size() - 1; i >= 0; i--) { // reverse order because last sstable has the highest priority SSTable ssTable = ssTables.get(i); Entry fromDisk = ssTable.get(key); if (fromDisk != null) { @@ -89,104 +95,121 @@ public Entry get(MemorySegment key) { @Override public void upsert(Entry entry) { - storage.put(entry.key(), entry); + upsertLock.readLock().lock(); + try { + if (!memTable.upsert(entry)) { // no overflow + return; + } + } finally { + upsertLock.readLock().unlock(); + } + tryToFlush(false); } @Override public void compact() throws IOException { - if (storage.isEmpty() && SSTable.isCompacted(ssTables)) { + if (SSTable.isCompacted(ssTables)) { return; } - Path compacted = SSTable.compact(() -> this.mergeIterator(null, null), storagePath); - ssTables = SSTable.replaceSSTablesWithCompacted(ssTablesArena, compacted, storagePath, ssTables); - } - @Override - public void flush() throws IOException { - if (storage.isEmpty()) return; - SSTable.save(storage.values(), ssTables.size(), storagePath); - ssTables = addNewSSTable(SSTable.loadOneByIndex(ssTablesArena, storagePath, ssTables.size())); - storage = createNewMemTable(); + compactionFuture = bgExecutor.submit(this::compactInBackground); } - private List addNewSSTable(SSTable newSSTable) { - List newSSTables = new ArrayList<>(ssTables.size() + 1); - newSSTables.addAll(ssTables); - newSSTables.add(newSSTable); - return newSSTables; + private void compactInBackground() { + try { + SSTable.compact( + () -> MergeIterator.createThroughSSTables( + SSTable.ssTableIterators(ssTables, null, null) + ), + storagePath + ); + ssTables = SSTable.load(ssTablesArena, storagePath); + } catch (IOException e) { + throw new CompactionException(e); + } } @Override - public void close() throws IOException { - if (!ssTablesArena.scope().isAlive()) { - return; - } - ssTablesArena.close(); - SSTable.save(storage.values(), ssTables.size(), storagePath); + public void flush() throws IOException { + tryToFlush(true); } - public static final class MemTableIterator extends LSMPointerIterator { - private final Iterator> iterator; - private Entry current; - - private MemTableIterator(Iterator> storageIterator) { - this.iterator = storageIterator; - if (iterator.hasNext()) { - current = iterator.next(); + private void tryToFlush(boolean tolerateToBackgroundFlushing) { + upsertLock.writeLock().lock(); + try { + if (flushingMemTable.isEmpty()) { + prepareFlush(); + } else { + if (tolerateToBackgroundFlushing) { + return; + } else { + throw new TooManyFlushesException(); + } } + } finally { + upsertLock.writeLock().unlock(); } + flushFuture = runFlushInBackground(); + } - @Override - int getPriority() { - return Integer.MAX_VALUE; - } - - @Override - protected MemorySegment getPointerKeySrc() { - return current.key(); - } - - @Override - protected long getPointerKeySrcOffset() { - return 0; - } - - @Override - boolean isPointerOnTombstone() { - return current.value() == null; - } + private void prepareFlush() { + flushingMemTable = memTable; + memTable = new MemTable(flushThresholdBytes); + } - @Override - void shift() { - if (!hasNext()) { - throw new NoSuchElementException(); + private Future runFlushInBackground() { + return bgExecutor.submit(() -> { + try { + flush(flushingMemTable, ssTables.size(), storagePath, ssTablesArena); + } catch (IOException e) { + throw new FlushingException(e); } - current = iterator.hasNext() ? iterator.next() : null; - } + }); + } - @Override - long getPointerSize() { - return Utils.getEntrySize(current); - } + private void flush( + MemTable memTable, + int fileIndex, + Path storagePath, + Arena ssTablesArena + ) throws IOException { + if (memTable.isEmpty()) return; + SSTable.save(memTable, fileIndex, storagePath); + ssTables = SSTable.load(ssTablesArena, storagePath); + flushingMemTable = new MemTable(-1); + } - @Override - protected long getPointerKeySrcSize() { - return current.key().byteSize(); + private void await(Future future) { + try { + future.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + throw new BackgroundExecutionException(e); } + } - @Override - public boolean hasNext() { - return current != null; + @Override + public void close() throws IOException { + if (closed.getAndSet(true)) { + return; // already closed } - - @Override - public Entry next() { - if (!hasNext()) { - throw new NoSuchElementException(); + bgExecutor.shutdown(); + try { + if (flushFuture != null) { + await(flushFuture); } - Entry entry = current; - current = iterator.hasNext() ? iterator.next() : null; - return entry; + if (compactionFuture != null) { + await(compactionFuture); + } + bgExecutor.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (ssTablesArena.scope().isAlive()) { + ssTablesArena.close(); } + SSTable.save(memTable, ssTables.size(), storagePath); + memTable = new MemTable(-1); } } diff --git a/src/main/java/ru/vk/itmo/viktorkorotkikh/LSMDaoOutOfMemoryException.java b/src/main/java/ru/vk/itmo/viktorkorotkikh/LSMDaoOutOfMemoryException.java new file mode 100644 index 000000000..166bfc2e8 --- /dev/null +++ b/src/main/java/ru/vk/itmo/viktorkorotkikh/LSMDaoOutOfMemoryException.java @@ -0,0 +1,7 @@ +package ru.vk.itmo.viktorkorotkikh; + +public class LSMDaoOutOfMemoryException extends RuntimeException { + public LSMDaoOutOfMemoryException() { + super("LSMDao memory tables is full. Please wait for background flushing to complete."); + } +} diff --git a/src/main/java/ru/vk/itmo/viktorkorotkikh/LSMPointerIterator.java b/src/main/java/ru/vk/itmo/viktorkorotkikh/LSMPointerIterator.java index 18eac5c06..9f815010d 100644 --- a/src/main/java/ru/vk/itmo/viktorkorotkikh/LSMPointerIterator.java +++ b/src/main/java/ru/vk/itmo/viktorkorotkikh/LSMPointerIterator.java @@ -5,7 +5,7 @@ import java.lang.foreign.MemorySegment; import java.util.Iterator; -abstract class LSMPointerIterator implements Iterator> { +public abstract class LSMPointerIterator implements Iterator> { abstract int getPriority(); diff --git a/src/main/java/ru/vk/itmo/viktorkorotkikh/MemTable.java b/src/main/java/ru/vk/itmo/viktorkorotkikh/MemTable.java new file mode 100644 index 000000000..6396a616a --- /dev/null +++ b/src/main/java/ru/vk/itmo/viktorkorotkikh/MemTable.java @@ -0,0 +1,146 @@ +package ru.vk.itmo.viktorkorotkikh; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Collection; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.NoSuchElementException; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicLong; + +public class MemTable { + private final NavigableMap> storage; + + private final long flushThresholdBytes; + + private final AtomicLong memTableByteSize = new AtomicLong(); + + public MemTable(long flushThresholdBytes) { + this.flushThresholdBytes = flushThresholdBytes; + this.storage = createNewMemTable(); + } + + private static NavigableMap> createNewMemTable() { + return new ConcurrentSkipListMap<>(MemorySegmentComparator.INSTANCE); + } + + private Iterator> storageIterator(MemorySegment from, MemorySegment to) { + if (from == null && to == null) { + return storage.sequencedValues().iterator(); + } + + if (from == null) { + return storage.headMap(to).sequencedValues().iterator(); + } + + if (to == null) { + return storage.tailMap(from).sequencedValues().iterator(); + } + + return storage.subMap(from, to).sequencedValues().iterator(); + } + + public MemTableIterator iterator(MemorySegment from, MemorySegment to, int priorityReduction) { + return new MemTableIterator(storageIterator(from, to), priorityReduction); + } + + public Entry get(MemorySegment key) { + return storage.get(key); + } + + public Collection> values() { + return storage.values(); + } + + public boolean upsert(Entry entry) { + long newEntrySize = Utils.getEntrySize(entry); + if (memTableByteSize.addAndGet(newEntrySize) - newEntrySize >= flushThresholdBytes) { + memTableByteSize.addAndGet(-newEntrySize); + throw new LSMDaoOutOfMemoryException(); + } + Entry previous = storage.put(entry.key(), entry); + if (previous != null) { + // entry already was in memTable, so we need to subtract size of previous entry + memTableByteSize.addAndGet(-Utils.getEntrySize(previous)); + } + return memTableByteSize.get() >= flushThresholdBytes; + } + + public boolean isEmpty() { + return memTableByteSize.get() == 0L; + } + + public long getByteSize() { + return memTableByteSize.get(); + } + + public static final class MemTableIterator extends LSMPointerIterator { + private final Iterator> iterator; + private Entry current; + + private final int priority; + + private MemTableIterator(Iterator> storageIterator, int priorityReduction) { + this.iterator = storageIterator; + if (iterator.hasNext()) { + current = iterator.next(); + } + this.priority = Integer.MAX_VALUE - priorityReduction; + } + + @Override + int getPriority() { + return priority; + } + + @Override + protected MemorySegment getPointerKeySrc() { + return current.key(); + } + + @Override + protected long getPointerKeySrcOffset() { + return 0; + } + + @Override + boolean isPointerOnTombstone() { + return current.value() == null; + } + + @Override + void shift() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + current = iterator.hasNext() ? iterator.next() : null; + } + + @Override + long getPointerSize() { + return Utils.getEntrySize(current); + } + + @Override + protected long getPointerKeySrcSize() { + return current.key().byteSize(); + } + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + Entry entry = current; + current = iterator.hasNext() ? iterator.next() : null; + return entry; + } + } +} diff --git a/src/main/java/ru/vk/itmo/viktorkorotkikh/MergeIterator.java b/src/main/java/ru/vk/itmo/viktorkorotkikh/MergeIterator.java index a80fee82b..68021ef65 100644 --- a/src/main/java/ru/vk/itmo/viktorkorotkikh/MergeIterator.java +++ b/src/main/java/ru/vk/itmo/viktorkorotkikh/MergeIterator.java @@ -13,19 +13,47 @@ public final class MergeIterator implements Iterator> { public static MergeIteratorWithTombstoneFilter create( LSMPointerIterator memTableIterator, + LSMPointerIterator flushingMemTableIterator, List ssTableIterators ) { - return new MergeIteratorWithTombstoneFilter(new MergeIterator(memTableIterator, ssTableIterators)); + return new MergeIteratorWithTombstoneFilter( + new MergeIterator(memTableIterator, flushingMemTableIterator, ssTableIterators) + ); + } + + public static MergeIteratorWithTombstoneFilter createThroughSSTables( + List ssTableIterators + ) { + return new MergeIteratorWithTombstoneFilter(new MergeIterator(ssTableIterators)); } - private MergeIterator(LSMPointerIterator memTableIterator, List ssTableIterators) { + private MergeIterator( + LSMPointerIterator memTableIterator, + LSMPointerIterator flushingMemTableIterator, + List ssTableIterators + ) { this.lsmPointerIterators = new PriorityQueue<>( - ssTableIterators.size() + 1, + ssTableIterators.size() + 2, LSMPointerIterator::compareByPointersWithPriority ); if (memTableIterator.hasNext()) { lsmPointerIterators.add(memTableIterator); } + if (flushingMemTableIterator.hasNext()) { + lsmPointerIterators.add(flushingMemTableIterator); + } + for (LSMPointerIterator iterator : ssTableIterators) { + if (iterator.hasNext()) { + lsmPointerIterators.add(iterator); + } + } + } + + private MergeIterator(List ssTableIterators) { + this.lsmPointerIterators = new PriorityQueue<>( + ssTableIterators.size(), + LSMPointerIterator::compareByPointersWithPriority + ); for (LSMPointerIterator iterator : ssTableIterators) { if (iterator.hasNext()) { lsmPointerIterators.add(iterator); diff --git a/src/main/java/ru/vk/itmo/viktorkorotkikh/SSTable.java b/src/main/java/ru/vk/itmo/viktorkorotkikh/SSTable.java index cf60dcb10..7cf53493a 100644 --- a/src/main/java/ru/vk/itmo/viktorkorotkikh/SSTable.java +++ b/src/main/java/ru/vk/itmo/viktorkorotkikh/SSTable.java @@ -4,23 +4,23 @@ import ru.vk.itmo.Entry; import java.io.IOException; +import java.io.UncheckedIOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.ArrayList; -import java.util.Collection; -import java.util.Comparator; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.function.Supplier; -import java.util.stream.Collectors; import java.util.stream.Stream; public final class SSTable { @@ -29,6 +29,8 @@ public final class SSTable { private static final String FILE_NAME = "sstable"; + private static final String INDEX_FILE_NAME = "index.idx"; + private static final String FILE_EXTENSION = ".db"; private static final String TMP_FILE_EXTENSION = ".tmp"; @@ -46,43 +48,32 @@ private SSTable(MemorySegment mappedSSTableFile, int index, boolean hasNoTombsto this.hasNoTombstones = hasNoTombstones; } - private static Comparator ssTablePathComparator() { - return (p1, p2) -> { - String p1String = p1.getFileName().toString(); - int p1Index = Integer.parseInt( - p1String.substring(FILE_NAME.length(), p1String.length() - FILE_EXTENSION.length()) - ); - String p2String = p2.getFileName().toString(); - int p2Index = Integer.parseInt( - p2String.substring(FILE_NAME.length(), p2String.length() - FILE_EXTENSION.length()) - ); - return Integer.compare(p1Index, p2Index); - }; - } - public static List load(Arena arena, Path basePath) throws IOException { - List ssTablePaths; - try (Stream paths = Files.walk(basePath, 1)) { - ssTablePaths = paths.filter(Files::isRegularFile) - .filter(filePath -> filePath.getFileName().toString().endsWith(FILE_EXTENSION)) - .sorted(ssTablePathComparator()) - .collect(Collectors.toList()); - } catch (NoSuchFileException e) { - return new ArrayList<>(); - } if (checkIfCompactedExists(basePath)) { - deleteOldSSTables(ssTablePaths); - return replaceSSTablesWithCompactedInternal( - arena, - basePath.resolve(FILE_NAME + 0 + FILE_EXTENSION + TMP_FILE_EXTENSION), - basePath - ); + finalizeCompaction(basePath); } - List ssTables = new ArrayList<>(ssTablePaths.size()); - for (int i = 0; i < ssTablePaths.size(); i++) { - Path ssTablePath = ssTablePaths.get(i); + + Path indexTmp = basePath.resolve(INDEX_FILE_NAME + TMP_FILE_EXTENSION); + Path indexFile = basePath.resolve(INDEX_FILE_NAME); + + if (!Files.exists(indexFile)) { + if (Files.exists(indexTmp)) { + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + if (!Files.exists(basePath)) { + Files.createDirectory(basePath); + } + Files.createFile(indexFile); + } + } + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + List ssTables = new ArrayList<>(existedFiles.size()); + for (int i = 0; i < existedFiles.size(); i++) { + Path ssTablePath = basePath.resolve(existedFiles.get(i)); ssTables.add(loadOne(arena, ssTablePath, i)); } + return ssTables; } @@ -98,11 +89,6 @@ public static SSTable loadOne(Arena arena, Path ssTablePath, int index) throws I } } - public static SSTable loadOneByIndex(Arena arena, Path basePath, int index) throws IOException { - Path ssTablePath = basePath.resolve(FILE_NAME + index + FILE_EXTENSION); - return loadOne(arena, ssTablePath, index); - } - public static boolean isCompacted(List ssTables) { return ssTables.isEmpty() || (ssTables.size() == 1 && ssTables.getFirst().hasNoTombstones); } @@ -126,28 +112,57 @@ public SSTableIterator iterator(MemorySegment from, MemorySegment to) { return new SSTableIterator(fromPosition, toPosition); } - public static void save(Collection> entries, int fileIndex, Path basePath) throws IOException { - if (entries.isEmpty()) return; - Path tmpSSTable = basePath.resolve(FILE_NAME + fileIndex + FILE_EXTENSION + TMP_FILE_EXTENSION); + public static List ssTableIterators( + List ssTables, + MemorySegment from, + MemorySegment to + ) { + return ssTables.stream().map(ssTable -> ssTable.iterator(from, to)).toList(); + } - Files.deleteIfExists(tmpSSTable); - Files.createFile(tmpSSTable); + public static void save(MemTable memTable, int fileIndex, Path basePath) throws IOException { + if (memTable.isEmpty()) return; - long entriesDataSize = 0; + final Path indexTmp = basePath.resolve(INDEX_FILE_NAME + TMP_FILE_EXTENSION); + final Path indexFile = basePath.resolve(INDEX_FILE_NAME); - for (Entry entry : entries) { - entriesDataSize += Utils.getEntrySize(entry); + try { + Files.createFile(indexFile); + } catch (FileAlreadyExistsException ignored) { + // it is ok, actually it is normal state } - save(entries::iterator, entries.size(), entriesDataSize, tmpSSTable); + Path tmpSSTable = basePath.resolve(FILE_NAME + fileIndex + FILE_EXTENSION + TMP_FILE_EXTENSION); + + Files.deleteIfExists(tmpSSTable); + Files.createFile(tmpSSTable); + + save(memTable.values()::iterator, memTable.values().size(), memTable.getByteSize(), tmpSSTable); + String newFileName = FILE_NAME + fileIndex + FILE_EXTENSION; Files.move( tmpSSTable, - basePath.resolve(FILE_NAME + fileIndex + FILE_EXTENSION), + basePath.resolve(newFileName), StandardCopyOption.ATOMIC_MOVE ); + + List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); + List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + list.add(newFileName); + Files.write( + indexTmp, + list, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.deleteIfExists(indexFile); + + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE); } - public static Path save( + private static Path save( Supplier>> iteratorSupplier, int entriesSize, long entriesDataSize, @@ -157,7 +172,6 @@ public static Path save( Files.deleteIfExists(tmpSSTable); return tmpSSTable; } - MemorySegment mappedSSTableFile; long entriesDataOffset = METADATA_SIZE + ENTRY_METADATA_SIZE * entriesSize; @@ -168,7 +182,7 @@ public static Path save( StandardOpenOption.TRUNCATE_EXISTING ) ) { - mappedSSTableFile = channel.map( + MemorySegment mappedSSTableFile = channel.map( FileChannel.MapMode.READ_WRITE, 0L, entriesDataOffset + entriesDataSize, @@ -225,63 +239,25 @@ private static void writeIndex( } } - public static Path compact( + public static void compact( Supplier data, Path basePath ) throws IOException { - Path tmpSSTable = basePath.resolve(FILE_NAME + 0 + FILE_EXTENSION + TMP_FILE_EXTENSION); + Path tmpSSTable = basePath.resolve("_compacted_" + FILE_NAME + FILE_EXTENSION + TMP_FILE_EXTENSION); Files.deleteIfExists(tmpSSTable); Files.createFile(tmpSSTable); EntriesMetadata entriesMetadata = data.get().countEntities(); - return save(data, entriesMetadata.count(), entriesMetadata.entriesDataSize(), tmpSSTable); - } - - public static List replaceSSTablesWithCompacted( - Arena arena, - Path compactedSSTable, - Path basePath, - List oldSSTables - ) throws IOException { - deleteOldSSTables(basePath, oldSSTables); - return replaceSSTablesWithCompactedInternal( - arena, - compactedSSTable, - basePath - ); + Path compacted = save(data, entriesMetadata.count(), entriesMetadata.entriesDataSize(), tmpSSTable); + Files.move(compacted, getCompactedFilePath(basePath), StandardCopyOption.ATOMIC_MOVE); + finalizeCompaction(basePath); } - private static void deleteOldSSTables(Path basePath, List oldSSTables) throws IOException { - for (SSTable oldSSTable : oldSSTables) { - Files.deleteIfExists(basePath.resolve(FILE_NAME + oldSSTable.index + FILE_EXTENSION)); - } - } - - private static void deleteOldSSTables(List oldSSTables) throws IOException { - for (Path oldSSTablePath : oldSSTables) { - Files.deleteIfExists(oldSSTablePath); - } - } - - private static List replaceSSTablesWithCompactedInternal( - Arena arena, - Path compactedSSTable, - Path basePath - ) throws IOException { - if (!Files.exists(compactedSSTable)) { - return new ArrayList<>(0); - } - Path newSSTable = Files.move( - compactedSSTable, - basePath.resolve(FILE_NAME + 0 + FILE_EXTENSION), - StandardCopyOption.ATOMIC_MOVE - ); - List newSSTables = new ArrayList<>(1); - newSSTables.add(loadOne(arena, newSSTable, 0)); - return newSSTables; + private static Path getCompactedFilePath(Path basePath) { + return basePath.resolve("_compacted_" + FILE_NAME + FILE_EXTENSION); } private static boolean checkIfCompactedExists(Path basePath) { - Path compacted = basePath.resolve(FILE_NAME + 0 + FILE_EXTENSION + TMP_FILE_EXTENSION); + Path compacted = getCompactedFilePath(basePath); if (!Files.exists(compacted)) { return false; } @@ -292,6 +268,53 @@ private static boolean checkIfCompactedExists(Path basePath) { } } + private static void finalizeCompaction(Path storagePath) throws IOException { + try (Stream stream = + Files.find( + storagePath, + 1, + (path, ignored) -> { + String fileName = path.getFileName().toString(); + return fileName.startsWith(FILE_NAME) && fileName.endsWith(FILE_EXTENSION); + })) { + stream.forEach(p -> { + try { + Files.delete(p); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + Path indexTmp = storagePath.resolve(INDEX_FILE_NAME + TMP_FILE_EXTENSION); + Path indexFile = storagePath.resolve(INDEX_FILE_NAME); + + Files.deleteIfExists(indexFile); + Files.deleteIfExists(indexTmp); + + Path compactionFile = getCompactedFilePath(storagePath); + boolean noData = Files.size(compactionFile) == 0; + + Files.write( + indexTmp, + noData ? Collections.emptyList() : Collections.singleton(FILE_NAME + "0" + FILE_EXTENSION), + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + + Files.move(indexTmp, indexFile, StandardCopyOption.ATOMIC_MOVE); + if (noData) { + Files.delete(compactionFile); + } else { + Files.move( + compactionFile, + storagePath.resolve(FILE_NAME + "0" + FILE_EXTENSION), + StandardCopyOption.ATOMIC_MOVE + ); + } + } + private static long writeMemorySegment( MemorySegment ssTableMemorySegment, MemorySegment memorySegmentToWrite, diff --git a/src/main/java/ru/vk/itmo/viktorkorotkikh/SSTableReadException.java b/src/main/java/ru/vk/itmo/viktorkorotkikh/SSTableReadException.java deleted file mode 100644 index 9fd06fa85..000000000 --- a/src/main/java/ru/vk/itmo/viktorkorotkikh/SSTableReadException.java +++ /dev/null @@ -1,7 +0,0 @@ -package ru.vk.itmo.viktorkorotkikh; - -public class SSTableReadException extends RuntimeException { - public SSTableReadException(Throwable cause) { - super(cause); - } -} diff --git a/src/main/java/ru/vk/itmo/viktorkorotkikh/TooManyFlushesException.java b/src/main/java/ru/vk/itmo/viktorkorotkikh/TooManyFlushesException.java new file mode 100644 index 000000000..709d246db --- /dev/null +++ b/src/main/java/ru/vk/itmo/viktorkorotkikh/TooManyFlushesException.java @@ -0,0 +1,4 @@ +package ru.vk.itmo.viktorkorotkikh; + +public class TooManyFlushesException extends RuntimeException { +} diff --git a/src/test/java/ru/vk/itmo/BaseTest.java b/src/test/java/ru/vk/itmo/BaseTest.java index 8ecc37b02..c4f38b8bb 100644 --- a/src/test/java/ru/vk/itmo/BaseTest.java +++ b/src/test/java/ru/vk/itmo/BaseTest.java @@ -6,6 +6,7 @@ import ru.vk.itmo.test.DaoFactory; import java.io.IOException; +import java.nio.channels.IllegalBlockingModeException; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; @@ -84,7 +85,7 @@ public void assertValueAt(Dao> dao, int index) throws IOEx assertSame(dao.get(keyAt(index)), entryAt(index)); } - public void sleep(int millis) { + public static void sleep(final int millis) { try { Thread.sleep(millis); } catch (InterruptedException e) { @@ -255,4 +256,35 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO return result[0]; } + public static void retry( + final long timeoutNanos, + final Runnable runnable) { + long elapsedNanos; + while (true) { + try { + long start = System.nanoTime(); + runnable.run(); + elapsedNanos = System.nanoTime() - start; + break; + } catch (Exception e) { + sleep(100); + } + } + + // Check timeout + if (elapsedNanos > timeoutNanos) { + throw new IllegalBlockingModeException(); + } + } + + public static void retry(Runnable runnable) { + while (true) { + try { + runnable.run(); + break; + } catch (Exception e) { + sleep(100); + } + } + } } diff --git a/src/test/java/ru/vk/itmo/BasicConcurrentTest.java b/src/test/java/ru/vk/itmo/BasicConcurrentTest.java index 0db9afaf4..82e3f435b 100644 --- a/src/test/java/ru/vk/itmo/BasicConcurrentTest.java +++ b/src/test/java/ru/vk/itmo/BasicConcurrentTest.java @@ -12,7 +12,7 @@ public class BasicConcurrentTest extends BaseTest { void test_10_000(Dao> dao) throws Exception { int count = 10_000; List> entries = entries("k", "v", count); - runInParallel(100, count, value -> dao.upsert(entries.get(value))).close(); + runInParallel(4, count, value -> dao.upsert(entries.get(value))).close(); assertSame(dao.all(), entries); } @@ -21,7 +21,7 @@ void test_10_000(Dao> dao) throws Exception { void testConcurrentRW_2_500(Dao> dao) throws Exception { int count = 2_500; List> entries = entries("k", "v", count); - runInParallel(100, count, value -> { + runInParallel(4, count, value -> { dao.upsert(entries.get(value)); assertContains(dao.all(), entries.get(value)); }).close(); @@ -36,7 +36,7 @@ void testConcurrentRead_8_000(Dao> dao) throws Exception { for (Entry entry : entries) { dao.upsert(entry); } - runInParallel(100, count, value -> assertContains(dao.all(), entries.get(value))).close(); + runInParallel(4, count, value -> assertContains(dao.all(), entries.get(value))).close(); assertSame(dao.all(), entries); } diff --git a/src/test/java/ru/vk/itmo/BasicTest.java b/src/test/java/ru/vk/itmo/BasicTest.java index 6fdcdab82..11a882ffd 100644 --- a/src/test/java/ru/vk/itmo/BasicTest.java +++ b/src/test/java/ru/vk/itmo/BasicTest.java @@ -142,12 +142,10 @@ void testHugeData(Dao> dao) throws Exception { final int entries = 100_000; for (int entry = 0; entry < entries; entry++) { - dao.upsert(entry(keyAt(entry), valueAt(entry))); + final int e = entry; - // Back off after 1K upserts to be able to flush - if (entry % 1000 == 0) { - Thread.sleep(1); - } + // Retry if autoflush is too slow + retry(() -> dao.upsert(entry(keyAt(e), valueAt(e)))); } for (int entry = 0; entry < entries; entry++) { diff --git a/src/test/java/ru/vk/itmo/CompactionTest.java b/src/test/java/ru/vk/itmo/CompactionTest.java index 3208ba8f3..ed7746a45 100644 --- a/src/test/java/ru/vk/itmo/CompactionTest.java +++ b/src/test/java/ru/vk/itmo/CompactionTest.java @@ -94,6 +94,8 @@ void overwrite(Dao> dao) throws Exception { // Check store size long smallSize = sizePersistentData(dao); + System.out.println(smallSize); + System.out.println(bigSize); // Heuristic assertTrue(smallSize * (overwrites - 1) < bigSize); diff --git a/src/test/java/ru/vk/itmo/ExpirationTest.java b/src/test/java/ru/vk/itmo/ExpirationTest.java new file mode 100644 index 000000000..d4771e60b --- /dev/null +++ b/src/test/java/ru/vk/itmo/ExpirationTest.java @@ -0,0 +1,342 @@ +package ru.vk.itmo; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.opentest4j.AssertionFailedError; +import ru.vk.itmo.solnyshkoksenia.DaoImpl; + +import java.io.File; +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import static java.nio.charset.StandardCharsets.UTF_8; +public class ExpirationTest { + private final static Config config = new Config(Path.of("var"), 100000); + + @Test + void testExpiration() throws IOException, InterruptedException { + try (DaoImpl dao = new DaoImpl(config)) { + for (int i = 0; i < 25; i++) { + dao.upsert(entry(i), 1L); + } + Thread.sleep(1L); + Assertions.assertIterableEquals(Collections.emptyList(), list(dao.all())); + } finally { + deleteDirectory(config.basePath().toFile()); + } + } + + @Test + void testLiving() throws IOException, InterruptedException { + try (DaoImpl dao = new DaoImpl(config)) { + for (int i = 0; i < 25; i++) { + dao.upsert(entry(i), 1L); + } + dao.upsert(entry(26), 1000 * 60L); + Thread.sleep(1L); + assertSame( + dao.all(), + List.of(entry(26)) + ); + } finally { + deleteDirectory(config.basePath().toFile()); + } + } + + @Test + void testRange() throws IOException { + try (DaoImpl dao = new DaoImpl(config)) { + List> values = new ArrayList<>(); + for (int i = 0; i < 25; i++) { + dao.upsert(entry(i), 1000 * 60L); + values.add(entry(i)); + } + assertSame( + dao.all(), + values + ); + } finally { + deleteDirectory(config.basePath().toFile()); + } + } + + @Test + void rangeAfterFlush() throws IOException { + try (DaoImpl dao = new DaoImpl(config)) { + List> values = new ArrayList<>(); + for (int i = 0; i < 25; i++) { + dao.upsert(entry(i), 1000 * 60L); + values.add(entry(i)); + } + dao.flush(); + assertSame( + dao.all(), + values + ); + } finally { + deleteDirectory(config.basePath().toFile()); + } + } + + @Test + void cutRangeAfterFlush() throws IOException { + try (DaoImpl dao = new DaoImpl(config)) { + List> values = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + dao.upsert(entry(i), 1000 * 60L); + } + for (int i = 10; i < 20; i++) { + dao.upsert(entry(i), 1000 * 60L); + values.add(entry(i)); + } + for (int i = 20; i < 30; i++) { + dao.upsert(entry(i), 1000 * 60L); + } + dao.flush(); + assertSame( + dao.get(entry(10).key(), entry(20).key()), + values + ); + } finally { + deleteDirectory(config.basePath().toFile()); + } + } + + @Test + void emptyRangeAfterFlush() throws IOException, InterruptedException { + try (DaoImpl dao = new DaoImpl(config)) { + for (int i = 0; i < 25; i++) { + dao.upsert(entry(i), 100L); + } + dao.flush(); + Thread.sleep(100L); + Assertions.assertIterableEquals(Collections.emptyList(), list(dao.all())); + } finally { + deleteDirectory(config.basePath().toFile()); + } + } + + @Test + void rangeAfterClose() throws IOException { + try { + DaoImpl dao = new DaoImpl(config); + List> values = new ArrayList<>(); + for (int i = 0; i < 25; i++) { + dao.upsert(entry(i), 1000 * 60L); + values.add(entry(i)); + } + dao.close(); + dao = new DaoImpl(config); + assertSame( + dao.all(), + values + ); + dao.close(); + } finally { + deleteDirectory(config.basePath().toFile()); + } + } + + @Test + void emptyRangeAfterClose() throws IOException, InterruptedException { + try { + DaoImpl dao = new DaoImpl(config); + for (int i = 0; i < 25; i++) { + dao.upsert(entry(i), 100L); + } + dao.close(); + Thread.sleep(100L); + dao = new DaoImpl(config); + Assertions.assertIterableEquals(Collections.emptyList(), list(dao.all())); + dao.close(); + } finally { + deleteDirectory(config.basePath().toFile()); + } + } + + @Test + void testExpiredGet() throws IOException, InterruptedException { + try (DaoImpl dao = new DaoImpl(config)) { + dao.upsert(entry(10), 1000L); + assertSame( + dao.get(entry(10).key()), + entry(10) + ); + Thread.sleep(1000L); + Assertions.assertNull(dao.get(entry(10).key())); + } finally { + deleteDirectory(config.basePath().toFile()); + } + } + + @Test + void expiredGetAfterFlush() throws IOException, InterruptedException { + try (DaoImpl dao = new DaoImpl(config)) { + for (int i = 0; i < 25; i++) { + dao.upsert(entry(i), 1000L); + } + dao.flush(); + assertSame( + dao.get(entry(10).key()), + entry(10) + ); + Thread.sleep(1000L); + Assertions.assertNull(dao.get(entry(10).key())); + } finally { + deleteDirectory(config.basePath().toFile()); + } + } + + @Test + void getAfterClose() throws IOException { + try { + DaoImpl dao = new DaoImpl(config); + for (int i = 0; i < 25; i++) { + dao.upsert(entry(i), 1000 * 60 * 60L); + } + dao.close(); + dao = new DaoImpl(config); + assertSame( + dao.get(entry(10).key()), + entry(10) + ); + dao.close(); + } finally { + deleteDirectory(config.basePath().toFile()); + } + } + + @Test + void expiredGetAfterClose() throws IOException, InterruptedException { + try { + DaoImpl dao = new DaoImpl(config); + for (int i = 0; i < 25; i++) { + dao.upsert(entry(i), 100L); + } + dao.close(); + Thread.sleep(100L); + dao = new DaoImpl(config); + Assertions.assertNull(dao.get(entry(10).key())); + dao.close(); + } finally { + deleteDirectory(config.basePath().toFile()); + } + } + + @Test + void rangeAfterCompaction() throws IOException { + try (DaoImpl dao = new DaoImpl(config)) { + List> values = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + dao.upsert(entry(i), 1000 * 60L); + values.add(entry(i)); + } + for (int i = 10; i < 20; i++) { + dao.upsert(entry(i), 1000 * 60L); + values.add(entry(i)); + } + for (int i = 20; i < 30; i++) { + dao.upsert(entry(i), 1000 * 60L); + values.add(entry(i)); + } + dao.flush(); + assertSame( + dao.all(), + values + ); + + for (int i = 0; i < 10; i++) { + dao.upsert(new BaseEntry<>(key(i), null), 1000 * 60L); + values.remove(0); + } + for (int i = 20; i < 30; i++) { + dao.upsert(new BaseEntry<>(key(i), null), 1000 * 60L); + values.remove(values.size() - 1); + } + dao.flush(); + dao.compact(); + + assertSame( + dao.all(), + values + ); + } finally { + deleteDirectory(config.basePath().toFile()); + } + } + + private boolean deleteDirectory(File directoryToBeDeleted) { + File[] allContents = directoryToBeDeleted.listFiles(); + if (allContents != null) { + for (File file : allContents) { + deleteDirectory(file); + } + } + return directoryToBeDeleted.delete(); + } + + public void assertSame(Entry entry, Entry expected) { + Entry entry1 = new BaseEntry<>(toString(entry.key()), toString(entry.value())); + Entry expectedEntry = new BaseEntry<>(toString(expected.key()), toString(expected.value())); + Assertions.assertEquals(expectedEntry, entry1, "wrong entry"); + } + public void assertSame(Iterator> iterator, List> expected) { + int index = 0; + for (Entry entry : expected) { + if (!iterator.hasNext()) { + throw new AssertionFailedError("No more entries in iterator: " + index + " from " + expected.size() + " entries iterated"); + } + int finalIndex = index; + Entry triple = iterator.next(); + Entry entry1 = new BaseEntry<>(toString(triple.key()), toString(triple.value())); + Entry expectedEntry = new BaseEntry<>(toString(entry.key()), toString(entry.value())); + Assertions.assertEquals(expectedEntry, entry1, () -> "wrong entry at index " + finalIndex + " from " + expected.size()); + index++; + } + if (iterator.hasNext()) { + throw new AssertionFailedError("Unexpected entry at index " + index + " from " + expected.size() + " elements: " + iterator.next()); + } + } + + public List list(Iterator iterator) { + List result = new ArrayList<>(); + iterator.forEachRemaining(result::add); + return result; + } + + public static Entry entry(int index) { + return new BaseEntry<>(key(index), value(index)); + } + + public static MemorySegment key(int index) { + return key("k", index); + } + + public static MemorySegment value(int index) { + return value("v", index); + } + + public static MemorySegment key(String prefix, int index) { + String paddedIdx = String.format("%010d", index); + return fromString(prefix + paddedIdx); + } + + public static MemorySegment value(String prefix, int index) { + String paddedIdx = String.format("%010d", index); + return fromString(prefix + paddedIdx); + } + + public static MemorySegment fromString(String data) { + return data == null ? null : MemorySegment.ofArray(data.getBytes(UTF_8)); + } + + public String toString(MemorySegment memorySegment) { + return memorySegment == null ? null : new String(memorySegment.toArray(ValueLayout.JAVA_BYTE), UTF_8); + } +} diff --git a/src/test/java/ru/vk/itmo/PersistentConcurrentTest.java b/src/test/java/ru/vk/itmo/PersistentConcurrentTest.java index b26878442..901f8e79f 100644 --- a/src/test/java/ru/vk/itmo/PersistentConcurrentTest.java +++ b/src/test/java/ru/vk/itmo/PersistentConcurrentTest.java @@ -15,13 +15,13 @@ public class PersistentConcurrentTest extends BaseTest { void testConcurrentRW_2_500_2(Dao> dao) throws Exception { int count = 2_500; List> entries = entries("k", "v", count); - runInParallel(100, count, value -> { + runInParallel(4, count, value -> { dao.upsert(entries.get(value)); }).close(); dao.close(); Dao> dao2 = DaoFactory.Factory.reopen(dao); - runInParallel(100, count, value -> { + runInParallel(4, count, value -> { assertSame(dao2.get(entries.get(value).key()), entries.get(value)); }).close(); } @@ -32,10 +32,10 @@ void testConcurrentRW_100_000_compact(Dao> dao) throws Exc List> entries = entries("k", "v", count); long timeoutNanosWarmup = TimeUnit.MILLISECONDS.toNanos(1000); - runInParallel(100, count, value -> { - tryRun(timeoutNanosWarmup, () -> dao.upsert(entries.get(value))); - tryRun(timeoutNanosWarmup, () -> dao.upsert(entry(keyAt(value), null))); - tryRun(timeoutNanosWarmup, () -> dao.upsert(entries.get(value))); + runInParallel(4, count, value -> { + retry(timeoutNanosWarmup, () -> dao.upsert(entries.get(value))); + retry(timeoutNanosWarmup, () -> dao.upsert(entry(keyAt(value), null))); + retry(timeoutNanosWarmup, () -> dao.upsert(entries.get(value))); }, () -> { for (int i = 0; i < 100; i++) { try { @@ -52,10 +52,10 @@ void testConcurrentRW_100_000_compact(Dao> dao) throws Exc // 200ms should be enough considering GC long timeoutNanos = TimeUnit.MILLISECONDS.toNanos(200); - runInParallel(100, count, value -> { - tryRun(timeoutNanos, () -> dao.upsert(entries.get(value))); - tryRun(timeoutNanos, () -> dao.upsert(entry(keyAt(value), null))); - tryRun(timeoutNanos, () -> dao.upsert(entries.get(value))); + runInParallel(4, count, value -> { + retry(timeoutNanos, () -> dao.upsert(entries.get(value))); + retry(timeoutNanos, () -> dao.upsert(entry(keyAt(value), null))); + retry(timeoutNanos, () -> dao.upsert(entries.get(value))); }, () -> { for (int i = 0; i < 100; i++) { try { @@ -72,7 +72,7 @@ void testConcurrentRW_100_000_compact(Dao> dao) throws Exc Dao> dao2 = DaoFactory.Factory.reopen(dao); runInParallel( - 100, + 4, count, value -> assertSame(dao2.get(entries.get(value).key()), entries.get(value))).close(); } @@ -89,26 +89,4 @@ private static void runAndMeasure( throw new IllegalBlockingModeException(); } } - - private static void tryRun( - long timeoutNanos, - Runnable runnable) throws InterruptedException { - long elapsedNanos; - while (true) { - try { - long start = System.nanoTime(); - runnable.run(); - elapsedNanos = System.nanoTime() - start; - break; - } catch (Exception e) { - //noinspection BusyWait - Thread.sleep(100); - } - } - - // Check timeout - if (elapsedNanos > timeoutNanos) { - throw new IllegalBlockingModeException(); - } - } } diff --git a/src/test/java/ru/vk/itmo/PersistentTest.java b/src/test/java/ru/vk/itmo/PersistentTest.java index 646dcae7a..e9b7f2497 100644 --- a/src/test/java/ru/vk/itmo/PersistentTest.java +++ b/src/test/java/ru/vk/itmo/PersistentTest.java @@ -2,7 +2,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Timeout; - import ru.vk.itmo.test.DaoFactory; import java.io.IOException; @@ -45,6 +44,7 @@ void variability(Dao> dao) throws IOException { for (final Entry entry : entries) { assertSame(dao.get(entry.key()), entry); } + dao.close(); } @DaoTest(stage = 2) @@ -59,13 +59,18 @@ void cleanup(Dao> dao) throws IOException { } @DaoTest(stage = 2) - void persistentPreventInMemoryStorage(Dao> dao) throws IOException { + void persistentPreventInMemoryStorage(Dao> dao) throws Exception { int keys = 175_000; int entityIndex = keys / 2 - 7; // Fill List> entries = entries(keys); - entries.forEach(dao::upsert); + for (int entry = 0; entry < keys; entry++) { + final int e = entry; + + // Retry if autoflush is too slow + retry(() -> dao.upsert(entries.get(e))); + } dao.close(); // Materialize to consume heap