Skip to content

Commit

Permalink
modules/zstd: Add block decoder module
Browse files Browse the repository at this point in the history 
This adds a decoder of block data. It decodes block header and
demuxes remaining input data into one of specific block decoders
depending on the type of the parsed block. Then it muxes outputs
from those decoders into single output channel.

Internal-tag: [#51873]
Signed-off-by: Pawel Czarnecki <[email protected]>
  • Loading branch information
@lpawelcz
lpawelcz committed Dec 27, 2023
1 parent 7ebb50f commit be431b3
Show file tree
Hide file tree
Showing 2 changed files with 183 additions and 0 deletions.
19 changes: 19 additions & 0 deletions xls/modules/zstd/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,3 +220,22 @@ xls_dslx_test(
name = "dec_demux_dslx_test",
library = ":dec_demux_dslx",
)

xls_dslx_library(
name = "block_dec_dslx",
srcs = [
"block_dec.x",
],
deps = [
":common_dslx",
":dec_demux_dslx",
":raw_block_dec_dslx",
":rle_block_dec_dslx",
":dec_mux_dslx",
],
)

xls_dslx_test(
name = "block_dec_dslx_test",
library = ":block_dec_dslx",
)
164 changes: 164 additions & 0 deletions xls/modules/zstd/block_dec.x
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
// Copyright 2023 The XLS Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import xls.modules.zstd.common
import xls.modules.zstd.dec_demux as demux
import xls.modules.zstd.raw_block_dec as raw
import xls.modules.zstd.rle_block_dec as rle
import xls.modules.zstd.dec_mux as mux

type BlockDataPacket = common::BlockDataPacket;
type BlockData = common::BlockData;
type BlockPacketLength = common::BlockPacketLength;

// Proc responsible for connecting internal procs used in Block data decoding.
// It handles incoming block data packets by redirecting those to demuxer which passes those to
// block decoder procs specific for given block type. Results are then gathered by mux which
// transfers decoded data further. The connections are visualised on the following diagram:
//
// Block Decoder
// ┌───────────────────────────────────────┐
// │ Raw Block Decoder │
// │ ┌───────────────────┐ │
// │ ┌─► ├┐ │
// │ Demux │ └───────────────────┘│ Mux │
// │┌─────┐│ Rle Block Decoder │ ┌─────┐│
// ││ ├┘ ┌───────────────────┐└─► ││
// ──┼► ├──► ├──► ├┼─►
// ││ ├┐ └───────────────────┘┌─► ││
// │└─────┘│ Cmp Block Decoder │ └─────┘│
// │ │ ┌───────────────────┐│ │
// │ └─► ├┘ │
// │ └───────────────────┘ │
// └───────────────────────────────────────┘

proc BlockDecoder {
input_r: chan<BlockDataPacket> in;
output_s: chan<BlockData> out;

config (input_r: chan<BlockDataPacket> in, output_s: chan<BlockData> out) {
let (demux_raw_s, demux_raw_r) = chan<BlockDataPacket>;
let (demux_rle_s, demux_rle_r) = chan<BlockDataPacket>;
let (demux_cmp_s, demux_cmp_r) = chan<BlockDataPacket>;
let (mux_raw_s, mux_raw_r) = chan<BlockDataPacket>;
let (mux_rle_s, mux_rle_r) = chan<BlockDataPacket>;
let (mux_cmp_s, mux_cmp_r) = chan<BlockDataPacket>;

spawn demux::DecoderDemux(input_r, demux_raw_s, demux_rle_s, demux_cmp_s);
spawn raw::RawBlockDecoder(demux_raw_r, mux_raw_s);
spawn rle::RleBlockDecoder(demux_rle_r, mux_rle_s);
// TODO(lpawelcz): 2023-11-28 change to compressed block decoder proc
spawn raw::RawBlockDecoder(demux_cmp_r, mux_cmp_s);
spawn mux::DecoderMux(mux_raw_r, mux_rle_r, mux_cmp_r, output_s);

(input_r, output_s)
}

init { }

next(tok: token, state: ()) { }
}

#[test_proc]
proc BlockDecoderTest {
terminator: chan<bool> out;
input_s: chan<BlockDataPacket> out;
output_r: chan<BlockData> in;

init {}

config (terminator: chan<bool> out) {
let (input_s, input_r) = chan<BlockDataPacket>;
let (output_s, output_r) = chan<BlockData>;

spawn BlockDecoder(input_r, output_s);

(terminator, input_s, output_r)
}

next(tok: token, state: ()) {
let EncodedDataBlocksPackets: BlockDataPacket[13] = [
// RAW Block 1 byte
BlockDataPacket { id: u32:0, last: true, last_block: false, data: BlockData:0xDE000008, length: BlockPacketLength:32 },
// RAW Block 2 bytes
BlockDataPacket { id: u32:1, last: true, last_block: false, data: BlockData:0xDEAD000010, length: BlockPacketLength:40 },
// RAW Block 4 bytes
BlockDataPacket { id: u32:2, last: true, last_block: false, data: BlockData:0xDEADBEEF000020, length: BlockPacketLength:56 },
// RAW Block 5 bytes (block header takes one full packet)
BlockDataPacket { id: u32:3, last: true, last_block: false, data: BlockData:0xDEADBEEFEF000028, length: BlockPacketLength:64 },
// RAW Block 24 bytes (multi-packet block header with unaligned data in the last packet)
BlockDataPacket { id: u32:4, last: false, last_block: false, data: BlockData:0x12345678900000C0, length: BlockPacketLength:64 },
BlockDataPacket { id: u32:4, last: false, last_block: false, data: BlockData:0x1234567890ABCDEF, length: BlockPacketLength:64 },
BlockDataPacket { id: u32:4, last: false, last_block: false, data: BlockData:0xFEDCBA0987654321, length: BlockPacketLength:64 },
BlockDataPacket { id: u32:4, last: true, last_block: false, data: BlockData:0xF0F0F0, length: BlockPacketLength:24 },

// RLE Block 1 byte
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0x6700000a, length: BlockPacketLength:32 },
// RLE Block 2 bytes
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0x45000012, length: BlockPacketLength:32 },
// RLE Block 4 bytes
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0x23000022, length: BlockPacketLength:32 },
// RLE Block 8 bytes (block takes one full packet)
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0x10000042, length: BlockPacketLength:32 },
// RLE Block 26 bytes (multi-packet block header with unaligned data in the last packet)
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0xDE0000d2, length: BlockPacketLength:32 },
];

let tok = for ((counter, block_packet), tok): ((u32, BlockDataPacket), token) in enumerate(EncodedDataBlocksPackets) {
let tok = send(tok, input_s, block_packet);
trace_fmt!("Sent #{} encoded block packet, {:#x}", counter + u32:1, block_packet);
(tok)
}(tok);

let DecodedDataBlocksPackets: BlockData[16] = [
// RAW Block 1 byte
BlockData:0xDE,
// RAW Block 2 bytes
BlockData:0xDEAD,
// RAW Block 4 bytes
BlockData:0xDEADBEEF,
// RAW Block 5 bytes (block header takes one full packet)
BlockData:0xDEADBEEFEF,
// RAW Block 24 bytes (multi-packet block header with unaligned data in the last packet)
BlockData:0x1234567890,
BlockData:0x1234567890ABCDEF,
BlockData:0xFEDCBA0987654321,
BlockData:0xF0F0F0,

// RLE Block 1 byte
BlockData:0x67,
// RLE Block 2 bytes
BlockData:0x4545,
// RLE Block 4 bytes
BlockData:0x23232323,
// RLE Block 8 bytes (block takes one full packet)
BlockData:0x1010101010101010,
// RLE Block 26 bytes (multi-packet block header with unaligned data in the last packet)
BlockData:0xDEDEDEDEDEDEDEDE,
BlockData:0xDEDEDEDEDEDEDEDE,
BlockData:0xDEDEDEDEDEDEDEDE,
BlockData:0xDEDE,
];

let tok = for ((counter, expected_block_packet), tok): ((u32, BlockData), token) in enumerate(DecodedDataBlocksPackets) {
let (tok, decoded_block_packet) = recv(tok, output_r);
trace_fmt!("Received #{} decoded block packet, data: 0x{:x}", counter + u32:1, decoded_block_packet);
trace_fmt!("Expected #{} decoded block packet, data: 0x{:x}", counter + u32:1, expected_block_packet);
assert_eq(decoded_block_packet, expected_block_packet);
(tok)
}(tok);

send(tok, terminator, true);
}
}

0 comments on commit be431b3

Please sign in to comment.