Skip to content

Commit 05c43a0

Browse files
coderforlifecoderforlife
committed
Initial commit. Mostly working, but some algorithms are slow and/or memory intensive.
1 parent 2c151c7 commit 05c43a0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+560508
-0
lines changed

‎.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
*.lznt1
2+
*.xpress
3+
*.xpress_huff
4+
15
# Compiled Object files
26
*.slo
37
*.lo

‎Bitstream.cpp

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#include "stdafx.h"
2+
#include "Bitstream.h"
3+
4+
5+
// Reading functions:
6+
void BSReadInit(InputBitstream* bstr, const_bytes in, size_t len)
7+
{
8+
assert(in);
9+
assert(4 < len);
10+
bstr->data.in = in;
11+
bstr->index = 4;
12+
bstr->len = len;
13+
bstr->mask = (GET_UINT16(in) << 16) | GET_UINT16(in+2);
14+
bstr->bits = 32;
15+
bstr->pntr[0] = NULL;
16+
bstr->pntr[1] = NULL;
17+
}
18+
uint32_t BSPeek(const InputBitstream* bstr, byte n) { return (n > bstr->bits) ? 0xFFFFFFFF : ((n == 0) ? 0 : (bstr->mask >> (32 - n))); }
19+
void BSSkip(InputBitstream* bstr, byte n)
20+
{
21+
bstr->mask <<= n;
22+
bstr->bits -= n;
23+
if (bstr->bits < 16 && bstr->index + 2 <= bstr->len)
24+
{
25+
bstr->mask |= GET_UINT16(bstr->data.in + bstr->index) << (16 - bstr->bits);
26+
bstr->bits |= 0x10; //bstr->bits += 16;
27+
bstr->index += 2;
28+
}
29+
}
30+
byte BSReadBit(InputBitstream* bstr) { byte x = 0xFF; if (bstr->bits) { x = (byte)(bstr->mask >> 31); BSSkip(bstr, 1); } return x; }
31+
uint32_t BSReadBits(InputBitstream* bstr, byte n) { uint32_t x = BSPeek(bstr, n); if (x != 0xFFFFFFFF) { BSSkip(bstr, n); } return x; }
32+
33+
34+
// Writing functions:
35+
void BSWriteInit(OutputBitstream* bstr, bytes out, size_t len)
36+
{
37+
assert(out);
38+
assert(4 <= len);
39+
bstr->data.out = out;
40+
bstr->index = 4;
41+
bstr->len = len;
42+
bstr->mask = 0;
43+
bstr->bits = 0;
44+
bstr->pntr[0] = (uint16_t*)out;
45+
bstr->pntr[1] = (uint16_t*)(out+2);
46+
}
47+
bool BSWriteBits(OutputBitstream* bstr, uint32_t b, byte n)
48+
{
49+
bstr->mask |= b << (32 - (bstr->bits += n));
50+
if (bstr->bits > 16)
51+
{
52+
if (bstr->pntr[1] == NULL) return false; // only 16 bits can fit into pntr[0]!
53+
SET_UINT16(bstr->pntr[0], bstr->mask >> 16);
54+
bstr->mask <<= 16;
55+
bstr->bits &= 0xF; //bstr->bits -= 16;
56+
bstr->pntr[0] = bstr->pntr[1];
57+
if (bstr->index + 2 > bstr->len)
58+
{
59+
// No more uint16s are available, however we can still write 16 more bits to pntr[0]
60+
bstr->pntr[1] = NULL;
61+
}
62+
else
63+
{
64+
bstr->pntr[1] = (uint16_t*)(bstr->data.out+bstr->index);
65+
bstr->index += 2;
66+
}
67+
}
68+
return true;
69+
}
70+
bool BSWriteByte(OutputBitstream* bstr, byte b)
71+
{
72+
if (bstr->index >= bstr->len) return false;
73+
bstr->data.out[bstr->index++] = b;
74+
return true;
75+
}
76+
void BSWriteFinish(OutputBitstream* bstr)
77+
{
78+
SET_UINT16(bstr->pntr[0], bstr->mask >> 16); // if !bits then mask is 0 anyways
79+
if (bstr->pntr[1]) *bstr->pntr[1] = 0;
80+
}

‎Bitstream.h

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#pragma once
2+
3+
////////////////////////////// Bitstreams //////////////////////////////////////////////////////////
4+
// A bitstream that allows either reading or writing, but not both at the same time.
5+
// It reads uint16s for bits and 16 bits can be reliably read at a time
6+
struct _Bitstream
7+
{
8+
union
9+
{
10+
const_bytes in; // Reading only: The input byte array
11+
bytes out; // Writing only: The output byte array
12+
} data;
13+
size_t index, len; // The current position and length of the stream
14+
uint32_t mask; // The next bits to be read/written in the bitstream
15+
byte bits; // The number of bits in mask that are valid
16+
uint16_t* pntr[2]; // Writing only: the uint16's to write the data in mask to when there are enough bits
17+
};
18+
typedef struct _Bitstream InputBitstream;
19+
typedef struct _Bitstream OutputBitstream;
20+
21+
22+
// Reading functions:
23+
void BSReadInit(InputBitstream* bstr, const_bytes in, size_t len);
24+
uint32_t BSPeek(const InputBitstream* bstr, byte n);
25+
void BSSkip(InputBitstream* bstr, byte n);
26+
byte BSReadBit(InputBitstream* bstr);
27+
uint32_t BSReadBits(InputBitstream* bstr, byte n);
28+
29+
30+
// Writing functions:
31+
void BSWriteInit(OutputBitstream* bstr, bytes out, size_t len);
32+
bool BSWriteBits(OutputBitstream* bstr, uint32_t b, byte n);
33+
bool BSWriteByte(OutputBitstream* bstr, byte b);
34+
void BSWriteFinish(OutputBitstream* bstr);

‎Dictionary.cpp

+170
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
#include "stdafx.h"
2+
#include "Dictionary.h"
3+
4+
// Implementation designed for being extremely fast at the expense of memory
5+
// usage. The base memory usage is 512 KB (or 768 KB on 64-bit systems). More
6+
// memory is always allocated but only as much as needed. Larger sized chunks
7+
// will consume more memory. For a series of 4 KB chunks, the extra consumed
8+
// memory is around 20-80 KB. For a series of 64 KB chunks, it is 200-800 KB.
9+
10+
// This implementation is ~30x faster than the 576 KB fixed-size Dictionary!
11+
12+
#ifdef __cplusplus_cli
13+
#pragma unmanaged
14+
#endif
15+
16+
#pragma optimize("t", on)
17+
18+
#define MAX_BYTE 0x100 // maximum byte value (+1 for 0)
19+
#define MIN(a, b) (((a) < (b)) ? (a) : (b)) // minimum of 2 values
20+
21+
// An entry within the dictionary, using a dynamically resized array of positions
22+
typedef struct _Entry // 8+ bytes (12+ bytes on 64-bit systems)
23+
{
24+
const_bytes* pos;
25+
uint16_t size, cap;
26+
} Entry;
27+
28+
// The dictionary
29+
struct _Dictionary // 512+ KB (768+ KB on 64-bit systems)
30+
{
31+
Entry entries[MAX_BYTE][MAX_BYTE];
32+
};
33+
34+
// Creates and returns an uninitialized dictionary struct
35+
Dictionary* Dictionary_Create()
36+
{
37+
Dictionary* d = (Dictionary*)malloc(sizeof(Dictionary));
38+
if (d) memset(d, 0, sizeof(Dictionary)); // need to set pos and cap to 0, might as well set size to 0
39+
#ifdef PRINT_ERRORS
40+
else PRINT_ERROR("Dictionary Creation Error: malloc failed\n");
41+
#endif
42+
return d;
43+
}
44+
45+
// Destroys a dictionary struct
46+
void Dictionary_Destroy(Dictionary* d)
47+
{
48+
uint_fast16_t i, j;
49+
for (i = 0; i < MAX_BYTE; ++i)
50+
for (j = 0; j < MAX_BYTE; ++j)
51+
free(d->entries[i][j].pos);
52+
free(d);
53+
}
54+
55+
// Resets a dictionary struct ready to start a new chunk
56+
// This should also be called after Dictionary_Create and before any Dictionary_Add/Dictionary_Find
57+
// Returns true on success, false on error (and sets errno)
58+
bool Dictionary_Reset(Dictionary* d)
59+
{
60+
uint_fast16_t i, j;
61+
for (i = 0; i < MAX_BYTE; ++i)
62+
for (j = 0; j < MAX_BYTE; ++j)
63+
d->entries[i][j].size = 0;
64+
return true;
65+
}
66+
67+
// Adds data to the dictionary, which will be used as a starting point during future finds
68+
// Max length is how many bytes can be read from data, regardless of the end of the chunk
69+
// Returns true on success, false on error
70+
bool Dictionary_Add(Dictionary* d, const_bytes data, const size_t max_len)
71+
{
72+
if (max_len >= 2)
73+
{
74+
const byte x = data[0], y = data[1];
75+
Entry* e = d->entries[x]+y;
76+
if (e->size >= e->cap)
77+
{
78+
const_bytes *temp = (const_bytes*)realloc(e->pos, (e->cap=(e->cap?((e->cap==0x8000)?0xFFFF:(e->cap<<1)):8))*sizeof(const_bytes));
79+
if (temp == NULL)
80+
{
81+
PRINT_ERROR("Dictionary Add Error: realloc failed\n");
82+
Dictionary_Destroy(d);
83+
return false;
84+
}
85+
e->pos = temp;
86+
}
87+
e->pos[e->size++] = data;
88+
}
89+
return true;
90+
}
91+
92+
// Finds the best symbol in the dictionary for the data at u[pos]
93+
// Returns the length of the string found, or 0 if nothing of length >= 3 was found
94+
// offset is set to the offset from the current position to the string
95+
uint_fast16_t Dictionary_Find(const Dictionary* d, const Dictionary* d2, const_bytes data, const uint_fast16_t max_len, const_bytes search, uint_fast16_t* offset)
96+
{
97+
static const Entry DummyEntry = { NULL, 0, 0 }; // using this instead of NULL reduces a lot of checks
98+
99+
if (max_len >= 3 && data-search > 0)
100+
{
101+
const byte x = data[0], y = data[1];
102+
const Entry* e = d->entries[x]+y, *e2 = d2 ? d2->entries[x]+y : &DummyEntry;
103+
if (e->size || e2->size) // a match is possible
104+
{
105+
const byte z = data[2];
106+
uint_fast16_t l = 0, o;
107+
int_fast32_t ep = e->size - 1; // need to support all uint16 values and <0
108+
109+
// Try short repeats - this does not use the Dictionary at all
110+
if (x == z && y == data[-1])
111+
{
112+
if (x == y) // x == y == z == data[-1]
113+
{
114+
// Repeating the last byte
115+
o = 1;
116+
l = 3;
117+
while (l < max_len && data[l] == x) { ++l; }
118+
--ep;
119+
if (data-search > 1 && x == data[-2])
120+
--ep;
121+
}
122+
else if (data-search > 1 && x == data[-2]) // x == z == data[-2], y == data[-1]
123+
{
124+
// Repeating the last two bytes
125+
o = 2;
126+
l = 3;
127+
while (l < max_len && data[l] == y) { ++l; if (l < max_len && data[l] == x) { ++l; } else break; }
128+
--ep;
129+
}
130+
131+
// Found the best match, stop now
132+
if (l == max_len) { *offset = o; return l; }
133+
}
134+
135+
// Do an exhaustive search (with the possible positions)
136+
if (ep < 0) { ep += (e=e2)->size-1; e2 = &DummyEntry; }
137+
do
138+
{
139+
for (; ep >= 0 && e->pos[ep] >= search; --ep)
140+
{
141+
const const_bytes ss = e->pos[ep];
142+
if (ss[2] == z)
143+
{
144+
const_bytes s = ss+3;
145+
uint_fast16_t i = 3;
146+
if (s == data) { s = ss; }
147+
while (i < max_len && data[i] == *s)
148+
{
149+
++i;
150+
if (++s == data) { s = ss; } // allow looping back, can have l > o
151+
}
152+
if (i > l) { o = (uint_fast16_t)(data-ss); if ((l = i) == max_len) { break; } }
153+
}
154+
}
155+
ep = (e=e2)->size - 1;
156+
e2 = &DummyEntry;
157+
} while (ep >= 0);
158+
159+
// Found a match, return it
160+
if (l >= 3)
161+
{
162+
*offset = o;
163+
return l;
164+
}
165+
}
166+
}
167+
168+
// No match found, return 0
169+
return 0;
170+
}

‎Dictionary.h

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#pragma once
2+
3+
/////////////////// Dictionary /////////////////////////////////////////////////
4+
// The dictionary system used for LZNT1 and XPRESS compression.
5+
//
6+
// Most of the compression time is spent in the dictionary - particularly Find and Add.
7+
//
8+
// The compressor does not care about the format of the dictionary struct, it is
9+
// completely agnostic to it and any of the function implementations.
10+
11+
12+
struct _Dictionary;
13+
typedef struct _Dictionary Dictionary;
14+
15+
16+
// Creates and returns an uninitialized dictionary struct
17+
// Returns NULL on error (and sets errno)
18+
Dictionary* Dictionary_Create();
19+
20+
// Destroys a dictionary struct
21+
void Dictionary_Destroy(Dictionary* d);
22+
23+
// Resets a dictionary struct ready to start a new chunk
24+
// This should also be called after Dictionary_Create and before any Dictionary_Add/Dictionary_Find
25+
// Returns true on success, false on error (and sets errno)
26+
bool Dictionary_Reset(Dictionary* d);
27+
28+
// Adds data to the dictionary, which will be used as a starting point during future finds
29+
// Max length is how many bytes can be read from data, regardless of the end of the chunk
30+
// Returns true on success, false on error
31+
bool Dictionary_Add(Dictionary* d, const_bytes data, const size_t max_len);
32+
33+
// Finds the best symbol in the dictionary(ies) for the data
34+
// The second dictionary may be NULL for independent chunks, or the dictionary for the previous chunk is overlap can occur
35+
// Returns the length of the string found, or 0 if nothing of length >= 3 was found
36+
// offset is set to the offset from the current position to the string
37+
uint_fast16_t Dictionary_Find(const Dictionary* d, const Dictionary* d2, const_bytes data, const uint_fast16_t max_len, const_bytes search, uint_fast16_t* offset);

‎build-c++.bat

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
@echo off
2+
3+
:: This builds using MinGW-w64 for 32 and 64 bit (http://mingw-w64.sourceforge.net/)
4+
:: Make sure both mingw-w32\bin and mingw-w64\bin are in the PATH
5+
6+
::-Werror
7+
set FLAGS=-mconsole -static-libgcc -static-libstdc++ -O3 -march=core2 -Wall -s
8+
set FILES=compression.cpp Dictionary.cpp Bitstream.cpp lznt1.cpp lzx.cpp xpress.cpp xpress_huff.cpp test.cpp
9+
set OUT=compression
10+
11+
echo Compiling 32-bit...
12+
i686-w64-mingw32-g++ %FLAGS% %FILES% -o %OUT%.exe
13+
14+
echo.
15+
16+
echo Compiling 64-bit...
17+
18+
x86_64-w64-mingw32-g++ %FLAGS% %FILES% -o %OUT%64.exe
19+
20+
pause

‎build-c.bat

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
@echo off
2+
3+
:: This builds using MinGW-w64 for 32 and 64 bit (http://mingw-w64.sourceforge.net/)
4+
:: Make sure both mingw-w32\bin and mingw-w64\bin are in the PATH
5+
6+
echo Compiling 32-bit...
7+
i686-w64-mingw32-gcc -mconsole -static-libgcc -O3 -march=core2 -o compression.exe -s ^
8+
compression.c Dictionary.c Bitstream.c lznt1.c lzx.c xpress.c xpress_huff.c test.c ^
9+
10+
echo.
11+
12+
echo Compiling 64-bit...
13+
14+
x86_64-w64-mingw32-gcc -mconsole -static-libgcc -O3 -march=core2 -o compression64.exe -s ^
15+
compression.c Dictionary.c Bitstream.c lznt1.c lzx.c xpress.c xpress_huff.c test.c
16+
17+
pause

0 commit comments

Comments
 (0)