TeaSpeak/TeaSpeakLibrary
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Some smaller changes and code cleanups

Browse Source
This commit is contained in:
WolverinDEV 2021-02-05 17:20:38 +01:00
parent d213c0ade0
commit ac48d30696
22 changed files with 1221 additions and 3516 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CMakeLists.txt
View File

@ -101,12 +101,7 @@ set(SOURCE_FILES
#Logger
src/log/LogUtils.cpp
src/log/LogSinks.cpp
src/qlz/QuickLZ.cpp
src/qlz/QuickLZ_L3.cpp
src/qlz/QuickLZ_L1.cpp
src/converters/converter.cpp
src/query/command3.cpp
@ -155,7 +150,6 @@ set(HEADER_FILES
src/misc/memtracker.h
src/misc/strobf.h
src/log/translation.h
src/log/LogUtils.h
src/PermissionManager.h

8
src/Error.cpp
View File

@ -12,7 +12,7 @@ using namespace ts;
#define define_error_description(type, description) \
{ error::type, str(type), description }
const std::vector<ErrorType> ts::avariableErrors = {
const std::vector<ErrorType> ts::availableErrors = {
{0x0000, "ok" , "ok" },
{0x0001, "undefined" , "undefined error" },
{0x0002, "not_implemented" , "not implemented" },
@ -201,19 +201,19 @@ const std::vector<ErrorType> ts::avariableErrors = {
{0xFFFF, "custom_error" , "costume" },
};
ErrorType ErrorType::Success = avariableErrors[0];
ErrorType ErrorType::Success = availableErrors[0];
ErrorType ErrorType::Costume = findError("custom_error");
ErrorType ErrorType::VSError = findError("vs_critical");
ErrorType ErrorType::DBEmpty = findError("database_empty_result");
ErrorType ts::findError(uint16_t errorId){
for(auto elm : avariableErrors)
for(auto elm : availableErrors)
if(elm.errorId == errorId) return elm;
return ErrorType{errorId, "undefined", "undefined"};
}
ErrorType ts::findError(std::string key){
for(auto elm : avariableErrors)
for(auto elm : availableErrors)
if(elm.name == key) return elm;
return ErrorType{1, key, "undefined"};
}

21
src/Error.h
View File

@ -10,10 +10,7 @@
#include <vector>
#include <map>
#define _NDEBUG
namespace ts {
struct CommandResult;
namespace permission {
enum PermissionType : uint16_t;
}
@ -365,8 +362,8 @@ namespace ts {
explicit command_result(command_result_bulk&&);
#ifndef _NDEBUG
/* if we're not using any debug we dont have to use a deconstructor. A deconstructor prevent a direct uint64_t return as described above */
#if !defined(_NDEBUG) && false
/* if we're not using any debug we dont have to use a destructor. A destructor prevent a direct uint64_t return as described above */
~command_result() {
assert(this->data == 0);
}
@ -457,15 +454,15 @@ namespace ts {
std::string message;
bool operator==(const ErrorType& ref) const {
return errorId == ref.errorId;
return this->errorId == ref.errorId;
}
bool operator!=(const ErrorType& ref) const { return !operator==(ref); }
ErrorType& operator=(const ErrorType& ref) {
errorId = ref.errorId;
name = ref.name;
message = ref.message;
this->errorId = ref.errorId;
this->name = ref.name;
this->message = ref.message;
return *this;
}
@ -477,9 +474,7 @@ namespace ts {
}
};
extern const std::vector<ErrorType> avariableErrors;
extern const std::vector<ErrorType> availableErrors;
extern ErrorType findError(uint16_t errorId);
extern ErrorType findError(std::string key);
}
#undef _NDEBUG
}

2
src/converters/converter.cpp
View File

@ -34,7 +34,7 @@ CONVERTER_PRIMITIVE_ST(float, std::stof(std::string{str}));
CONVERTER_PRIMITIVE_ST(double, std::stod(std::string{str}));
CONVERTER_PRIMITIVE_ST(long_double, std::stold(std::string{str}));
#if __x86_64__
#if UINTPTR_WIDTH >= 64
CONVERTER_PRIMITIVE_ST(long_long_unsigned_int_t, std::stoull(std::string{str}));
#endif

8
src/log/LogSinks.cpp
View File

@ -98,8 +98,9 @@ namespace logger {
const auto& mapping = level_mapping;
#endif
size_t level = msg.level.value;
if(level >= mapping.size())
if(level >= mapping.size()) {
level = mapping.size() - 1;
}
append(mapping[level]);
}
@ -122,10 +123,11 @@ namespace logger {
index = found;
append(spdlog::details::os::default_eol);
if(++index)
if(++index) {
dest.append(prefix_begin, prefix_end);
else
} else {
break;
}
}
}

5
src/log/translation.cpp
View File

@ -1,5 +0,0 @@
//
// Created by wolverindev on 12.06.18.
//
#include "translation.h"

13
src/log/translation.h
View File

@ -1,13 +0,0 @@
#pragma once
namespace tr {
enum Messages {
kick_invalid_badges,
kick_invalid_packet,
kick_invalid_command,
crash_instance,
shutdown_instance,
shutdown_server,
};
}

27
src/lookup/ip.h
View File

@ -29,11 +29,13 @@ namespace lookup {
bucket_t* bucket = &this->buckets[hash];
std::lock_guard lock{this->bucket_locks[hash]};
while(bucket->entry_count == kBukkitSize && bucket->next)
while(bucket->entry_count == kBukkitSize && bucket->next) {
bucket = bucket->next;
}
if(bucket->entry_count == kBukkitSize)
if(bucket->entry_count == kBukkitSize) {
bucket = (bucket->next = new bucket_t{});
}
auto& entry = bucket->entries[bucket->entry_count++];
addr_converter{}(entry.address, address);
@ -55,9 +57,11 @@ namespace lookup {
size_t entry_index;
do {
for(entry_index = 0; entry_index < bucket->entry_count; entry_index++)
if(auto& entry{bucket->entries[entry_index]}; cmp(entry.address, addr))
for(entry_index = 0; entry_index < bucket->entry_count; entry_index++) {
if(auto& entry{bucket->entries[entry_index]}; cmp(entry.address, addr)) {
goto entry_found;
}
}
} while((bucket = bucket->next));
/* entry hasn't been found */
@ -66,8 +70,9 @@ namespace lookup {
entry_found:
next_bucket = bucket;
while(next_bucket->next && next_bucket->next->entry_count > 0)
while(next_bucket->next && next_bucket->next->entry_count > 0) {
next_bucket = next_bucket->next;
}
/* swap the entry with the last entry and just remove the value */
next_bucket->entry_count--;
@ -76,8 +81,9 @@ namespace lookup {
}
inline void cleanup() {
for(size_t bucket_index{0}; bucket_index < kBukkitListSize; bucket_index++)
for(size_t bucket_index{0}; bucket_index < kBukkitListSize; bucket_index++) {
cleanup_bucket(bucket_index);
}
}
inline void cleanup_bucket(size_t index) {
@ -87,8 +93,9 @@ namespace lookup {
auto& bucket = this->buckets[index];
bucket_t* prev{nullptr}, curr{&bucket};
while(curr.entry_count > 0 && curr.next)
while(curr.entry_count > 0 && curr.next) {
prev = std::exchange(curr, curr.next);
}
if(curr.entry_count == 0) {
prev->next = nullptr;
@ -119,9 +126,11 @@ namespace lookup {
{
std::lock_guard lock{this->bucket_locks[hash]};
do {
for(size_t index{0}; index < bucket->entry_count; index++)
if(auto& entry{bucket->entries[index]}; cmp(entry.address, addr))
for(size_t index{0}; index < bucket->entry_count; index++) {
if(auto& entry{bucket->entries[index]}; cmp(entry.address, addr)) {
return entry.entry;
}
}
} while((bucket = bucket->next));
}

113
src/protocol/CompressionHandler.cpp
View File

@ -3,42 +3,11 @@
#define QLZ_COMPRESSION_LEVEL 1
#define QLZ_MEMORY_SAFE
#include "qlz/QuickLZ.h"
#include "buffers.h"
using namespace ts;
using namespace ts::connection;
using namespace std;
namespace ts::compression {
class thread_buffer {
public:
void* get_buffer(size_t size) {
if(size > 1024 * 1024 *5) /* we don't want to keep such big buffers in memory */
return malloc(size);
if(this->buffer_length < size) {
free(this->buffer_ptr);
size = std::max(size, (size_t) 1024);
this->buffer_ptr = malloc(size);
this->buffer_length = size;
}
return buffer_ptr;
}
void free_buffer(void* ptr) {
if(ptr == this->buffer_ptr) return;
free(ptr);
}
~thread_buffer() {
free(this->buffer_ptr);
}
private:
void* buffer_ptr{nullptr};
size_t buffer_length{0};
};
class qlz_states {
public:
qlz_states() noexcept {
@ -53,25 +22,28 @@ namespace ts::compression {
qlz_state_compress* state_compress{nullptr};
qlz_state_decompress* state_decompress{nullptr};
private:
};
thread_local thread_buffer qlz_buffer{};
thread_local qlz_states qlz_states{};
size_t qlz_decompressed_size(const void* payload, size_t payload_length) {
if(payload_length < 9) return 0; /* payload too small */
if(payload_length < 9) {
return 0; /* payload too small */
}
return qlz_size_decompressed((char*) payload) + 400;
}
bool qlz_decompress_payload(const void* payload, void* buffer, size_t* buffer_size) {
if(!qlz_states.state_decompress) return false;
if(!qlz_states.state_decompress) {
return false;
}
assert(payload != buffer);
size_t data_length = qlz_decompress((char*) payload, (char*) buffer, qlz_states.state_decompress);
if(data_length <= 0)
if(data_length <= 0) {
return false;
}
/* test for overflow */
if(data_length > *buffer_size) terminate();
@ -80,76 +52,29 @@ namespace ts::compression {
}
size_t qlz_compressed_size(const void* payload, size_t payload_length) {
(void) payload;
assert(payload_length >= 9);
return payload_length + 400; // http://www.quicklz.com/manual.html
}
bool qlz_compress_payload(const void* payload, size_t payload_length, void* buffer, size_t* buffer_length) {
if(!qlz_states.state_compress) return false;
if(!qlz_states.state_compress) {
return false;
}
assert(payload != buffer);
assert(*buffer_length >= qlz_compressed_size(payload, payload_length));
size_t compressed_length = qlz_compress(payload, (char*) buffer, payload_length, qlz_states.state_compress);
if(compressed_length > *buffer_length) terminate();
if(compressed_length > *buffer_length) {
terminate();
}
if(compressed_length <= 0)
if(compressed_length <= 0) {
return false;
}
*buffer_length = compressed_length;
return true;
}
}
bool CompressionHandler::compress(protocol::BasicPacket* packet, std::string &error) {
auto packet_payload = packet->data();
auto header_length = packet->length() - packet_payload.length();
size_t max_compressed_payload_size = compression::qlz_compressed_size(packet_payload.data_ptr(), packet_payload.length());
auto target_buffer = buffer::allocate_buffer(max_compressed_payload_size + header_length);
size_t compressed_size{max_compressed_payload_size};
if(!compression::qlz_compress_payload(packet_payload.data_ptr(), packet_payload.length(), &target_buffer[header_length], &compressed_size)) return false;
memcpy(target_buffer.data_ptr(), packet->buffer().data_ptr(), header_length);
packet->buffer(target_buffer.range(0, compressed_size + header_length));
return true;
}
bool CompressionHandler::decompress(protocol::BasicPacket* packet, std::string &error) {
auto expected_length = compression::qlz_decompressed_size(packet->data().data_ptr(), packet->data().length());
if(expected_length > this->max_packet_size){ //Max 16MB. (97% Compression!)
error = "Invalid packet size. (Calculated target length of " + to_string(expected_length) + ". Max length: " + to_string(this->max_packet_size) + ")";
return false;
} else if(expected_length == 0) {
error = "Failed to calculate decompressed packet length";
return false;
}
auto header_length = packet->header().length() + packet->mac().length();
auto buffer = buffer::allocate_buffer(expected_length + header_length);
size_t compressed_size{expected_length};
if(!compression::qlz_decompress_payload(packet->data().data_ptr(), &buffer[header_length], &compressed_size)) return false;
memcpy(buffer.data_ptr(), packet->buffer().data_ptr(), header_length);
packet->buffer(buffer.range(0, compressed_size + header_length));
return true;
}
bool CompressionHandler::progressPacketIn(protocol::BasicPacket* packet, std::string &error) {
if(packet->isCompressed()) {
if(!this->decompress(packet, error)) return false;
packet->setCompressed(false);
}
return true;
}
bool CompressionHandler::progressPacketOut(protocol::BasicPacket* packet, std::string& error) {
if(packet->has_flag(protocol::PacketFlag::Compressed) && !packet->isCompressed()) {
if(!this->compress(packet, error)) return false;
packet->setCompressed(true);
}
return true;
}
CompressionHandler::CompressionHandler() { }
CompressionHandler::~CompressionHandler() { }
}

29
src/protocol/CompressionHandler.h
View File

@ -2,28 +2,11 @@
#include "Packet.h"
namespace ts {
namespace compression {
/* Attention: These methods does not validate the data! */
size_t qlz_decompressed_size(const void* payload, size_t payload_length);
bool qlz_decompress_payload(const void* payload, void* buffer, size_t* buffer_size); //Attention: payload & buffer must be differen!
namespace ts::compression {
/* Attention: These methods does not validate the data! */
size_t qlz_decompressed_size(const void* payload, size_t payload_length);
bool qlz_decompress_payload(const void* payload, void* buffer, size_t* buffer_size); //Attention: payload & buffer must be different!
size_t qlz_compressed_size(const void* payload, size_t payload_length);
bool qlz_compress_payload(const void* payload, size_t payload_length, void* buffer, size_t* buffer_length);
}
namespace connection {
class CompressionHandler {
public:
CompressionHandler();
virtual ~CompressionHandler();
bool progressPacketOut(protocol::BasicPacket*, std::string&);
bool progressPacketIn(protocol::BasicPacket*, std::string&);
size_t max_packet_size = 16 * 1024;
private:
bool compress(protocol::BasicPacket*, std::string &error);
bool decompress(protocol::BasicPacket*, std::string &error);
};
}
size_t qlz_compressed_size(const void* payload, size_t payload_length);
bool qlz_compress_payload(const void* payload, size_t payload_length, void* buffer, size_t* buffer_length);
}

71
src/protocol/CryptHandler.cpp
View File

@ -18,6 +18,8 @@ using namespace ts::protocol;
CryptHandler::CryptHandler() {
memtrack::allocated<CryptHandler>(this);
this->cipher_code = find_cipher("rijndael");
assert(this->cipher_code >= 0);
this->reset();
}
@ -26,23 +28,26 @@ CryptHandler::~CryptHandler() {
}
void CryptHandler::reset() {
this->useDefaultChipherKeyNonce = true;
this->encryption_initialized_ = true;
this->iv_struct_length = 0;
memset(this->iv_struct, 0, sizeof(this->iv_struct));
memcpy(this->current_mac, CryptHandler::default_mac, sizeof(CryptHandler::default_mac));
for(auto& cache : this->cache_key_client)
for(auto& cache : this->cache_key_client) {
cache.generation = 0xFFEF;
for(auto& cache : this->cache_key_server)
}
for(auto& cache : this->cache_key_server) {
cache.generation = 0xFFEF;
}
}
#define SHARED_KEY_BUFFER_LENGTH (256)
bool CryptHandler::setupSharedSecret(const std::string& alpha, const std::string& beta, ecc_key *publicKey, ecc_key *ownKey, std::string &error) {
bool CryptHandler::setupSharedSecret(const std::string& alpha, const std::string& beta, ecc_key *remote_public_key, ecc_key *own_private_key, std::string &error) {
size_t buffer_length = SHARED_KEY_BUFFER_LENGTH;
uint8_t buffer[SHARED_KEY_BUFFER_LENGTH];
int err;
if((err = ecc_shared_secret(ownKey, publicKey, buffer, (unsigned long*) &buffer_length)) != CRYPT_OK){
if((err = ecc_shared_secret(own_private_key, remote_public_key, buffer, (unsigned long*) &buffer_length)) != CRYPT_OK){
error = "Could not calculate shared secret. Message: " + string(error_to_string(err));
return false;
}
@ -72,13 +77,13 @@ bool CryptHandler::setupSharedSecret(const std::string& alpha, const std::string
digest::sha1((const char*) iv_buffer, SHA_DIGEST_LENGTH, mac_buffer);
memcpy(this->current_mac, mac_buffer, 8);
this->useDefaultChipherKeyNonce = false;
this->encryption_initialized_ = false;
}
return true;
}
void _fe_neg(fe h, const fe f) {
void fe_neg_(fe h, const fe f) {
int32_t f0 = f[0];
int32_t f1 = f[1];
int32_t f2 = f[2];
@ -118,8 +123,8 @@ inline void keyMul(uint8_t(& target_buffer)[32], const uint8_t* publicKey /* com
ge_frombytes_negate_vartime(&keyA, publicKey);
if(negate) {
_fe_neg(*(fe*) &keyA.X, *(const fe*) &keyA.X); /* undo negate */
_fe_neg(*(fe*) &keyA.T, *(const fe*) &keyA.T); /* undo negate */
fe_neg_(*(fe*) &keyA.X, *(const fe*) &keyA.X); /* undo negate */
fe_neg_(*(fe*) &keyA.T, *(const fe*) &keyA.T); /* undo negate */
}
ge_scalarmult_vartime(&result, privateKey, &keyA);
@ -149,7 +154,7 @@ bool CryptHandler::setupSharedSecretNew(const std::string &alpha, const std::str
uint8_t mac_buffer[SHA_DIGEST_LENGTH];
digest::sha1((char*) this->iv_struct, 64, mac_buffer);
memcpy(this->current_mac, mac_buffer, 8);
this->useDefaultChipherKeyNonce = false;
this->encryption_initialized_ = false;
}
return true;
@ -185,7 +190,8 @@ bool CryptHandler::generate_key_nonce(
} else {
buffer[0] = 0x30;
}
buffer[1] = (char) (type & 0xF);
buffer[1] = (char) (type & 0xFU);
le2be32(generation, buffer, 2);
memcpy(&buffer[6], this->iv_struct, this->iv_struct_length);
@ -199,8 +205,8 @@ bool CryptHandler::generate_key_nonce(
}
//Xor the key
key[0] ^= (uint8_t) ((packet_id >> 8) & 0xFFU);
key[1] ^=(packet_id & 0xFFU);
key[0] ^= (uint8_t) (packet_id >> 8U);
key[1] ^= (uint8_t) packet_id;
return true;
}
@ -211,8 +217,9 @@ bool CryptHandler::verify_encryption(const pipes::buffer_view &packet, uint16_t
key_t key{};
nonce_t nonce{};
if(!generate_key_nonce(true, (protocol::PacketType) (packet[12] & 0xF), packet_id, generation, key, nonce))
if(!generate_key_nonce(true, (protocol::PacketType) ((uint8_t) packet[12] & 0xFU), packet_id, generation, key, nonce)) {
return false;
}
auto mac = packet.view(0, 8);
auto header = packet.view(8, 5);
@ -220,14 +227,13 @@ bool CryptHandler::verify_encryption(const pipes::buffer_view &packet, uint16_t
auto length = data.length();
/* static shareable void buffer */
const static unsigned long void_target_length = 2048;
static uint8_t void_target_buffer[2048];
if(void_target_length < length)
const static unsigned long void_target_length{2048};
uint8_t void_target_buffer[2048];
if(void_target_length < length) {
return false;
}
//TODO: Cache find_cipher
err = eax_decrypt_verify_memory(find_cipher("rijndael"),
err = eax_decrypt_verify_memory(this->cipher_code,
(uint8_t *) key.data(), /* the key */
(size_t) key.size(), /* key is 16 bytes */
(uint8_t *) nonce.data(), /* the nonce */
@ -245,18 +251,16 @@ bool CryptHandler::verify_encryption(const pipes::buffer_view &packet, uint16_t
return err == CRYPT_OK && success;
}
#define tmp_buffer_size (2048)
bool CryptHandler::decrypt(const void *header, size_t header_length, void *payload, size_t payload_length, const void *mac, const key_t &key, const nonce_t &nonce, std::string &error) {
if(tmp_buffer_size < payload_length) {
error = "buffer too large";
bool CryptHandler::decrypt(const void *header, size_t header_length, void *payload, size_t payload_length, const void *mac, const key_t &key, const nonce_t &nonce, std::string &error) const {
const static unsigned long kTempBufferLength{2048};
uint8_t tmp_buffer[kTempBufferLength];
if(kTempBufferLength < payload_length) {
error = "packet too large";
return false;
}
uint8_t tmp_buffer[tmp_buffer_size];
int success;
//TODO: Cache cipher
auto err = eax_decrypt_verify_memory(find_cipher("rijndael"),
auto err = eax_decrypt_verify_memory(this->cipher_code,
(const uint8_t *) key.data(), /* the key */
(unsigned long) key.size(), /* key is 16 bytes */
(const uint8_t *) nonce.data(), /* the nonce */
@ -289,17 +293,11 @@ bool CryptHandler::encrypt(
void *payload, size_t payload_length,
void *mac,
const key_t &key, const nonce_t &nonce, std::string &error) {
if(tmp_buffer_size < payload_length) {
error = "buffer too large";
return false;
}
uint8_t tmp_buffer[tmp_buffer_size];
size_t tag_length{8};
uint8_t tag_buffer[16];
static_assert(sizeof(unsigned long) <= sizeof(tag_length));
auto err = eax_encrypt_authenticate_memory(find_cipher("rijndael"),
auto err = eax_encrypt_authenticate_memory(this->cipher_code,
(uint8_t *) key.data(), /* the key */
(unsigned long) key.size(), /* key is 16 bytes */
(uint8_t *) nonce.data(), /* the nonce */
@ -308,7 +306,7 @@ bool CryptHandler::encrypt(
(unsigned long) header_length, /* header length */
(uint8_t *) payload, /* The plain text */
(unsigned long) payload_length, /* Plain text length */
(uint8_t *) tmp_buffer, /* The result buffer */
(uint8_t *) payload, /* The result buffer */
(uint8_t *) tag_buffer,
(unsigned long *) &tag_length
);
@ -320,6 +318,5 @@ bool CryptHandler::encrypt(
}
memcpy(mac, tag_buffer, 8);
memcpy(payload, tmp_buffer, payload_length);
return true;
}

57
src/protocol/CryptHandler.h
View File

@ -8,20 +8,6 @@
namespace ts::connection {
class CryptHandler {
enum Methode {
TEAMSPEAK_3_1,
TEAMSPEAK_3
};
struct KeyCache {
uint16_t generation = 0xFFEF;
union {
struct {
uint8_t key[16];
uint8_t nonce[16];
};
uint8_t key_nonce[32];
};
};
public:
typedef std::array<uint8_t, 16> key_t;
typedef std::array<uint8_t, 16> nonce_t;
@ -31,58 +17,63 @@ namespace ts::connection {
void reset();
//TeamSpeak old
bool setupSharedSecret(const std::string& alpha, const std::string& beta, ecc_key* publicKey, ecc_key* ownKey, std::string &error);
bool setupSharedSecret(const std::string& alpha, const std::string& beta, const std::string& sharedKey, std::string &error);
bool setupSharedSecret(const std::string& /* alpha */, const std::string& /* beta */, ecc_key* /* remote_public_key */, ecc_key* /* own_private_key */, std::string &/* error */);
bool setupSharedSecret(const std::string& /* alpha */, const std::string& /* beta */, const std::string& /* shared_key */, std::string &/* error */);
//TeamSpeak new
bool setupSharedSecretNew(const std::string& alpha, const std::string& beta, const char privateKey[32], const char publicKey[32]);
/* mac must be 8 bytes long! */
bool encrypt(
const void* /* header */, size_t /* header length */,
void* /* payload */, size_t /* payload length */,
void* /* mac */,
void* /* mac */, /* mac must be 8 bytes long! */
const key_t& /* key */, const nonce_t& /* nonce */,
std::string& /* error */);
/* mac must be 8 bytes long! */
bool decrypt(
const void* /* header */, size_t /* header length */,
void* /* payload */, size_t /* payload length */,
const void* /* mac */,
const void* /* mac */, /* mac must be 8 bytes long! */
const key_t& /* key */, const nonce_t& /* nonce */,
std::string& /* error */);
std::string& /* error */) const;
bool generate_key_nonce(bool /* to server */, uint8_t /* packet type */, uint16_t /* packet id */, uint16_t /* generation */, key_t& /* key */, nonce_t& /* nonce */);
bool verify_encryption(const pipes::buffer_view& data, uint16_t packet_id, uint16_t generation);
bool verify_encryption(const pipes::buffer_view& /* data */, uint16_t /* packet id */, uint16_t /* generation */);
inline void write_default_mac(void* buffer) {
memcpy(buffer, this->current_mac, 8);
}
[[nodiscard]] inline bool encryption_initialized() const { return !this->useDefaultChipherKeyNonce; }
[[nodiscard]] inline bool encryption_initialized() const { return !this->encryption_initialized_; }
static constexpr key_t kDefaultKey{'c', ':', '\\', 'w', 'i', 'n', 'd', 'o', 'w', 's', '\\', 's', 'y', 's', 't', 'e'}; //c:\windows\syste
static constexpr nonce_t kDefaultNonce{'m', '\\', 'f', 'i', 'r', 'e', 'w', 'a', 'l', 'l', '3', '2', '.', 'c', 'p', 'l'}; //m\firewall32.cpl
private:
static constexpr char default_mac[8] = {'T', 'S', '3', 'I', 'N', 'I', 'T', '1'}; //TS3INIT1
struct KeyCache {
uint16_t generation = 0xFFEF;
union {
struct {
uint8_t key[16];
uint8_t nonce[16];
};
uint8_t key_nonce[32];
};
};
bool generate_key_nonce(protocol::BasicPacket* packet, bool use_default, uint8_t(&)[16] /* key */, uint8_t(&)[16] /* nonce */);
//The default key and nonce
bool useDefaultChipherKeyNonce = true;
bool encryption_initialized_{false};
int cipher_code{-1};
/* for the old protocol SHA1 length for the new 64 bytes */
uint8_t iv_struct[64];
uint8_t iv_struct_length = 0;
uint8_t iv_struct_length{0};
uint8_t current_mac[8];
uint8_t current_mac[8]{};
std::mutex cache_key_lock;
std::array<KeyCache, protocol::PACKET_MAX> cache_key_client;
std::array<KeyCache, protocol::PACKET_MAX> cache_key_server;
std::mutex cache_key_lock{};
std::array<KeyCache, protocol::PACKET_MAX> cache_key_client{};
std::array<KeyCache, protocol::PACKET_MAX> cache_key_server{};
static_assert(sizeof(current_mac) == sizeof(default_mac), "invalid mac");
static_assert(sizeof(iv_struct) == 64, "invalid iv struct");

229
src/protocol/Packet.cpp
View File

@ -3,236 +3,14 @@
//
#include <cstring>
#include <iostream>
#include <bitset>
#include <src/misc/spin_mutex.h>
#include "Packet.h"
#include "buffers.h"
#include "misc/endianness.h"
#include "./Packet.h"
#include "../misc/endianness.h"
#include "../misc/spin_mutex.h"
using namespace std;
namespace ts {
namespace protocol {
PacketTypeInfo::PacketTypeInfo(const std::string& name, PacketType type, bool ack, int max_length) noexcept {
this->data = new PacketTypeProperties{name, type, max_length, ack};
this->owns_data = true;
if(type < 0x0F)
types.insert({type, *this});
}
PacketTypeInfo::~PacketTypeInfo() {
if(this->owns_data)
delete this->data;
}
PacketTypeInfo::PacketTypeInfo(const PacketTypeInfo &red) : data(red.data) { }
std::map<int, PacketTypeInfo> PacketTypeInfo::types;
PacketTypeInfo PacketTypeInfo::fromid(int id) {
for(const auto& elm : types)
if(elm.first == id) return elm.second;
return PacketTypeInfo::Undefined;
}
PacketTypeInfo PacketTypeInfo::Voice = {"Voice", PacketType::VOICE, false, 1024};
PacketTypeInfo PacketTypeInfo::VoiceWhisper = {"VoiceWhisper", PacketType::VOICE_WHISPER, false, 1024};
PacketTypeInfo PacketTypeInfo::Command = {"Command", PacketType::COMMAND, true, 487};
PacketTypeInfo PacketTypeInfo::CommandLow = {"CommandLow", PacketType::COMMAND_LOW, true, 487};
PacketTypeInfo PacketTypeInfo::Ping = {"Ping", PacketType::PING, false, 1024};
PacketTypeInfo PacketTypeInfo::Pong = {"Pong", PacketType::PONG, false, 1024};
PacketTypeInfo PacketTypeInfo::Ack = {"Ack", PacketType::ACK, false, 1024};
PacketTypeInfo PacketTypeInfo::AckLow = {"AckLow", PacketType::ACK_LOW, false, 1024};
PacketTypeInfo PacketTypeInfo::Init1 = {"Init1", PacketType::INIT1, false, 1024};
PacketTypeInfo PacketTypeInfo::Undefined = {"Undefined", PacketType::UNDEFINED, false, 1024};
namespace PacketFlag {
std::string to_string(PacketFlag flag){
switch(flag){
case Fragmented:
return "Fragmented";
case NewProtocol:
return "NewProtocol";
case Compressed:
return "Compressed";
case Unencrypted:
return "Unencrypted";
default:
return "None";
}
}
}
BasicPacket::BasicPacket(size_t header_length, size_t data_length) {
this->_header_length = (uint8_t) header_length;
this->_buffer = pipes::buffer(MAC_SIZE + this->_header_length + data_length);
memset(this->_buffer.data_ptr(), 0, this->_buffer.length());
}
BasicPacket::~BasicPacket() {}
void BasicPacket::append_data(const std::vector<pipes::buffer> &data) {
size_t length = 0;
for(const auto& buffer : data)
length += buffer.length();
/* we've to allocate a new buffer because out buffer is fixed in size */
size_t index = this->_buffer.length();
auto new_buffer = buffer::allocate_buffer(length + index);
new_buffer.write(this->_buffer, index);
for(const auto& buffer : data) {
new_buffer.write(buffer, buffer.length(), index);
index += buffer.length();
}
this->_buffer = new_buffer;
}
std::string BasicPacket::flags() const {
std::string result;
if(this->has_flag(PacketFlag::Unencrypted)) result += string(result.empty() ? "" : " | ") + "Unencrypted";
if(this->has_flag(PacketFlag::Compressed)) result += string(result.empty() ? "" : " | ") + "Compressed";
if(this->has_flag(PacketFlag::Fragmented)) result += string(result.empty() ? "" : " | ") + "Fragmented";
if(this->has_flag(PacketFlag::NewProtocol)) result += string(result.empty() ? "" : " | ") + "NewProtocol";
if(result.empty()) result = "none";
return result;
}
void BasicPacket::applyPacketId(PacketIdManager& manager) {
this->applyPacketId(manager.nextPacketId(this->type()), manager.generationId(this->type()));
}
void BasicPacket::applyPacketId(uint16_t packetId, uint16_t generationId) {
if(this->memory_state.id_branded)
throw std::logic_error("Packet already got a packet id!");
this->memory_state.id_branded = true;
this->setPacketId(packetId, generationId);
}
/**
* @param buffer -> [mac][Header [uint16 BE packetId | [uint8](4bit flags | 4bit type)]][Data]
* @return
*/
std::unique_ptr<ServerPacket> ServerPacket::from_buffer(const pipes::buffer_view &buffer) {
auto result = make_unique<ServerPacket>();
result->_buffer = buffer.own_buffer();
result->_header_length = SERVER_HEADER_SIZE;
return result;
}
ServerPacket::ServerPacket(uint8_t flagMask, const pipes::buffer_view& data) : BasicPacket(SERVER_HEADER_SIZE, data.length()) {
this->header()[2] = flagMask;
memcpy(this->data().data_ptr(), data.data_ptr(), data.length());
}
ServerPacket::ServerPacket(const PacketTypeInfo& type, const pipes::buffer_view& data) : BasicPacket(SERVER_HEADER_SIZE, data.length()) {
this->header()[2] |= (uint8_t) type.type();
memcpy(this->data().data_ptr(), data.data_ptr(), data.length());
}
ServerPacket::ServerPacket(ts::protocol::PacketTypeInfo type, size_t data_length) : BasicPacket(SERVER_HEADER_SIZE, data_length) {
this->header()[2] |= type.type();
}
ServerPacket::~ServerPacket() {}
uint16_t ServerPacket::packetId() const {
return be2le16(&this->header()[0]);
}
void ServerPacket::setPacketId(uint16_t pkId, uint16_t gen) {
le2be16(pkId, &this->header()[0]);
this->genId = gen;
}
uint16_t ServerPacket::generationId() const {
return this->genId;
}
PacketTypeInfo ServerPacket::type() const {
return PacketTypeInfo::fromid(this->header()[2] & 0xF);
}
std::unique_ptr<ClientPacket> ClientPacket::from_buffer(const pipes::buffer_view &buffer) {
auto result = make_unique<ClientPacket>();
result->_buffer = buffer.own_buffer();
result->_header_length = CLIENT_HEADER_SIZE;
return result;
}
ClientPacket::ClientPacket(const PacketTypeInfo &type, const pipes::buffer_view& data) : BasicPacket(CLIENT_HEADER_SIZE, data.length()) {
this->header()[4] = type.type() & 0xF;
memcpy(this->data().data_ptr(), data.data_ptr(), data.length());
}
ClientPacket::ClientPacket(const PacketTypeInfo &type, uint8_t flag_mask, const pipes::buffer_view& data) : ClientPacket(type, data) {
this->header()[4] |= flag_mask;
}
ClientPacket::~ClientPacket() {}
uint16_t ClientPacket::packetId() const {
return be2le16(&this->header()[0]);
}
uint16_t ClientPacket::generationId() const {
return this->genId;
}
PacketTypeInfo ClientPacket::type() const {
return PacketTypeInfo::fromid(this->header()[4] & 0xF);
}
void ClientPacket::type(const ts::protocol::PacketTypeInfo &type) {
auto& field = this->header().data_ptr<uint8_t>()[4];
field &= (uint8_t) ~0xF;
field |= type.type();
}
void ClientPacket::setPacketId(uint16_t pkId, uint16_t gen) {
this->header()[0] = (uint8_t) ((pkId >> 8) & 0xFF);
this->header()[1] = (uint8_t) ((pkId >> 0) & 0xFF);
this->genId = gen;
}
uint16_t ClientPacket::clientId() const {
return be2le16(&this->header()[2]);
}
void ClientPacket::clientId(uint16_t clId) {
this->header()[2] = clId >> 8;
this->header()[3] = clId & 0xFF;
}
/* New packet parser API */
bool PacketParser::is_encrypted() const {
if(this->decrypted) return false;
return (this->flags() & PacketFlag::Unencrypted) == 0;
}
bool PacketParser::is_compressed() const {
if(this->uncompressed) return false;
return (this->flags() & PacketFlag::Compressed) > 0;
}
bool PacketParser::is_fragmented() const {
if(this->defragmented) return false;
return (this->flags() & PacketFlag::Fragmented) > 0;
}
uint16_t ClientPacketParser::packet_id() const { return be2le16(this->_buffer.data_ptr<uint8_t>(), ClientPacketParser::kHeaderOffset + 0); }
uint16_t ClientPacketParser::client_id() const { return be2le16(this->_buffer.data_ptr<uint8_t>(), ClientPacketParser::kHeaderOffset + 2); }
uint8_t ClientPacketParser::type() const { return (uint8_t) this->_buffer[ClientPacketParser::kHeaderOffset + 4] & 0xFU; }
@ -361,7 +139,6 @@ namespace ts {
}
#else
void protocol::OutgoingServerPacket::object_freed() {
//TODO: Bukkit list?
deconstruct_osp(this);
::free(this);
}

668
src/protocol/Packet.h
View File

@ -8,459 +8,249 @@
#include <pipes/buffer.h>
#include "../query/Command.h"
namespace ts {
namespace protocol {
enum PacketType : uint8_t {
VOICE = 0x00,
VOICE_WHISPER = 0x01,
COMMAND = 0x02,
COMMAND_LOW = 0x03,
PING = 0x04,
PONG = 0x05,
ACK = 0x06,
ACK_LOW = 0x07,
INIT1 = 0x08,
namespace ts::protocol {
enum PacketType : uint8_t {
VOICE = 0x00,
VOICE_WHISPER = 0x01,
COMMAND = 0x02,
COMMAND_LOW = 0x03,
PING = 0x04,
PONG = 0x05,
ACK = 0x06,
ACK_LOW = 0x07,
INIT1 = 0x08,
PACKET_MAX = INIT1,
UNDEFINED = 0xFF
};
PACKET_MAX = INIT1,
UNDEFINED = 0xFF
};
struct PacketTypeProperties {
std::string name;
PacketType type;
int max_length;
bool requireAcknowledge;
};
class PacketIdManager {
public:
PacketIdManager() = default;
~PacketIdManager() = default;
PacketIdManager(const PacketIdManager& ref) = delete;
PacketIdManager(PacketIdManager&& ref) = delete;
class PacketTypeInfo {
public:
static PacketTypeInfo Voice;
static PacketTypeInfo VoiceWhisper;
static PacketTypeInfo Command;
static PacketTypeInfo CommandLow;
static PacketTypeInfo Ping;
static PacketTypeInfo Pong;
static PacketTypeInfo Ack;
static PacketTypeInfo AckLow;
static PacketTypeInfo Init1;
static PacketTypeInfo Undefined;
[[nodiscard]] uint16_t nextPacketId(const PacketType &type) {
return (uint16_t) (this->packet_counter[(uint8_t) type & 0xFU]++);
}
static PacketTypeInfo fromid(int id);
[[nodiscard]] uint16_t currentPacketId(const PacketType &type) {
return (uint16_t) (this->packet_counter[(uint8_t) type & 0xFU]);
}
std::string name() const { return data->name; }
PacketType type() const { return data->type; }
[[nodiscard]] uint16_t generationId(const PacketType &type) {
return (uint16_t) (this->packet_counter[(uint8_t) type & 0xFU] >> 16U);
}
bool requireAcknowledge(){ return data->requireAcknowledge; }
[[nodiscard]] uint32_t generate_full_id(const PacketType& type) {
return this->packet_counter[type]++;
}
bool operator==(const PacketTypeInfo& other) const {
return other.data->type == this->data->type;
}
void reset() {
memset(&this->packet_counter[0], 0, sizeof(uint32_t) * 16);
}
bool operator!=(const PacketTypeInfo& other){
return other.data->type != this->data->type;
}
private:
std::array<uint32_t, 16> packet_counter{};
};
int max_length() const { return data->max_length; }
inline bool fragmentable() { return *this == PacketTypeInfo::Command || *this == PacketTypeInfo::CommandLow; }
inline bool compressable() { return *this == PacketTypeInfo::Command || *this == PacketTypeInfo::CommandLow; }
enum struct PacketFlag {
None = 0x00,
Fragmented = 0x10, //If packet type voice then its toggle the CELT Mono
NewProtocol = 0x20,
Compressed = 0x40, //If packet type voice than its the header
Unencrypted = 0x80
};
typedef uint8_t PacketFlags;
PacketTypeInfo(const PacketTypeInfo&);
PacketTypeInfo(PacketTypeInfo&& remote) : data(remote.data) {}
constexpr const char* packet_flag_to_string(const PacketFlag& flag) {
switch(flag){
case PacketFlag::Fragmented:
return "Fragmented";
~PacketTypeInfo();
private:
static std::map<int, PacketTypeInfo> types;
PacketTypeInfo(const std::string&, PacketType, bool, int) noexcept;
PacketTypeProperties* data;
bool owns_data = false;
};
case PacketFlag::NewProtocol:
return "NewProtocol";
class PacketIdManager {
public:
PacketIdManager() {
this->reset();
}
case PacketFlag::Compressed:
return "Compressed";
~PacketIdManager() = default;
PacketIdManager(const PacketIdManager& ref) = delete;
PacketIdManager(PacketIdManager&& ref) = delete;
case PacketFlag::Unencrypted:
return "Unencrypted";
[[nodiscard]] uint16_t nextPacketId(const PacketTypeInfo &type){
return static_cast<uint16_t>(this->packetCounter[type.type()]++ & 0xFFFF);
}
[[nodiscard]] uint16_t currentPacketId(const PacketTypeInfo &type){
return static_cast<uint16_t>(this->packetCounter[type.type()] & 0xFFFF);
}
[[nodiscard]] uint16_t generationId(const PacketTypeInfo &type){
return static_cast<uint16_t>((this->packetCounter[type.type()] >> 16) & 0xFFFF);
}
[[nodiscard]] uint32_t generate_full_id(const PacketType& type) {
return this->packetCounter[type]++;
}
void reset() {
memset(&this->packetCounter[0], 0, sizeof(uint32_t) * 16);
}
private:
uint32_t packetCounter[16]{};
};
namespace PacketFlag {
enum PacketFlag : uint8_t {
None = 0x00,
Fragmented = 0x10, //If packet type voice then its toggle the CELT Mono
NewProtocol = 0x20,
Compressed = 0x40, //If packet type voice than its the header
Unencrypted = 0x80
};
typedef uint8_t PacketFlags;
std::string to_string(PacketFlag flag);
case PacketFlag::None:
default:
return "None";
}
#define MAC_SIZE 8
#define SERVER_HEADER_SIZE 3
#define CLIENT_HEADER_SIZE 5
class BasicPacket {
public:
explicit BasicPacket(size_t header_length, size_t data_length);
virtual ~BasicPacket();
BasicPacket(const BasicPacket&) = delete;
BasicPacket(BasicPacket&&) = delete;
virtual uint16_t packetId() const = 0;
virtual uint16_t generationId() const = 0;
virtual PacketTypeInfo type() const = 0;
/* packet flag info */
inline bool has_flag(PacketFlag::PacketFlag flag) const { return this->_flags_type_byte() & flag; }
inline uint8_t flag_mask() const { return this->_flags_type_byte(); };
[[nodiscard]] std::string flags() const;
/* manipulate flags */
inline void set_flags(PacketFlag::PacketFlags flags) {
uint8_t& byte = this->_flags_type_byte();
byte &= 0xF; /* clear all flags */
byte |= (flags & 0xF0);
}
inline void enable_flag(PacketFlag::PacketFlag flag){ this->toggle_flag(flag, true); }
inline void toggle_flag(PacketFlag::PacketFlag flag, bool state) {
if(state)
this->_flags_type_byte() |= flag;
else
this->_flags_type_byte() &= (uint8_t) ~flag;
}
virtual void applyPacketId(PacketIdManager &);
virtual void applyPacketId(uint16_t, uint16_t);
void setListener(std::unique_ptr<threads::Future<bool>> listener){
if(!this->type().requireAcknowledge())
throw std::logic_error("Packet type does not support a acknowledge listener!");
this->listener = std::move(listener);
}
inline std::unique_ptr<threads::Future<bool>>& getListener() { return this->listener; }
inline size_t length() const { return this->_buffer.length(); }
inline const pipes::buffer_view mac() const { return this->_buffer.view(0, MAC_SIZE); }
inline pipes::buffer mac() { return this->_buffer.range(0, MAC_SIZE); }
inline size_t mac_length() const { return MAC_SIZE; }
inline const pipes::buffer_view header() const { return this->_buffer.view(MAC_SIZE, this->_header_length); }
inline pipes::buffer header() { return this->_buffer.range(MAC_SIZE, this->_header_length); }
inline size_t header_length() const { return this->_header_length; }
inline size_t data_length() const { return this->_buffer.length() - (MAC_SIZE + this->_header_length); }
inline const pipes::buffer_view data() const { return this->_buffer.view(MAC_SIZE + this->_header_length); }
inline pipes::buffer data() { return this->_buffer.range(MAC_SIZE + this->_header_length); }
void append_data(const std::vector<pipes::buffer> &data);
inline void data(const pipes::buffer_view &data){
this->_buffer.resize(MAC_SIZE + this->_header_length + data.length());
memcpy((char*) this->_buffer.data_ptr() + MAC_SIZE + this->_header_length, data.data_ptr(), data.length());
}
inline void mac(const pipes::buffer_view &_new){
assert(_new.length() >= MAC_SIZE);
memcpy(this->_buffer.data_ptr(), _new.data_ptr(), MAC_SIZE);
}
[[nodiscard]] inline bool isEncrypted() const { return this->memory_state.encrypted; }
inline void setEncrypted(bool flag){ this->memory_state.encrypted = flag; }
[[nodiscard]] inline bool isCompressed() const { return this->memory_state.compressed; }
inline void setCompressed(bool flag){ this->memory_state.compressed = flag; }
[[nodiscard]] inline bool isFragmentEntry() const { return this->memory_state.fragment_entry; }
inline void setFragmentedEntry(bool flag){ this->memory_state.fragment_entry = flag; }
Command asCommand();
//Has the size of a byte
union {
#ifdef WIN32
__pragma(pack(push, 1))
#endif
struct {
bool encrypted: 1;
bool compressed: 1;
bool fragment_entry: 1;
bool id_branded: 1;
}
#ifdef WIN32
__pragma(pack(pop));
#else
__attribute__((packed));
#endif
uint8_t flags = 0;
} memory_state;
pipes::buffer buffer() { return this->_buffer; }
void buffer(pipes::buffer buffer) {
assert(buffer.length() >= this->_header_length + MAC_SIZE);
this->_buffer = std::move(buffer);
}
protected:
BasicPacket() = default;
virtual const uint8_t& _flags_type_byte() const = 0;
virtual uint8_t& _flags_type_byte() = 0;
virtual void setPacketId(uint16_t, uint16_t) = 0;
uint8_t _header_length;
pipes::buffer _buffer;
uint16_t genId = 0;
std::unique_ptr<threads::Future<bool>> listener;
};
/**
* Packet from the client
*/
class ClientPacket : public BasicPacket {
friend std::unique_ptr<ClientPacket> std::make_unique<ClientPacket>();
public:
static constexpr size_t META_MAC_SIZE = 8;
static constexpr size_t META_HEADER_SIZE = CLIENT_HEADER_SIZE;
static constexpr size_t META_SIZE = META_MAC_SIZE + META_HEADER_SIZE;
[[nodiscard]] static std::unique_ptr<ClientPacket> from_buffer(const pipes::buffer_view& buffer);
ClientPacket(const PacketTypeInfo& type, const pipes::buffer_view& data);
ClientPacket(const PacketTypeInfo& type, uint8_t flag_mask, const pipes::buffer_view& data);
~ClientPacket() override;
ClientPacket(const ClientPacket&) = delete;
ClientPacket(ClientPacket&&) = delete;
uint16_t clientId() const;
void clientId(uint16_t);
uint16_t packetId() const override;
uint16_t generationId() const override;
void generationId(uint16_t generation) { this->genId = generation; }
PacketTypeInfo type() const override;
void type(const PacketTypeInfo&);
private:
ClientPacket() = default;
const uint8_t &_flags_type_byte() const override {
return this->header().data_ptr<uint8_t>()[4];
}
uint8_t &_flags_type_byte() override {
return this->header().data_ptr<uint8_t>()[4];
}
void setPacketId(uint16_t, uint16_t) override;
};
class PacketParser {
public:
PacketParser(const PacketParser&) = delete;
explicit PacketParser(pipes::buffer_view buffer) : _buffer{std::move(buffer)} {}
[[nodiscard]] inline const void* data_ptr() const { return this->_buffer.data_ptr(); }
[[nodiscard]] inline void* mutable_data_ptr() { return (void*) this->_buffer.data_ptr(); }
[[nodiscard]] inline pipes::buffer_view buffer() const { return this->_buffer; }
[[nodiscard]] inline pipes::buffer_view mac() const { return this->_buffer.view(0, 8); }
[[nodiscard]] virtual pipes::buffer_view payload() const = 0;
[[nodiscard]] virtual size_t payload_length() const = 0;
[[nodiscard]] inline uint32_t full_packet_id() const { return this->packet_id() | (this->estimated_generation() << 16U); }
[[nodiscard]] virtual uint16_t packet_id() const = 0;
[[nodiscard]] virtual uint8_t type() const = 0;
[[nodiscard]] virtual uint8_t flags() const = 0;
[[nodiscard]] bool is_encrypted() const;
[[nodiscard]] bool is_compressed() const;
[[nodiscard]] bool is_fragmented() const;
[[nodiscard]] uint16_t estimated_generation() const { return this->generation; }
void set_estimated_generation(uint16_t gen) { this->generation = gen; }
inline void set_decrypted() { this->decrypted = true; }
inline void set_uncompressed() { this->uncompressed = true; }
inline void set_defragmented() { this->defragmented = true; }
protected:
uint16_t generation{};
bool decrypted{false}, uncompressed{false}, defragmented{false};
pipes::buffer_view _buffer{};
};
class ClientPacketParser : public PacketParser {
public:
constexpr static auto kHeaderOffset = 8;
constexpr static auto kHeaderLength = CLIENT_HEADER_SIZE;
constexpr static auto kPayloadOffset = kHeaderOffset + CLIENT_HEADER_SIZE;
explicit ClientPacketParser(pipes::buffer_view buffer) : PacketParser{std::move(buffer)} {}
ClientPacketParser(const ClientPacketParser&) = delete;
[[nodiscard]] inline bool valid() const {
if(this->_buffer.length() < kPayloadOffset) return false;
return this->type() <= 8;
}
[[nodiscard]] inline pipes::buffer_view payload() const override { return this->_buffer.view(kPayloadOffset); }
[[nodiscard]] inline size_t payload_length() const override { return this->_buffer.length() - kPayloadOffset; }
[[nodiscard]] uint16_t client_id() const;
[[nodiscard]] uint16_t packet_id() const override;
[[nodiscard]] uint8_t type() const override;
[[nodiscard]] uint8_t flags() const override;
};
/**
* Packet from the server
*/
class ServerPacket : public BasicPacket {
friend std::unique_ptr<ServerPacket> std::make_unique<ServerPacket>();
public:
static constexpr size_t META_MAC_SIZE = 8;
static constexpr size_t META_HEADER_SIZE = SERVER_HEADER_SIZE;
static constexpr size_t META_SIZE = META_MAC_SIZE + META_HEADER_SIZE;
[[nodiscard]] static std::unique_ptr<ServerPacket> from_buffer(const pipes::buffer_view& buffer);
ServerPacket(uint8_t flagMask, const pipes::buffer_view& data);
ServerPacket(const PacketTypeInfo& type, const pipes::buffer_view& data);
ServerPacket(PacketTypeInfo type, size_t /* data length */);
~ServerPacket() override;
ServerPacket(const ServerPacket&) = delete;
ServerPacket(ServerPacket&&) = delete;
[[nodiscard]] uint16_t packetId() const override;
[[nodiscard]] uint16_t generationId() const override;
void generationId(uint16_t generation) { this->genId = generation; }
[[nodiscard]] PacketTypeInfo type() const override;
private:
ServerPacket() = default;
[[nodiscard]] const uint8_t &_flags_type_byte() const override {
return this->header().data_ptr<uint8_t>()[2];
}
uint8_t &_flags_type_byte() override {
return this->header().data_ptr<uint8_t>()[2];
}
void setPacketId(uint16_t, uint16_t) override;
};
class ServerPacketP {
public:
constexpr static auto kHeaderOffset = 8;
constexpr static auto kHeaderLength = SERVER_HEADER_SIZE;
constexpr static auto kPayloadOffset = kHeaderOffset + SERVER_HEADER_SIZE;
};
class ServerPacketParser : public PacketParser, public ServerPacketP {
public:
explicit ServerPacketParser(pipes::buffer_view buffer) : PacketParser{std::move(buffer)} {}
ServerPacketParser(const ServerPacketParser&) = delete;
[[nodiscard]] inline bool valid() const {
if(this->_buffer.length() < kPayloadOffset) return false;
return this->type() <= 8;
}
[[nodiscard]] inline pipes::buffer_view payload() const override { return this->_buffer.view(kPayloadOffset); }
[[nodiscard]] inline size_t payload_length() const override { return this->_buffer.length() - kPayloadOffset; }
[[nodiscard]] uint16_t packet_id() const override;
[[nodiscard]] uint8_t type() const override;
[[nodiscard]] uint8_t flags() const override;
};
struct OutgoingServerPacket {
public:
/* general info */
std::atomic<uint16_t> ref_count;
size_t payload_size;
OutgoingServerPacket* next; /* used within the write/process queue */
uint16_t generation;
/* actual packet data */
uint8_t mac[8];
uint8_t packet_id_bytes[2];
uint8_t type_and_flags;
uint8_t payload[1]; /* variable size */
[[nodiscard]] inline const void* packet_data() const {
return this->mac;
}
[[nodiscard]] inline size_t packet_length() const {
return this->payload_size + (8 + 2 + 1);
}
inline auto ref() {
auto count = ++ref_count;
assert(count > 1);
return count;
}
inline void unref() {
if(--this->ref_count == 0)
this->object_freed();
}
/* some helper methods */
inline void set_packet_id(uint16_t id) {
this->packet_id_bytes[0] = id >> 8U;
this->packet_id_bytes[1] = id & 0xFFU;
}
[[nodiscard]] inline uint16_t packet_id() const {
return (uint16_t) (this->packet_id_bytes[0] << 8U) | this->packet_id_bytes[1];
}
[[nodiscard]] inline auto packet_type() const {
return (PacketType) (this->type_and_flags & 0xFU);
}
private:
void object_freed();
};
/* This will allocate a new outgoing packet. To delete just unref the packet! */
OutgoingServerPacket* allocate_outgoing_packet(size_t /* payload size */);
}
#define MAC_SIZE 8
#define SERVER_HEADER_SIZE 3
#define CLIENT_HEADER_SIZE 5
class PacketParser {
public:
PacketParser(const PacketParser&) = delete;
explicit PacketParser(pipes::buffer_view buffer) : _buffer{std::move(buffer)} {}
[[nodiscard]] inline const void* data_ptr() const { return this->_buffer.data_ptr(); }
[[nodiscard]] inline void* mutable_data_ptr() { return (void*) this->_buffer.data_ptr(); }
[[nodiscard]] inline pipes::buffer_view buffer() const { return this->_buffer; }
[[nodiscard]] inline pipes::buffer_view mac() const { return this->_buffer.view(0, 8); }
[[nodiscard]] virtual pipes::buffer_view payload() const = 0;
[[nodiscard]] virtual size_t payload_length() const = 0;
[[nodiscard]] inline uint32_t full_packet_id() const { return this->packet_id() | (uint32_t) ((uint32_t) this->estimated_generation() << 16U); }
[[nodiscard]] virtual uint16_t packet_id() const = 0;
[[nodiscard]] virtual uint8_t type() const = 0;
[[nodiscard]] virtual uint8_t flags() const = 0;
[[nodiscard]] inline bool has_flag(const PacketFlag& flag) const { return this->flags() & (uint8_t) flag; }
[[nodiscard]] inline bool is_encrypted() const {
return !this->decrypted && !this->has_flag(PacketFlag::Unencrypted);
}
[[nodiscard]] inline bool is_compressed() const {
return !this->uncompressed && this->has_flag(PacketFlag::Compressed);
}
[[nodiscard]] inline bool is_fragmented() const {
return !this->defragmented && this->has_flag(PacketFlag::Fragmented);
}
[[nodiscard]] uint16_t estimated_generation() const { return this->generation; }
void set_estimated_generation(uint16_t gen) { this->generation = gen; }
inline void set_decrypted() { this->decrypted = true; }
inline void set_uncompressed() { this->uncompressed = true; }
inline void set_defragmented() { this->defragmented = true; }
protected:
uint16_t generation{};
bool decrypted{false}, uncompressed{false}, defragmented{false};
pipes::buffer_view _buffer{};
};
class ClientPacketParser : public PacketParser {
public:
constexpr static auto kHeaderOffset = 8;
constexpr static auto kHeaderLength = CLIENT_HEADER_SIZE;
constexpr static auto kPayloadOffset = kHeaderOffset + CLIENT_HEADER_SIZE;
explicit ClientPacketParser(pipes::buffer_view buffer) : PacketParser{std::move(buffer)} {}
ClientPacketParser(const ClientPacketParser&) = delete;
[[nodiscard]] inline bool valid() const {
if(this->_buffer.length() < kPayloadOffset) return false;
return this->type() <= 8;
}
[[nodiscard]] inline pipes::buffer_view payload() const override { return this->_buffer.view(kPayloadOffset); }
[[nodiscard]] inline size_t payload_length() const override { return this->_buffer.length() - kPayloadOffset; }
[[nodiscard]] uint16_t client_id() const;
[[nodiscard]] uint16_t packet_id() const override;
[[nodiscard]] uint8_t type() const override;
[[nodiscard]] uint8_t flags() const override;
};
class ServerPacketParser : public PacketParser {
public:
constexpr static auto kHeaderOffset = 8;
constexpr static auto kHeaderLength = SERVER_HEADER_SIZE;
constexpr static auto kPayloadOffset = kHeaderOffset + SERVER_HEADER_SIZE;
explicit ServerPacketParser(pipes::buffer_view buffer) : PacketParser{std::move(buffer)} {}
ServerPacketParser(const ServerPacketParser&) = delete;
[[nodiscard]] inline bool valid() const {
if(this->_buffer.length() < kPayloadOffset) return false;
return this->type() <= 8;
}
[[nodiscard]] inline pipes::buffer_view payload() const override { return this->_buffer.view(kPayloadOffset); }
[[nodiscard]] inline size_t payload_length() const override { return this->_buffer.length() - kPayloadOffset; }
[[nodiscard]] uint16_t packet_id() const override;
[[nodiscard]] uint8_t type() const override;
[[nodiscard]] uint8_t flags() const override;
};
struct OutgoingServerPacket {
public:
/* general info */
std::atomic<uint16_t> ref_count;
size_t payload_size;
OutgoingServerPacket* next; /* used within the write/process queue */
uint16_t generation;
/* actual packet data */
uint8_t mac[8];
uint8_t packet_id_bytes[2];
uint8_t type_and_flags;
uint8_t payload[1]; /* variable size */
[[nodiscard]] inline const void* packet_data() const {
return this->mac;
}
[[nodiscard]] inline size_t packet_length() const {
return this->payload_size + (8 + 2 + 1);
}
inline auto ref() {
auto count = ++ref_count;
assert(count > 1);
return count;
}
inline void unref() {
if(--this->ref_count == 0)
this->object_freed();
}
/* some helper methods */
inline void set_packet_id(uint16_t id) {
this->packet_id_bytes[0] = id >> 8U;
this->packet_id_bytes[1] = id & 0xFFU;
}
[[nodiscard]] inline uint16_t packet_id() const {
return (uint16_t) (this->packet_id_bytes[0] << 8U) | this->packet_id_bytes[1];
}
[[nodiscard]] inline auto packet_type() const {
return (PacketType) (this->type_and_flags & 0xFU);
}
private:
void object_freed();
};
/* This will allocate a new outgoing packet. To delete just unref the packet! */
OutgoingServerPacket* allocate_outgoing_packet(size_t /* payload size */);
inline PacketFlags& operator|=(PacketFlags& flags, const PacketFlag& flag) {
flags |= (uint8_t) flag;
return flags;
}
inline PacketFlags operator|(PacketFlags flags, const PacketFlag& flag) {
return flags |= flag;
}
inline PacketFlags& operator&=(PacketFlags& flags, const PacketFlag& flag) {
flags &= (uint8_t) flag;
return flags;
}
inline PacketFlags operator&(PacketFlags flags, const PacketFlag& flag) {
return flags &= flag;
}
inline PacketFlags operator|(const PacketFlag& flag_a, const PacketFlag& flag_b) {
return (uint8_t) flag_a | (uint8_t) flag_b;
}
}

14
src/protocol/generation.cpp
View File

@ -2,22 +2,22 @@
using namespace ts::protocol;
generation_estimator::generation_estimator() {
GenerationEstimator::GenerationEstimator() {
this->reset();
}
void generation_estimator::reset() {
void GenerationEstimator::reset() {
this->last_generation = 0;
this->last_packet_id = 0;
}
uint16_t generation_estimator::visit_packet(uint16_t packet_id) {
if(this->last_packet_id >= generation_estimator::overflow_area_begin) {
uint16_t GenerationEstimator::visit_packet(uint16_t packet_id) {
if(this->last_packet_id >= GenerationEstimator::overflow_area_begin) {
if(packet_id > this->last_packet_id) {
/* normal behaviour */
this->last_packet_id = packet_id;
return this->last_generation;
} else if(packet_id < generation_estimator::overflow_area_end) {
} else if(packet_id < GenerationEstimator::overflow_area_end) {
/* we're within a new generation */
this->last_packet_id = packet_id;
return ++this->last_generation;
@ -25,8 +25,8 @@ uint16_t generation_estimator::visit_packet(uint16_t packet_id) {
/* old packet */
return this->last_generation;
}
} else if(this->last_packet_id <= generation_estimator::overflow_area_end) {
if(packet_id >= generation_estimator::overflow_area_begin) /* old packet */
} else if(this->last_packet_id <= GenerationEstimator::overflow_area_end) {
if(packet_id >= GenerationEstimator::overflow_area_begin) /* old packet */
return this->last_generation - 1;
if(packet_id > this->last_packet_id)
this->last_packet_id = packet_id;

4
src/protocol/generation.h
View File

@ -3,9 +3,9 @@
#include <cstdint>
namespace ts::protocol {
class generation_estimator {
class GenerationEstimator {
public:
generation_estimator();
GenerationEstimator();
void reset();
[[nodiscard]] uint16_t visit_packet(uint16_t /* packet id */);

825
src/qlz/QuickLZ.cpp
View File

@ -9,7 +9,7 @@
// 1.5.0 final
#include "QuickLZ.h"
#include "./QuickLZ.h"
#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0
#error quicklz.c and quicklz.h have different versions
@ -19,8 +19,829 @@
#define X86X64
#endif
#define MINOFFSET 2
#define UNCONDITIONAL_MATCHLEN 6
#define UNCOMPRESSED_END 4
#define CWORD_LEN 4
#if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0
#define OFFSET_BASE source
#define CAST (ui32)(size_t)
#else
#define OFFSET_BASE 0
#define CAST
#endif
int qlz_get_setting(int setting)
{
switch (setting)
{
case 0: return QLZ_COMPRESSION_LEVEL;
case 1: return sizeof(qlz_state_compress);
case 2: return sizeof(qlz_state_decompress);
case 3: return QLZ_STREAMING_BUFFER;
#ifdef QLZ_MEMORY_SAFE
case 6: return 1;
#else
case 6: return 0;
#endif
case 7: return QLZ_VERSION_MAJOR;
case 8: return QLZ_VERSION_MINOR;
case 9: return QLZ_VERSION_REVISION;
}
return -1;
}
#if QLZ_COMPRESSION_LEVEL == 1
static int same(const unsigned char *src, size_t n)
{
while(n > 0 && *(src + n) == *src)
n--;
return n == 0 ? 1 : 0;
}
#endif
static void reset_table_compress(qlz_state_compress *state)
{
int i;
for(i = 0; i < QLZ_HASH_VALUES; i++)
{
#if QLZ_COMPRESSION_LEVEL == 1
state->hash[i].offset = 0;
#else
state->hash_counter[i] = 0;
#endif
}
}
static void reset_table_decompress(qlz_state_decompress *state)
{
int i;
(void)state;
(void)i;
#if QLZ_COMPRESSION_LEVEL == 2
for(i = 0; i < QLZ_HASH_VALUES; i++)
{
state->hash_counter[i] = 0;
}
#endif
}
static __inline ui32 hash_func(ui32 i)
{
#if QLZ_COMPRESSION_LEVEL == 2
return ((i >> 9) ^ (i >> 13) ^ i) & (QLZ_HASH_VALUES - 1);
#else
return ((i >> 12) ^ i) & (QLZ_HASH_VALUES - 1);
#endif
}
static __inline ui32 fast_read(void const *src, ui32 bytes)
{
#ifndef X86X64
unsigned char *p = (unsigned char*)src;
switch (bytes)
{
case 4:
return(*p | *(p + 1) << 8 | *(p + 2) << 16 | *(p + 3) << 24);
case 3:
return(*p | *(p + 1) << 8 | *(p + 2) << 16);
case 2:
return(*p | *(p + 1) << 8);
case 1:
return(*p);
}
return 0;
#else
if (bytes >= 1 && bytes <= 4)
return *((ui32*)src);
else
return 0;
#endif
}
static __inline ui32 hashat(const unsigned char *src)
{
ui32 fetch, hash;
fetch = fast_read(src, 3);
hash = hash_func(fetch);
return hash;
}
static __inline void fast_write(ui32 f, void *dst, size_t bytes)
{
#ifndef X86X64
unsigned char *p = (unsigned char*)dst;
switch (bytes)
{
case 4:
*p = (unsigned char)f;
*(p + 1) = (unsigned char)(f >> 8);
*(p + 2) = (unsigned char)(f >> 16);
*(p + 3) = (unsigned char)(f >> 24);
return;
case 3:
*p = (unsigned char)f;
*(p + 1) = (unsigned char)(f >> 8);
*(p + 2) = (unsigned char)(f >> 16);
return;
case 2:
*p = (unsigned char)f;
*(p + 1) = (unsigned char)(f >> 8);
return;
case 1:
*p = (unsigned char)f;
return;
}
#else
switch (bytes)
{
case 4:
*((ui32*)dst) = f;
return;
case 3:
*((ui32*)dst) = f;
return;
case 2:
*((ui16 *)dst) = (ui16)f;
return;
case 1:
*((unsigned char*)dst) = (unsigned char)f;
return;
}
#endif
}
size_t qlz_size_decompressed(const char *source)
{
ui32 n, r;
n = (((*source) & 2) == 2) ? 4 : 1;
r = fast_read(source + 1 + n, n);
r = r & (0xffffffff >> ((4 - n)*8));
return r;
}
size_t qlz_size_compressed(const char *source)
{
ui32 n, r;
n = (((*source) & 2) == 2) ? 4 : 1;
r = fast_read(source + 1, n);
r = r & (0xffffffff >> ((4 - n)*8));
return r;
}
size_t qlz_size_header(const char *source)
{
size_t n = 2*((((*source) & 2) == 2) ? 4 : 1) + 1;
return n;
}
}
static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n)
{
// Caution if modifying memcpy_up! Overlap of dst and src must be special handled.
#ifndef X86X64
unsigned char *end = dst + n;
while(dst < end)
{
*dst = *src;
dst++;
src++;
}
#else
ui32 f = 0;
do
{
*(ui32 *)(dst + f) = *(ui32 *)(src + f);
f += MINOFFSET + 1;
}
while (f < n);
#endif
}
static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s)
{
#if QLZ_COMPRESSION_LEVEL == 1
ui32 hash;
hash = hashat(s);
state->hash[hash].offset = s;
state->hash_counter[hash] = 1;
#elif QLZ_COMPRESSION_LEVEL == 2
ui32 hash;
unsigned char c;
hash = hashat(s);
c = state->hash_counter[hash];
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s;
c++;
state->hash_counter[hash] = c;
#endif
(void)state;
(void)s;
}
#if QLZ_COMPRESSION_LEVEL <= 2
static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max)
{
while(*lh < max)
{
(*lh)++;
update_hash(state, *lh);
}
}
#endif
static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state)
{
const unsigned char *last_byte = source + size - 1;
const unsigned char *src = source;
unsigned char *cword_ptr = destination;
unsigned char *dst = destination + CWORD_LEN;
ui32 cword_val = 1U << 31;
const unsigned char *last_matchstart = last_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END;
ui32 fetch = 0;
unsigned int lits = 0;
(void) lits;
if(src <= last_matchstart)
fetch = fast_read(src, 3);
while(src <= last_matchstart)
{
if ((cword_val & 1) == 1)
{
// store uncompressed if compression ratio is too low
if (src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5))
return 0;
fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
cword_ptr = dst;
dst += CWORD_LEN;
cword_val = 1U << 31;
fetch = fast_read(src, 3);
}
#if QLZ_COMPRESSION_LEVEL == 1
{
const unsigned char *o;
ui32 hash, cached;
hash = hash_func(fetch);
cached = fetch ^ state->hash[hash].cache;
state->hash[hash].cache = fetch;
o = state->hash[hash].offset + OFFSET_BASE;
state->hash[hash].offset = CAST(src - OFFSET_BASE);
#ifdef X86X64
if ((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
{
if(cached != 0)
{
#else
if (cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
{
if (*(o + 3) != *(src + 3))
{
#endif
hash <<= 4;
cword_val = (cword_val >> 1) | (1U << 31);
fast_write((3 - 2) | hash, dst, 2);
src += 3;
dst += 2;
}
else
{
const unsigned char *old_src = src;
size_t matchlen;
hash <<= 4;
cword_val = (cword_val >> 1) | (1U << 31);
src += 4;
if(*(o + (src - old_src)) == *src)
{
src++;
if(*(o + (src - old_src)) == *src)
{
size_t q = last_byte - UNCOMPRESSED_END - (src - 5) + 1;
size_t remaining = q > 255 ? 255 : q;
src++;
while(*(o + (src - old_src)) == *src && (size_t)(src - old_src) < remaining)
src++;
}
}
matchlen = src - old_src;
if (matchlen < 18)
{
fast_write((ui32)(matchlen - 2) | hash, dst, 2);
dst += 2;
}
else
{
fast_write((ui32)(matchlen << 16) | hash, dst, 3);
dst += 3;
}
}
fetch = fast_read(src, 3);
lits = 0;
}
else
{
lits++;
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
#ifdef X86X64
fetch = fast_read(src, 3);
#else
fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16);
#endif
}
}
#elif QLZ_COMPRESSION_LEVEL >= 2
{
const unsigned char *o, *offset2;
ui32 hash, matchlen, k, m, best_k = 0;
unsigned char c;
size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1);
(void)best_k;
//hash = hashat(src);
fetch = fast_read(src, 3);
hash = hash_func(fetch);
c = state->hash_counter[hash];
offset2 = state->hash[hash].offset[0];
if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0)
{
matchlen = 3;
if(*(offset2 + matchlen) == *(src + matchlen))
{
matchlen = 4;
while(*(offset2 + matchlen) == *(src + matchlen) && matchlen < remaining)
matchlen++;
}
}
else
matchlen = 0;
for(k = 1; k < QLZ_POINTERS && c > k; k++)
{
o = state->hash[hash].offset[k];
#if QLZ_COMPRESSION_LEVEL == 3
if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
#elif QLZ_COMPRESSION_LEVEL == 2
if(*(src + matchlen) == *(o + matchlen) && ((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
#endif
{
m = 3;
while(*(o + m) == *(src + m) && m < remaining)
m++;
#if QLZ_COMPRESSION_LEVEL == 3
if ((m > matchlen) || (m == matchlen && o > offset2))
#elif QLZ_COMPRESSION_LEVEL == 2
if (m > matchlen)
#endif
{
offset2 = o;
matchlen = m;
best_k = k;
}
}
}
o = offset2;
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
c++;
state->hash_counter[hash] = c;
#if QLZ_COMPRESSION_LEVEL == 3
if(matchlen > 2 && src - o < 131071)
{
ui32 u;
size_t offset = src - o;
for(u = 1; u < matchlen; u++)
{
hash = hashat(src + u);
c = state->hash_counter[hash]++;
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u;
}
cword_val = (cword_val >> 1) | (1U << 31);
src += matchlen;
if(matchlen == 3 && offset <= 63)
{
*dst = (unsigned char)(offset << 2);
dst++;
}
else if (matchlen == 3 && offset <= 16383)
{
ui32 f = (ui32)((offset << 2) | 1);
fast_write(f, dst, 2);
dst += 2;
}
else if (matchlen <= 18 && offset <= 1023)
{
ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2;
fast_write(f, dst, 2);
dst += 2;
}
else if(matchlen <= 33)
{
ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3;
fast_write(f, dst, 3);
dst += 3;
}
else
{
ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3;
fast_write(f, dst, 4);
dst += 4;
}
}
else
{
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
}
#elif QLZ_COMPRESSION_LEVEL == 2
if(matchlen > 2)
{
cword_val = (cword_val >> 1) | (1U << 31);
src += matchlen;
if (matchlen < 10)
{
ui32 f = best_k | ((matchlen - 2) << 2) | (hash << 5);
fast_write(f, dst, 2);
dst += 2;
}
else
{
ui32 f = best_k | (matchlen << 16) | (hash << 5);
fast_write(f, dst, 3);
dst += 3;
}
}
else
{
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
}
#endif
}
#endif
}
while (src <= last_byte)
{
if ((cword_val & 1) == 1)
{
fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
cword_ptr = dst;
dst += CWORD_LEN;
cword_val = 1U << 31;
}
#if QLZ_COMPRESSION_LEVEL < 3
if (src <= last_byte - 3)
{
#if QLZ_COMPRESSION_LEVEL == 1
ui32 hash, fetch;
fetch = fast_read(src, 3);
hash = hash_func(fetch);
state->hash[hash].offset = CAST(src - OFFSET_BASE);
state->hash[hash].cache = fetch;
#elif QLZ_COMPRESSION_LEVEL == 2
ui32 hash;
unsigned char c;
hash = hashat(src);
c = state->hash_counter[hash];
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
c++;
state->hash_counter[hash] = c;
#endif
}
#endif
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
}
while((cword_val & 1) != 1)
cword_val = (cword_val >> 1);
fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
// min. size must be 9 bytes so that the qlz_size functions can take 9 bytes as argument
return dst - destination < 9 ? 9 : dst - destination;
}
static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history)
{
const unsigned char *src = source + qlz_size_header((const char *)source);
unsigned char *dst = destination;
const unsigned char *last_destination_byte = destination + size - 1;
ui32 cword_val = 1;
const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END;
unsigned char *last_hashed = destination - 1;
const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1;
static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0};
(void) last_source_byte;
(void) last_hashed;
(void) state;
(void) history;
for(;;)
{
ui32 fetch;
if (cword_val == 1)
{
#ifdef QLZ_MEMORY_SAFE
if(src + CWORD_LEN - 1 > last_source_byte)
return 0;
#endif
cword_val = fast_read(src, CWORD_LEN);
src += CWORD_LEN;
}
#ifdef QLZ_MEMORY_SAFE
if(src + 4 - 1 > last_source_byte)
return 0;
#endif
fetch = fast_read(src, 4);
if ((cword_val & 1) == 1)
{
ui32 matchlen;
const unsigned char *offset2;
#if QLZ_COMPRESSION_LEVEL == 1
ui32 hash;
cword_val = cword_val >> 1;
hash = (fetch >> 4) & 0xfff;
offset2 = (const unsigned char *)(size_t)state->hash[hash].offset;
if((fetch & 0xf) != 0)
{
matchlen = (fetch & 0xf) + 2;
src += 2;
}
else
{
matchlen = *(src + 2);
src += 3;
}
#elif QLZ_COMPRESSION_LEVEL == 2
ui32 hash;
unsigned char c;
cword_val = cword_val >> 1;
hash = (fetch >> 5) & 0x7ff;
c = (unsigned char)(fetch & 0x3);
offset2 = state->hash[hash].offset[c];
if((fetch & (28)) != 0)
{
matchlen = ((fetch >> 2) & 0x7) + 2;
src += 2;
}
else
{
matchlen = *(src + 2);
src += 3;
}
#elif QLZ_COMPRESSION_LEVEL == 3
ui32 offset;
cword_val = cword_val >> 1;
if ((fetch & 3) == 0)
{
offset = (fetch & 0xff) >> 2;
matchlen = 3;
src++;
}
else if ((fetch & 2) == 0)
{
offset = (fetch & 0xffff) >> 2;
matchlen = 3;
src += 2;
}
else if ((fetch & 1) == 0)
{
offset = (fetch & 0xffff) >> 6;
matchlen = ((fetch >> 2) & 15) + 3;
src += 2;
}
else if ((fetch & 127) != 3)
{
offset = (fetch >> 7) & 0x1ffff;
matchlen = ((fetch >> 2) & 0x1f) + 2;
src += 3;
}
else
{
offset = (fetch >> 15);
matchlen = ((fetch >> 7) & 255) + 3;
src += 4;
}
offset2 = dst - offset;
#endif
#ifdef QLZ_MEMORY_SAFE
if(offset2 < history || offset2 > dst - MINOFFSET - 1)
return 0;
if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1))
return 0;
#endif
memcpy_up(dst, offset2, matchlen);
dst += matchlen;
#if QLZ_COMPRESSION_LEVEL <= 2
update_hash_upto(state, &last_hashed, dst - matchlen);
last_hashed = dst - 1;
#endif
}
else
{
if (dst < last_matchstart)
{
unsigned int n = bitlut[cword_val & 0xf];
#ifdef X86X64
*(ui32 *)dst = *(ui32 *)src;
#else
memcpy_up(dst, src, 4);
#endif
cword_val = cword_val >> n;
dst += n;
src += n;
#if QLZ_COMPRESSION_LEVEL <= 2
update_hash_upto(state, &last_hashed, dst - 3);
#endif
}
else
{
while(dst <= last_destination_byte)
{
if (cword_val == 1)
{
src += CWORD_LEN;
cword_val = 1U << 31;
}
#ifdef QLZ_MEMORY_SAFE
if(src >= last_source_byte + 1)
return 0;
#endif
*dst = *src;
dst++;
src++;
cword_val = cword_val >> 1;
}
#if QLZ_COMPRESSION_LEVEL <= 2
update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant
#endif
return size;
}
}
}
}
size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state)
{
size_t r;
ui32 compressed;
size_t base;
if(size == 0 || size > 0xffffffff - 400)
return 0;
if(size < 216)
base = 3;
else
base = 9;
#if QLZ_STREAMING_BUFFER > 0
if (state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER)
#endif
{
reset_table_compress(state);
r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, state);
#if QLZ_STREAMING_BUFFER > 0
reset_table_compress(state);
#endif
if(r == base)
{
memcpy(destination + base, source, size);
r = size + base;
compressed = 0;
}
else
{
compressed = 1;
}
state->stream_counter = 0;
}
#if QLZ_STREAMING_BUFFER > 0
else
{
unsigned char *src = state->stream_buffer + state->stream_counter;
memcpy(src, source, size);
r = base + qlz_compress_core(src, (unsigned char*)destination + base, size, state);
if(r == base)
{
memcpy(destination + base, src, size);
r = size + base;
compressed = 0;
reset_table_compress(state);
}
else
{
compressed = 1;
}
state->stream_counter += size;
}
#endif
if(base == 3)
{
*destination = (unsigned char)(0 | compressed);
*(destination + 1) = (unsigned char)r;
*(destination + 2) = (unsigned char)size;
}
else
{
*destination = (unsigned char)(2 | compressed);
fast_write((ui32)r, destination + 1, 4);
fast_write((ui32)size, destination + 5, 4);
}
*destination |= (QLZ_COMPRESSION_LEVEL << 2);
*destination |= (1 << 6);
*destination |= ((QLZ_STREAMING_BUFFER == 0 ? 0 : (QLZ_STREAMING_BUFFER == 100000 ? 1 : (QLZ_STREAMING_BUFFER == 1000000 ? 2 : 3))) << 4);
// 76543210
// 01SSLLHC
return r;
}
size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state)
{
size_t dsiz = qlz_size_decompressed(source);
#if QLZ_STREAMING_BUFFER > 0
if (state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER)
#endif
{
if((*source & 1) == 1)
{
reset_table_decompress(state);
dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination);
}
else
{
memcpy(destination, source + qlz_size_header(source), dsiz);
}
state->stream_counter = 0;
reset_table_decompress(state);
}
#if QLZ_STREAMING_BUFFER > 0
else
{
unsigned char *dst = state->stream_buffer + state->stream_counter;
if((*source & 1) == 1)
{
dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer);
}
else
{
memcpy(dst, source + qlz_size_header(source), dsiz);
reset_table_decompress(state);
}
memcpy(destination, dst, dsiz);
state->stream_counter += dsiz;
}
#endif
return dsiz;
}

97
src/qlz/QuickLZ.h
View File

@ -18,25 +18,10 @@
// 1.5.0 final
#ifndef QLZ_COMPRESSION_LEVEL
// 1 gives fastest compression speed. 3 gives fastest decompression speed and best
// compression ratio.
#define QLZ_COMPRESSION_LEVEL 3
//#define QLZ_COMPRESSION_LEVEL 2
//#define QLZ_COMPRESSION_LEVEL 3
#endif
#ifndef QLZ_STREAMING_BUFFER
// If > 0, zero out both states prior to first call to qlz_compress() or qlz_decompress()
// and decompress packets in the same order as they were compressed
/* Fixed definitions for TeaSpeak. */
#define QLZ_COMPRESSION_LEVEL 1
#define QLZ_STREAMING_BUFFER 0
//#define QLZ_STREAMING_BUFFER 100000
//#define QLZ_STREAMING_BUFFER 1000000
// Guarantees that decompression of corrupted data cannot crash. Decreases decompression
// speed 10-20%. Compression speed not affected.
//#define QLZ_MEMORY_SAFE
#endif
#define QLZ_MEMORY_SAFE
#define QLZ_VERSION_MAJOR 1
#define QLZ_VERSION_MINOR 5
@ -67,64 +52,68 @@ typedef unsigned short int ui16;
// Detect if pointer size is 64-bit. It's not fatal if some 64-bit target is not detected because this is only for adding an optional 64-bit optimization.
#if defined _LP64 || defined __LP64__ || defined __64BIT__ || _ADDR64 || defined _WIN64 || defined __arch64__ || __WORDSIZE == 64 || (defined __sparc && defined __sparcv9) || defined __x86_64 || defined __amd64 || defined __x86_64__ || defined _M_X64 || defined _M_IA64 || defined __ia64 || defined __IA64__
#define QLZ_PTR_64
#define QLZ_PTR_64
#endif
// hash entry
typedef struct {
typedef struct
{
#if QLZ_COMPRESSION_LEVEL == 1
ui32 cache;
ui32 cache;
#if defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0
unsigned int offset;
unsigned int offset;
#else
const unsigned char *offset;
const unsigned char *offset;
#endif
#else
const unsigned char *offset[QLZ_POINTERS];
const unsigned char *offset[QLZ_POINTERS];
#endif
} qlz_hash_compress;
typedef struct {
typedef struct
{
#if QLZ_COMPRESSION_LEVEL == 1
const unsigned char *offset;
const unsigned char *offset;
#else
const unsigned char *offset[QLZ_POINTERS];
const unsigned char *offset[QLZ_POINTERS];
#endif
} qlz_hash_decompress;
// states
typedef struct {
#if QLZ_STREAMING_BUFFER > 0
unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
#endif
size_t stream_counter;
qlz_hash_compress hash[QLZ_HASH_VALUES];
unsigned char hash_counter[QLZ_HASH_VALUES];
typedef struct
{
#if QLZ_STREAMING_BUFFER > 0
unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
#endif
size_t stream_counter;
qlz_hash_compress hash[QLZ_HASH_VALUES];
unsigned char hash_counter[QLZ_HASH_VALUES];
} qlz_state_compress;
#if QLZ_COMPRESSION_LEVEL == 1 || QLZ_COMPRESSION_LEVEL == 2
typedef struct {
typedef struct
{
#if QLZ_STREAMING_BUFFER > 0
unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
#endif
qlz_hash_decompress hash[QLZ_HASH_VALUES];
unsigned char hash_counter[QLZ_HASH_VALUES];
size_t stream_counter;
} qlz_state_decompress;
qlz_hash_decompress hash[QLZ_HASH_VALUES];
unsigned char hash_counter[QLZ_HASH_VALUES];
size_t stream_counter;
} qlz_state_decompress;
#elif QLZ_COMPRESSION_LEVEL == 3
typedef struct
{
typedef struct
{
#if QLZ_STREAMING_BUFFER > 0
unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
#endif
#if QLZ_COMPRESSION_LEVEL <= 2
qlz_hash_decompress hash[QLZ_HASH_VALUES];
qlz_hash_decompress hash[QLZ_HASH_VALUES];
#endif
size_t stream_counter;
} qlz_state_decompress;
size_t stream_counter;
} qlz_state_decompress;
#endif
@ -132,22 +121,6 @@ typedef struct
extern "C" {
#endif
#if QLZ_COMPRESSION_LEVEL == 1
#define qlz_size_decompressed qlz_size_decompressed_level1
#define qlz_size_compressed qlz_size_compressed_level1
#define qlz_compress qlz_compress_level1
#define qlz_decompress qlz_decompress_level1
#define qlz_get_setting qlz_get_setting_level1
#elif QLZ_COMPRESSION_LEVEL == 2
#define DEFINE_QLZ_FN(type, name, ...) type name##_level2(#__VA_ARGS__);
#elif QLZ_COMPRESSION_LEVEL == 3
#define qlz_size_decompressed qlz_size_decompressed_level3
#define qlz_size_compressed qlz_size_compressed_level3
#define qlz_compress qlz_compress_level3
#define qlz_decompress qlz_decompress_level3
#define qlz_get_setting qlz_get_setting_level3
#endif
// Public functions of QuickLZ
size_t qlz_size_decompressed(const char *source);
size_t qlz_size_compressed(const char *source);

843
src/qlz/QuickLZ_L1.cpp
View File

@ -1,843 +0,0 @@
// Fast data compression library
// Copyright (C) 2006-2011 Lasse Mikkel Reinhold
// lar@quicklz.com
//
// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything
// released into public must be open source) or under a commercial license if such
// has been acquired (see http://www.quicklz.com/order.html). The commercial license
// does not cover derived or ported versions created by third parties under GPL.
// 1.5.0 final
#define QLZ_COMPRESSION_LEVEL 1
#define QLZ_MEMORY_SAFE
#include "QuickLZ.h"
#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0
#error quicklz.c and quicklz.h have different versions
#endif
#if (defined(__X86__) || defined(__i386__) || defined(i386) || defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || defined(_M_X64))
#define X86X64
#endif
#define MINOFFSET 2
#define UNCONDITIONAL_MATCHLEN 6
#define UNCOMPRESSED_END 4
#define CWORD_LEN 4
#if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0
#define OFFSET_BASE source
#define CAST (ui32)(size_t)
#else
#define OFFSET_BASE 0
#define CAST
#endif
int qlz_get_setting(int setting)
{
switch (setting)
{
case 0: return QLZ_COMPRESSION_LEVEL;
case 1: return sizeof(qlz_state_compress);
case 2: return sizeof(qlz_state_decompress);
case 3: return QLZ_STREAMING_BUFFER;
#ifdef QLZ_MEMORY_SAFE
case 6: return 1;
#else
case 6: return 0;
#endif
case 7: return QLZ_VERSION_MAJOR;
case 8: return QLZ_VERSION_MINOR;
case 9: return QLZ_VERSION_REVISION;
}
return -1;
}
#if QLZ_COMPRESSION_LEVEL == 1
static int same(const unsigned char *src, size_t n)
{
while(n > 0 && *(src + n) == *src)
n--;
return n == 0 ? 1 : 0;
}
#endif
static void reset_table_compress(qlz_state_compress *state)
{
int i;
for(i = 0; i < QLZ_HASH_VALUES; i++)
{
#if QLZ_COMPRESSION_LEVEL == 1
state->hash[i].offset = 0;
#else
state->hash_counter[i] = 0;
#endif
}
}
static void reset_table_decompress(qlz_state_decompress *state)
{
int i;
(void)state;
(void)i;
#if QLZ_COMPRESSION_LEVEL == 2
for(i = 0; i < QLZ_HASH_VALUES; i++)
{
state->hash_counter[i] = 0;
}
#endif
}
static __inline ui32 hash_func(ui32 i)
{
#if QLZ_COMPRESSION_LEVEL == 2
return ((i >> 9) ^ (i >> 13) ^ i) & (QLZ_HASH_VALUES - 1);
#else
return ((i >> 12) ^ i) & (QLZ_HASH_VALUES - 1);
#endif
}
static __inline ui32 fast_read(void const *src, ui32 bytes)
{
#ifndef X86X64
unsigned char *p = (unsigned char*)src;
switch (bytes)
{
case 4:
return(*p | *(p + 1) << 8 | *(p + 2) << 16 | *(p + 3) << 24);
case 3:
return(*p | *(p + 1) << 8 | *(p + 2) << 16);
case 2:
return(*p | *(p + 1) << 8);
case 1:
return(*p);
}
return 0;
#else
if (bytes >= 1 && bytes <= 4)
return *((ui32*)src);
else
return 0;
#endif
}
static __inline ui32 hashat(const unsigned char *src)
{
ui32 fetch, hash;
fetch = fast_read(src, 3);
hash = hash_func(fetch);
return hash;
}
static __inline void fast_write(ui32 f, void *dst, size_t bytes)
{
#ifndef X86X64
unsigned char *p = (unsigned char*)dst;
switch (bytes)
{
case 4:
*p = (unsigned char)f;
*(p + 1) = (unsigned char)(f >> 8);
*(p + 2) = (unsigned char)(f >> 16);
*(p + 3) = (unsigned char)(f >> 24);
return;
case 3:
*p = (unsigned char)f;
*(p + 1) = (unsigned char)(f >> 8);
*(p + 2) = (unsigned char)(f >> 16);
return;
case 2:
*p = (unsigned char)f;
*(p + 1) = (unsigned char)(f >> 8);
return;
case 1:
*p = (unsigned char)f;
return;
}
#else
switch (bytes)
{
case 4:
*((ui32*)dst) = f;
return;
case 3:
*((ui32*)dst) = f;
return;
case 2:
*((ui16 *)dst) = (ui16)f;
return;
case 1:
*((unsigned char*)dst) = (unsigned char)f;
return;
}
#endif
}
size_t qlz_size_decompressed(const char *source)
{
ui32 n, r;
n = (((*source) & 2) == 2) ? 4 : 1;
r = fast_read(source + 1 + n, n);
r = r & (0xffffffff >> ((4 - n)*8));
return r;
}
size_t qlz_size_compressed(const char *source)
{
ui32 n, r;
n = (((*source) & 2) == 2) ? 4 : 1;
r = fast_read(source + 1, n);
r = r & (0xffffffff >> ((4 - n)*8));
return r;
}
static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n)
{
// Caution if modifying memcpy_up! Overlap of dst and src must be special handled.
#ifndef X86X64
unsigned char *end = dst + n;
while(dst < end)
{
*dst = *src;
dst++;
src++;
}
#else
ui32 f = 0;
do
{
*(ui32 *)(dst + f) = *(ui32 *)(src + f);
f += MINOFFSET + 1;
}
while (f < n);
#endif
}
static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s)
{
#if QLZ_COMPRESSION_LEVEL == 1
ui32 hash;
hash = hashat(s);
state->hash[hash].offset = s;
state->hash_counter[hash] = 1;
#elif QLZ_COMPRESSION_LEVEL == 2
ui32 hash;
unsigned char c;
hash = hashat(s);
c = state->hash_counter[hash];
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s;
c++;
state->hash_counter[hash] = c;
#endif
(void)state;
(void)s;
}
#if QLZ_COMPRESSION_LEVEL <= 2
static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max)
{
while(*lh < max)
{
(*lh)++;
update_hash(state, *lh);
}
}
#endif
static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state)
{
const unsigned char *last_byte = source + size - 1;
const unsigned char *src = source;
unsigned char *cword_ptr = destination;
unsigned char *dst = destination + CWORD_LEN;
ui32 cword_val = 1U << 31;
const unsigned char *last_matchstart = last_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END;
ui32 fetch = 0;
unsigned int lits = 0;
(void) lits;
if(src <= last_matchstart)
fetch = fast_read(src, 3);
while(src <= last_matchstart)
{
if ((cword_val & 1) == 1)
{
// store uncompressed if compression ratio is too low
if (src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5))
return 0;
fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
cword_ptr = dst;
dst += CWORD_LEN;
cword_val = 1U << 31;
fetch = fast_read(src, 3);
}
#if QLZ_COMPRESSION_LEVEL == 1
{
const unsigned char *o;
ui32 hash, cached;
hash = hash_func(fetch);
cached = fetch ^ state->hash[hash].cache;
state->hash[hash].cache = fetch;
o = state->hash[hash].offset + OFFSET_BASE;
state->hash[hash].offset = CAST(src - OFFSET_BASE);
#ifdef X86X64
if ((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
{
if(cached != 0)
{
#else
if (cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
{
if (*(o + 3) != *(src + 3))
{
#endif
hash <<= 4;
cword_val = (cword_val >> 1) | (1U << 31);
fast_write((3 - 2) | hash, dst, 2);
src += 3;
dst += 2;
}
else
{
const unsigned char *old_src = src;
size_t matchlen;
hash <<= 4;
cword_val = (cword_val >> 1) | (1U << 31);
src += 4;
if(*(o + (src - old_src)) == *src)
{
src++;
if(*(o + (src - old_src)) == *src)
{
size_t q = last_byte - UNCOMPRESSED_END - (src - 5) + 1;
size_t remaining = q > 255 ? 255 : q;
src++;
while(*(o + (src - old_src)) == *src && (size_t)(src - old_src) < remaining)
src++;
}
}
matchlen = src - old_src;
if (matchlen < 18)
{
fast_write((ui32)(matchlen - 2) | hash, dst, 2);
dst += 2;
}
else
{
fast_write((ui32)(matchlen << 16) | hash, dst, 3);
dst += 3;
}
}
fetch = fast_read(src, 3);
lits = 0;
}
else
{
lits++;
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
#ifdef X86X64
fetch = fast_read(src, 3);
#else
fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16);
#endif
}
}
#elif QLZ_COMPRESSION_LEVEL >= 2
{
const unsigned char *o, *offset2;
ui32 hash, matchlen, k, m, best_k = 0;
unsigned char c;
size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1);
(void)best_k;
//hash = hashat(src);
fetch = fast_read(src, 3);
hash = hash_func(fetch);
c = state->hash_counter[hash];
offset2 = state->hash[hash].offset[0];
if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0)
{
matchlen = 3;
if(*(offset2 + matchlen) == *(src + matchlen))
{
matchlen = 4;
while(*(offset2 + matchlen) == *(src + matchlen) && matchlen < remaining)
matchlen++;
}
}
else
matchlen = 0;
for(k = 1; k < QLZ_POINTERS && c > k; k++)
{
o = state->hash[hash].offset[k];
#if QLZ_COMPRESSION_LEVEL == 3
if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
#elif QLZ_COMPRESSION_LEVEL == 2
if(*(src + matchlen) == *(o + matchlen) && ((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
#endif
{
m = 3;
while(*(o + m) == *(src + m) && m < remaining)
m++;
#if QLZ_COMPRESSION_LEVEL == 3
if ((m > matchlen) || (m == matchlen && o > offset2))
#elif QLZ_COMPRESSION_LEVEL == 2
if (m > matchlen)
#endif
{
offset2 = o;
matchlen = m;
best_k = k;
}
}
}
o = offset2;
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
c++;
state->hash_counter[hash] = c;
#if QLZ_COMPRESSION_LEVEL == 3
if(matchlen > 2 && src - o < 131071)
{
ui32 u;
size_t offset = src - o;
for(u = 1; u < matchlen; u++)
{
hash = hashat(src + u);
c = state->hash_counter[hash]++;
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u;
}
cword_val = (cword_val >> 1) | (1U << 31);
src += matchlen;
if(matchlen == 3 && offset <= 63)
{
*dst = (unsigned char)(offset << 2);
dst++;
}
else if (matchlen == 3 && offset <= 16383)
{
ui32 f = (ui32)((offset << 2) | 1);
fast_write(f, dst, 2);
dst += 2;
}
else if (matchlen <= 18 && offset <= 1023)
{
ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2;
fast_write(f, dst, 2);
dst += 2;
}
else if(matchlen <= 33)
{
ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3;
fast_write(f, dst, 3);
dst += 3;
}
else
{
ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3;
fast_write(f, dst, 4);
dst += 4;
}
}
else
{
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
}
#elif QLZ_COMPRESSION_LEVEL == 2
if(matchlen > 2)
{
cword_val = (cword_val >> 1) | (1U << 31);
src += matchlen;
if (matchlen < 10)
{
ui32 f = best_k | ((matchlen - 2) << 2) | (hash << 5);
fast_write(f, dst, 2);
dst += 2;
}
else
{
ui32 f = best_k | (matchlen << 16) | (hash << 5);
fast_write(f, dst, 3);
dst += 3;
}
}
else
{
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
}
#endif
}
#endif
}
while (src <= last_byte)
{
if ((cword_val & 1) == 1)
{
fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
cword_ptr = dst;
dst += CWORD_LEN;
cword_val = 1U << 31;
}
#if QLZ_COMPRESSION_LEVEL < 3
if (src <= last_byte - 3)
{
#if QLZ_COMPRESSION_LEVEL == 1
ui32 hash, fetch;
fetch = fast_read(src, 3);
hash = hash_func(fetch);
state->hash[hash].offset = CAST(src - OFFSET_BASE);
state->hash[hash].cache = fetch;
#elif QLZ_COMPRESSION_LEVEL == 2
ui32 hash;
unsigned char c;
hash = hashat(src);
c = state->hash_counter[hash];
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
c++;
state->hash_counter[hash] = c;
#endif
}
#endif
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
}
while((cword_val & 1) != 1)
cword_val = (cword_val >> 1);
fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
// min. size must be 9 bytes so that the qlz_size functions can take 9 bytes as argument
return dst - destination < 9 ? 9 : dst - destination;
}
extern size_t qlz_size_header(const char *source);
static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history)
{
const unsigned char *src = source + qlz_size_header((const char *)source);
unsigned char *dst = destination;
const unsigned char *last_destination_byte = destination + size - 1;
ui32 cword_val = 1;
const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END;
unsigned char *last_hashed = destination - 1;
const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1;
static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0};
(void) last_source_byte;
(void) last_hashed;
(void) state;
(void) history;
for(;;)
{
ui32 fetch;
if (cword_val == 1)
{
#ifdef QLZ_MEMORY_SAFE
if(src + CWORD_LEN - 1 > last_source_byte)
return 0;
#endif
cword_val = fast_read(src, CWORD_LEN);
src += CWORD_LEN;
}
#ifdef QLZ_MEMORY_SAFE
if(src + 4 - 1 > last_source_byte)
return 0;
#endif
fetch = fast_read(src, 4);
if ((cword_val & 1) == 1)
{
ui32 matchlen;
const unsigned char *offset2;
#if QLZ_COMPRESSION_LEVEL == 1
ui32 hash;
cword_val = cword_val >> 1;
hash = (fetch >> 4) & 0xfff;
offset2 = (const unsigned char *)(size_t)state->hash[hash].offset;
if((fetch & 0xf) != 0)
{
matchlen = (fetch & 0xf) + 2;
src += 2;
}
else
{
matchlen = *(src + 2);
src += 3;
}
#elif QLZ_COMPRESSION_LEVEL == 2
ui32 hash;
unsigned char c;
cword_val = cword_val >> 1;
hash = (fetch >> 5) & 0x7ff;
c = (unsigned char)(fetch & 0x3);
offset2 = state->hash[hash].offset[c];
if((fetch & (28)) != 0)
{
matchlen = ((fetch >> 2) & 0x7) + 2;
src += 2;
}
else
{
matchlen = *(src + 2);
src += 3;
}
#elif QLZ_COMPRESSION_LEVEL == 3
ui32 offset;
cword_val = cword_val >> 1;
if ((fetch & 3) == 0)
{
offset = (fetch & 0xff) >> 2;
matchlen = 3;
src++;
}
else if ((fetch & 2) == 0)
{
offset = (fetch & 0xffff) >> 2;
matchlen = 3;
src += 2;
}
else if ((fetch & 1) == 0)
{
offset = (fetch & 0xffff) >> 6;
matchlen = ((fetch >> 2) & 15) + 3;
src += 2;
}
else if ((fetch & 127) != 3)
{
offset = (fetch >> 7) & 0x1ffff;
matchlen = ((fetch >> 2) & 0x1f) + 2;
src += 3;
}
else
{
offset = (fetch >> 15);
matchlen = ((fetch >> 7) & 255) + 3;
src += 4;
}
offset2 = dst - offset;
#endif
#ifdef QLZ_MEMORY_SAFE
if(offset2 < history || offset2 > dst - MINOFFSET - 1)
return 0;
if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1))
return 0;
#endif
memcpy_up(dst, offset2, matchlen);
dst += matchlen;
#if QLZ_COMPRESSION_LEVEL <= 2
update_hash_upto(state, &last_hashed, dst - matchlen);
last_hashed = dst - 1;
#endif
}
else
{
if (dst < last_matchstart)
{
unsigned int n = bitlut[cword_val & 0xf];
#ifdef X86X64
*(ui32 *)dst = *(ui32 *)src;
#else
memcpy_up(dst, src, 4);
#endif
cword_val = cword_val >> n;
dst += n;
src += n;
#if QLZ_COMPRESSION_LEVEL <= 2
update_hash_upto(state, &last_hashed, dst - 3);
#endif
}
else
{
while(dst <= last_destination_byte)
{
if (cword_val == 1)
{
src += CWORD_LEN;
cword_val = 1U << 31;
}
#ifdef QLZ_MEMORY_SAFE
if(src >= last_source_byte + 1)
return 0;
#endif
*dst = *src;
dst++;
src++;
cword_val = cword_val >> 1;
}
#if QLZ_COMPRESSION_LEVEL <= 2
update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant
#endif
return size;
}
}
}
}
size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state)
{
size_t r;
ui32 compressed;
size_t base;
if(size == 0 || size > 0xffffffff - 400)
return 0;
if(size < 216)
base = 3;
else
base = 9;
#if QLZ_STREAMING_BUFFER > 0
if (state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER)
#endif
{
reset_table_compress(state);
r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, state);
#if QLZ_STREAMING_BUFFER > 0
reset_table_compress(state);
#endif
if(r == base)
{
memcpy(destination + base, source, size);
r = size + base;
compressed = 0;
}
else
{
compressed = 1;
}
state->stream_counter = 0;
}
#if QLZ_STREAMING_BUFFER > 0
else
{
unsigned char *src = state->stream_buffer + state->stream_counter;
memcpy(src, source, size);
r = base + qlz_compress_core(src, (unsigned char*)destination + base, size, state);
if(r == base)
{
memcpy(destination + base, src, size);
r = size + base;
compressed = 0;
reset_table_compress(state);
}
else
{
compressed = 1;
}
state->stream_counter += size;
}
#endif
if(base == 3)
{
*destination = (unsigned char)(0 | compressed);
*(destination + 1) = (unsigned char)r;
*(destination + 2) = (unsigned char)size;
}
else
{
*destination = (unsigned char)(2 | compressed);
fast_write((ui32)r, destination + 1, 4);
fast_write((ui32)size, destination + 5, 4);
}
*destination |= (QLZ_COMPRESSION_LEVEL << 2);
*destination |= (1 << 6);
*destination |= ((QLZ_STREAMING_BUFFER == 0 ? 0 : (QLZ_STREAMING_BUFFER == 100000 ? 1 : (QLZ_STREAMING_BUFFER == 1000000 ? 2 : 3))) << 4);
// 76543210
// 01SSLLHC
return r;
}
size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state)
{
size_t dsiz = qlz_size_decompressed(source);
#if QLZ_STREAMING_BUFFER > 0
if (state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER)
#endif
{
if((*source & 1) == 1)
{
reset_table_decompress(state);
dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination);
}
else
{
memcpy(destination, source + qlz_size_header(source), dsiz);
}
state->stream_counter = 0;
reset_table_decompress(state);
}
#if QLZ_STREAMING_BUFFER > 0
else
{
unsigned char *dst = state->stream_buffer + state->stream_counter;
if((*source & 1) == 1)
{
dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer);
}
else
{
memcpy(dst, source + qlz_size_header(source), dsiz);
reset_table_decompress(state);
}
memcpy(destination, dst, dsiz);
state->stream_counter += dsiz;
}
#endif
return dsiz;
}

848
src/qlz/QuickLZ_L2.cpp
View File

@ -1,848 +0,0 @@
// Fast data compression library
// Copyright (C) 2006-2011 Lasse Mikkel Reinhold
// lar@quicklz.com
//
// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything
// released into public must be open source) or under a commercial license if such
// has been acquired (see http://www.quicklz.com/order.html). The commercial license
// does not cover derived or ported versions created by third parties under GPL.
// 1.5.0 final
#define QLZ_COMPRESSION_LEVEL 2
#include "QuickLZ.h"
#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0
#error quicklz.c and quicklz.h have different versions
#endif
#if (defined(__X86__) || defined(__i386__) || defined(i386) || defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || defined(_M_X64))
#define X86X64
#endif
#define MINOFFSET 2
#define UNCONDITIONAL_MATCHLEN 6
#define UNCOMPRESSED_END 4
#define CWORD_LEN 4
#if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0
#define OFFSET_BASE source
#define CAST (ui32)(size_t)
#else
#define OFFSET_BASE 0
#define CAST
#endif
int qlz_get_setting(int setting)
{
switch (setting)
{
case 0: return QLZ_COMPRESSION_LEVEL;
case 1: return sizeof(qlz_state_compress);
case 2: return sizeof(qlz_state_decompress);
case 3: return QLZ_STREAMING_BUFFER;
#ifdef QLZ_MEMORY_SAFE
case 6: return 1;
#else
case 6: return 0;
#endif
case 7: return QLZ_VERSION_MAJOR;
case 8: return QLZ_VERSION_MINOR;
case 9: return QLZ_VERSION_REVISION;
}
return -1;
}
#if QLZ_COMPRESSION_LEVEL == 1
static int same(const unsigned char *src, size_t n)
{
while(n > 0 && *(src + n) == *src)
n--;
return n == 0 ? 1 : 0;
}
#endif
static void reset_table_compress(qlz_state_compress *state)
{
int i;
for(i = 0; i < QLZ_HASH_VALUES; i++)
{
#if QLZ_COMPRESSION_LEVEL == 1
state->hash[i].offset = 0;
#else
state->hash_counter[i] = 0;
#endif
}
}
static void reset_table_decompress(qlz_state_decompress *state)
{
int i;
(void)state;
(void)i;
#if QLZ_COMPRESSION_LEVEL == 2
for(i = 0; i < QLZ_HASH_VALUES; i++)
{
state->hash_counter[i] = 0;
}
#endif
}
static __inline ui32 hash_func(ui32 i)
{
#if QLZ_COMPRESSION_LEVEL == 2
return ((i >> 9) ^ (i >> 13) ^ i) & (QLZ_HASH_VALUES - 1);
#else
return ((i >> 12) ^ i) & (QLZ_HASH_VALUES - 1);
#endif
}
static __inline ui32 fast_read(void const *src, ui32 bytes)
{
#ifndef X86X64
unsigned char *p = (unsigned char*)src;
switch (bytes)
{
case 4:
return(*p | *(p + 1) << 8 | *(p + 2) << 16 | *(p + 3) << 24);
case 3:
return(*p | *(p + 1) << 8 | *(p + 2) << 16);
case 2:
return(*p | *(p + 1) << 8);
case 1:
return(*p);
}
return 0;
#else
if (bytes >= 1 && bytes <= 4)
return *((ui32*)src);
else
return 0;
#endif
}
static __inline ui32 hashat(const unsigned char *src)
{
ui32 fetch, hash;
fetch = fast_read(src, 3);
hash = hash_func(fetch);
return hash;
}
static __inline void fast_write(ui32 f, void *dst, size_t bytes)
{
#ifndef X86X64
unsigned char *p = (unsigned char*)dst;
switch (bytes)
{
case 4:
*p = (unsigned char)f;
*(p + 1) = (unsigned char)(f >> 8);
*(p + 2) = (unsigned char)(f >> 16);
*(p + 3) = (unsigned char)(f >> 24);
return;
case 3:
*p = (unsigned char)f;
*(p + 1) = (unsigned char)(f >> 8);
*(p + 2) = (unsigned char)(f >> 16);
return;
case 2:
*p = (unsigned char)f;
*(p + 1) = (unsigned char)(f >> 8);
return;
case 1:
*p = (unsigned char)f;
return;
}
#else
switch (bytes)
{
case 4:
*((ui32*)dst) = f;
return;
case 3:
*((ui32*)dst) = f;
return;
case 2:
*((ui16 *)dst) = (ui16)f;
return;
case 1:
*((unsigned char*)dst) = (unsigned char)f;
return;
}
#endif
}
size_t qlz_size_decompressed(const char *source)
{
ui32 n, r;
n = (((*source) & 2) == 2) ? 4 : 1;
r = fast_read(source + 1 + n, n);
r = r & (0xffffffff >> ((4 - n)*8));
return r;
}
size_t qlz_size_compressed(const char *source)
{
ui32 n, r;
n = (((*source) & 2) == 2) ? 4 : 1;
r = fast_read(source + 1, n);
r = r & (0xffffffff >> ((4 - n)*8));
return r;
}
size_t qlz_size_header(const char *source)
{
size_t n = 2*((((*source) & 2) == 2) ? 4 : 1) + 1;
return n;
}
static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n)
{
// Caution if modifying memcpy_up! Overlap of dst and src must be special handled.
#ifndef X86X64
unsigned char *end = dst + n;
while(dst < end)
{
*dst = *src;
dst++;
src++;
}
#else
ui32 f = 0;
do
{
*(ui32 *)(dst + f) = *(ui32 *)(src + f);
f += MINOFFSET + 1;
}
while (f < n);
#endif
}
static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s)
{
#if QLZ_COMPRESSION_LEVEL == 1
ui32 hash;
hash = hashat(s);
state->hash[hash].offset = s;
state->hash_counter[hash] = 1;
#elif QLZ_COMPRESSION_LEVEL == 2
ui32 hash;
unsigned char c;
hash = hashat(s);
c = state->hash_counter[hash];
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s;
c++;
state->hash_counter[hash] = c;
#endif
(void)state;
(void)s;
}
#if QLZ_COMPRESSION_LEVEL <= 2
static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max)
{
while(*lh < max)
{
(*lh)++;
update_hash(state, *lh);
}
}
#endif
static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state)
{
const unsigned char *last_byte = source + size - 1;
const unsigned char *src = source;
unsigned char *cword_ptr = destination;
unsigned char *dst = destination + CWORD_LEN;
ui32 cword_val = 1U << 31;
const unsigned char *last_matchstart = last_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END;
ui32 fetch = 0;
unsigned int lits = 0;
(void) lits;
if(src <= last_matchstart)
fetch = fast_read(src, 3);
while(src <= last_matchstart)
{
if ((cword_val & 1) == 1)
{
// store uncompressed if compression ratio is too low
if (src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5))
return 0;
fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
cword_ptr = dst;
dst += CWORD_LEN;
cword_val = 1U << 31;
fetch = fast_read(src, 3);
}
#if QLZ_COMPRESSION_LEVEL == 1
{
const unsigned char *o;
ui32 hash, cached;
hash = hash_func(fetch);
cached = fetch ^ state->hash[hash].cache;
state->hash[hash].cache = fetch;
o = state->hash[hash].offset + OFFSET_BASE;
state->hash[hash].offset = CAST(src - OFFSET_BASE);
#ifdef X86X64
if ((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
{
if(cached != 0)
{
#else
if (cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
{
if (*(o + 3) != *(src + 3))
{
#endif
hash <<= 4;
cword_val = (cword_val >> 1) | (1U << 31);
fast_write((3 - 2) | hash, dst, 2);
src += 3;
dst += 2;
}
else
{
const unsigned char *old_src = src;
size_t matchlen;
hash <<= 4;
cword_val = (cword_val >> 1) | (1U << 31);
src += 4;
if(*(o + (src - old_src)) == *src)
{
src++;
if(*(o + (src - old_src)) == *src)
{
size_t q = last_byte - UNCOMPRESSED_END - (src - 5) + 1;
size_t remaining = q > 255 ? 255 : q;
src++;
while(*(o + (src - old_src)) == *src && (size_t)(src - old_src) < remaining)
src++;
}
}
matchlen = src - old_src;
if (matchlen < 18)
{
fast_write((ui32)(matchlen - 2) | hash, dst, 2);
dst += 2;
}
else
{
fast_write((ui32)(matchlen << 16) | hash, dst, 3);
dst += 3;
}
}
fetch = fast_read(src, 3);
lits = 0;
}
else
{
lits++;
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
#ifdef X86X64
fetch = fast_read(src, 3);
#else
fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16);
#endif
}
}
#elif QLZ_COMPRESSION_LEVEL >= 2
{
const unsigned char *o, *offset2;
ui32 hash, matchlen, k, m, best_k = 0;
unsigned char c;
size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1);
(void)best_k;
//hash = hashat(src);
fetch = fast_read(src, 3);
hash = hash_func(fetch);
c = state->hash_counter[hash];
offset2 = state->hash[hash].offset[0];
if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0)
{
matchlen = 3;
if(*(offset2 + matchlen) == *(src + matchlen))
{
matchlen = 4;
while(*(offset2 + matchlen) == *(src + matchlen) && matchlen < remaining)
matchlen++;
}
}
else
matchlen = 0;
for(k = 1; k < QLZ_POINTERS && c > k; k++)
{
o = state->hash[hash].offset[k];
#if QLZ_COMPRESSION_LEVEL == 3
if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
#elif QLZ_COMPRESSION_LEVEL == 2
if(*(src + matchlen) == *(o + matchlen) && ((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
#endif
{
m = 3;
while(*(o + m) == *(src + m) && m < remaining)
m++;
#if QLZ_COMPRESSION_LEVEL == 3
if ((m > matchlen) || (m == matchlen && o > offset2))
#elif QLZ_COMPRESSION_LEVEL == 2
if (m > matchlen)
#endif
{
offset2 = o;
matchlen = m;
best_k = k;
}
}
}
o = offset2;
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
c++;
state->hash_counter[hash] = c;
#if QLZ_COMPRESSION_LEVEL == 3
if(matchlen > 2 && src - o < 131071)
{
ui32 u;
size_t offset = src - o;
for(u = 1; u < matchlen; u++)
{
hash = hashat(src + u);
c = state->hash_counter[hash]++;
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u;
}
cword_val = (cword_val >> 1) | (1U << 31);
src += matchlen;
if(matchlen == 3 && offset <= 63)
{
*dst = (unsigned char)(offset << 2);
dst++;
}
else if (matchlen == 3 && offset <= 16383)
{
ui32 f = (ui32)((offset << 2) | 1);
fast_write(f, dst, 2);
dst += 2;
}
else if (matchlen <= 18 && offset <= 1023)
{
ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2;
fast_write(f, dst, 2);
dst += 2;
}
else if(matchlen <= 33)
{
ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3;
fast_write(f, dst, 3);
dst += 3;
}
else
{
ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3;
fast_write(f, dst, 4);
dst += 4;
}
}
else
{
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
}
#elif QLZ_COMPRESSION_LEVEL == 2
if(matchlen > 2)
{
cword_val = (cword_val >> 1) | (1U << 31);
src += matchlen;
if (matchlen < 10)
{
ui32 f = best_k | ((matchlen - 2) << 2) | (hash << 5);
fast_write(f, dst, 2);
dst += 2;
}
else
{
ui32 f = best_k | (matchlen << 16) | (hash << 5);
fast_write(f, dst, 3);
dst += 3;
}
}
else
{
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
}
#endif
}
#endif
}
while (src <= last_byte)
{
if ((cword_val & 1) == 1)
{
fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
cword_ptr = dst;
dst += CWORD_LEN;
cword_val = 1U << 31;
}
#if QLZ_COMPRESSION_LEVEL < 3
if (src <= last_byte - 3)
{
#if QLZ_COMPRESSION_LEVEL == 1
ui32 hash, fetch;
fetch = fast_read(src, 3);
hash = hash_func(fetch);
state->hash[hash].offset = CAST(src - OFFSET_BASE);
state->hash[hash].cache = fetch;
#elif QLZ_COMPRESSION_LEVEL == 2
ui32 hash;
unsigned char c;
hash = hashat(src);
c = state->hash_counter[hash];
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
c++;
state->hash_counter[hash] = c;
#endif
}
#endif
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
}
while((cword_val & 1) != 1)
cword_val = (cword_val >> 1);
fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
// min. size must be 9 bytes so that the qlz_size functions can take 9 bytes as argument
return dst - destination < 9 ? 9 : dst - destination;
}
static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history)
{
const unsigned char *src = source + qlz_size_header((const char *)source);
unsigned char *dst = destination;
const unsigned char *last_destination_byte = destination + size - 1;
ui32 cword_val = 1;
const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END;
unsigned char *last_hashed = destination - 1;
const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1;
static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0};
(void) last_source_byte;
(void) last_hashed;
(void) state;
(void) history;
for(;;)
{
ui32 fetch;
if (cword_val == 1)
{
#ifdef QLZ_MEMORY_SAFE
if(src + CWORD_LEN - 1 > last_source_byte)
return 0;
#endif
cword_val = fast_read(src, CWORD_LEN);
src += CWORD_LEN;
}
#ifdef QLZ_MEMORY_SAFE
if(src + 4 - 1 > last_source_byte)
return 0;
#endif
fetch = fast_read(src, 4);
if ((cword_val & 1) == 1)
{
ui32 matchlen;
const unsigned char *offset2;
#if QLZ_COMPRESSION_LEVEL == 1
ui32 hash;
cword_val = cword_val >> 1;
hash = (fetch >> 4) & 0xfff;
offset2 = (const unsigned char *)(size_t)state->hash[hash].offset;
if((fetch & 0xf) != 0)
{
matchlen = (fetch & 0xf) + 2;
src += 2;
}
else
{
matchlen = *(src + 2);
src += 3;
}
#elif QLZ_COMPRESSION_LEVEL == 2
ui32 hash;
unsigned char c;
cword_val = cword_val >> 1;
hash = (fetch >> 5) & 0x7ff;
c = (unsigned char)(fetch & 0x3);
offset2 = state->hash[hash].offset[c];
if((fetch & (28)) != 0)
{
matchlen = ((fetch >> 2) & 0x7) + 2;
src += 2;
}
else
{
matchlen = *(src + 2);
src += 3;
}
#elif QLZ_COMPRESSION_LEVEL == 3
ui32 offset;
cword_val = cword_val >> 1;
if ((fetch & 3) == 0)
{
offset = (fetch & 0xff) >> 2;
matchlen = 3;
src++;
}
else if ((fetch & 2) == 0)
{
offset = (fetch & 0xffff) >> 2;
matchlen = 3;
src += 2;
}
else if ((fetch & 1) == 0)
{
offset = (fetch & 0xffff) >> 6;
matchlen = ((fetch >> 2) & 15) + 3;
src += 2;
}
else if ((fetch & 127) != 3)
{
offset = (fetch >> 7) & 0x1ffff;
matchlen = ((fetch >> 2) & 0x1f) + 2;
src += 3;
}
else
{
offset = (fetch >> 15);
matchlen = ((fetch >> 7) & 255) + 3;
src += 4;
}
offset2 = dst - offset;
#endif
#ifdef QLZ_MEMORY_SAFE
if(offset2 < history || offset2 > dst - MINOFFSET - 1)
return 0;
if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1))
return 0;
#endif
memcpy_up(dst, offset2, matchlen);
dst += matchlen;
#if QLZ_COMPRESSION_LEVEL <= 2
update_hash_upto(state, &last_hashed, dst - matchlen);
last_hashed = dst - 1;
#endif
}
else
{
if (dst < last_matchstart)
{
unsigned int n = bitlut[cword_val & 0xf];
#ifdef X86X64
*(ui32 *)dst = *(ui32 *)src;
#else
memcpy_up(dst, src, 4);
#endif
cword_val = cword_val >> n;
dst += n;
src += n;
#if QLZ_COMPRESSION_LEVEL <= 2
update_hash_upto(state, &last_hashed, dst - 3);
#endif
}
else
{
while(dst <= last_destination_byte)
{
if (cword_val == 1)
{
src += CWORD_LEN;
cword_val = 1U << 31;
}
#ifdef QLZ_MEMORY_SAFE
if(src >= last_source_byte + 1)
return 0;
#endif
*dst = *src;
dst++;
src++;
cword_val = cword_val >> 1;
}
#if QLZ_COMPRESSION_LEVEL <= 2
update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant
#endif
return size;
}
}
}
}
size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state)
{
size_t r;
ui32 compressed;
size_t base;
if(size == 0 || size > 0xffffffff - 400)
return 0;
if(size < 216)
base = 3;
else
base = 9;
#if QLZ_STREAMING_BUFFER > 0
if (state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER)
#endif
{
reset_table_compress(state);
r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, state);
#if QLZ_STREAMING_BUFFER > 0
reset_table_compress(state);
#endif
if(r == base)
{
memcpy(destination + base, source, size);
r = size + base;
compressed = 0;
}
else
{
compressed = 1;
}
state->stream_counter = 0;
}
#if QLZ_STREAMING_BUFFER > 0
else
{
unsigned char *src = state->stream_buffer + state->stream_counter;
memcpy(src, source, size);
r = base + qlz_compress_core(src, (unsigned char*)destination + base, size, state);
if(r == base)
{
memcpy(destination + base, src, size);
r = size + base;
compressed = 0;
reset_table_compress(state);
}
else
{
compressed = 1;
}
state->stream_counter += size;
}
#endif
if(base == 3)
{
*destination = (unsigned char)(0 | compressed);
*(destination + 1) = (unsigned char)r;
*(destination + 2) = (unsigned char)size;
}
else
{
*destination = (unsigned char)(2 | compressed);
fast_write((ui32)r, destination + 1, 4);
fast_write((ui32)size, destination + 5, 4);
}
*destination |= (QLZ_COMPRESSION_LEVEL << 2);
*destination |= (1 << 6);
*destination |= ((QLZ_STREAMING_BUFFER == 0 ? 0 : (QLZ_STREAMING_BUFFER == 100000 ? 1 : (QLZ_STREAMING_BUFFER == 1000000 ? 2 : 3))) << 4);
// 76543210
// 01SSLLHC
return r;
}
size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state)
{
size_t dsiz = qlz_size_decompressed(source);
#if QLZ_STREAMING_BUFFER > 0
if (state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER)
#endif
{
if((*source & 1) == 1)
{
reset_table_decompress(state);
dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination);
}
else
{
memcpy(destination, source + qlz_size_header(source), dsiz);
}
state->stream_counter = 0;
reset_table_decompress(state);
}
#if QLZ_STREAMING_BUFFER > 0
else
{
unsigned char *dst = state->stream_buffer + state->stream_counter;
if((*source & 1) == 1)
{
dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer);
}
else
{
memcpy(dst, source + qlz_size_header(source), dsiz);
reset_table_decompress(state);
}
memcpy(destination, dst, dsiz);
state->stream_counter += dsiz;
}
#endif
return dsiz;
}

843
src/qlz/QuickLZ_L3.cpp
View File

@ -1,843 +0,0 @@
// Fast data compression library
// Copyright (C) 2006-2011 Lasse Mikkel Reinhold
// lar@quicklz.com
//
// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything
// released into public must be open source) or under a commercial license if such
// has been acquired (see http://www.quicklz.com/order.html). The commercial license
// does not cover derived or ported versions created by third parties under GPL.
// 1.5.0 final
#define QLZ_COMPRESSION_LEVEL 3
#include "QuickLZ.h"
#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0
#error quicklz.c and quicklz.h have different versions
#endif
#if (defined(__X86__) || defined(__i386__) || defined(i386) || defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || defined(_M_X64))
#define X86X64
#endif
#define MINOFFSET 2
#define UNCONDITIONAL_MATCHLEN 6
#define UNCOMPRESSED_END 4
#define CWORD_LEN 4
#if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0
#define OFFSET_BASE source
#define CAST (ui32)(size_t)
#else
#define OFFSET_BASE 0
#define CAST
#endif
int qlz_get_setting(int setting)
{
switch (setting)
{
case 0: return QLZ_COMPRESSION_LEVEL;
case 1: return sizeof(qlz_state_compress);
case 2: return sizeof(qlz_state_decompress);
case 3: return QLZ_STREAMING_BUFFER;
#ifdef QLZ_MEMORY_SAFE
case 6: return 1;
#else
case 6: return 0;
#endif
case 7: return QLZ_VERSION_MAJOR;
case 8: return QLZ_VERSION_MINOR;
case 9: return QLZ_VERSION_REVISION;
}
return -1;
}
#if QLZ_COMPRESSION_LEVEL == 1
static int same(const unsigned char *src, size_t n)
{
while(n > 0 && *(src + n) == *src)
n--;
return n == 0 ? 1 : 0;
}
#endif
static void reset_table_compress(qlz_state_compress *state)
{
int i;
for(i = 0; i < QLZ_HASH_VALUES; i++)
{
#if QLZ_COMPRESSION_LEVEL == 1
state->hash[i].offset = 0;
#else
state->hash_counter[i] = 0;
#endif
}
}
static void reset_table_decompress(qlz_state_decompress *state)
{
int i;
(void)state;
(void)i;
#if QLZ_COMPRESSION_LEVEL == 2
for(i = 0; i < QLZ_HASH_VALUES; i++)
{
state->hash_counter[i] = 0;
}
#endif
}
static __inline ui32 hash_func(ui32 i)
{
#if QLZ_COMPRESSION_LEVEL == 2
return ((i >> 9) ^ (i >> 13) ^ i) & (QLZ_HASH_VALUES - 1);
#else
return ((i >> 12) ^ i) & (QLZ_HASH_VALUES - 1);
#endif
}
static __inline ui32 fast_read(void const *src, ui32 bytes)
{
#ifndef X86X64
unsigned char *p = (unsigned char*)src;
switch (bytes)
{
case 4:
return(*p | *(p + 1) << 8 | *(p + 2) << 16 | *(p + 3) << 24);
case 3:
return(*p | *(p + 1) << 8 | *(p + 2) << 16);
case 2:
return(*p | *(p + 1) << 8);
case 1:
return(*p);
}
return 0;
#else
if (bytes >= 1 && bytes <= 4)
return *((ui32*)src);
else
return 0;
#endif
}
static __inline ui32 hashat(const unsigned char *src)
{
ui32 fetch, hash;
fetch = fast_read(src, 3);
hash = hash_func(fetch);
return hash;
}
static __inline void fast_write(ui32 f, void *dst, size_t bytes)
{
#ifndef X86X64
unsigned char *p = (unsigned char*)dst;
switch (bytes)
{
case 4:
*p = (unsigned char)f;
*(p + 1) = (unsigned char)(f >> 8);
*(p + 2) = (unsigned char)(f >> 16);
*(p + 3) = (unsigned char)(f >> 24);
return;
case 3:
*p = (unsigned char)f;
*(p + 1) = (unsigned char)(f >> 8);
*(p + 2) = (unsigned char)(f >> 16);
return;
case 2:
*p = (unsigned char)f;
*(p + 1) = (unsigned char)(f >> 8);
return;
case 1:
*p = (unsigned char)f;
return;
}
#else
switch (bytes)
{
case 4:
*((ui32*)dst) = f;
return;
case 3:
*((ui32*)dst) = f;
return;
case 2:
*((ui16 *)dst) = (ui16)f;
return;
case 1:
*((unsigned char*)dst) = (unsigned char)f;
return;
}
#endif
}
size_t qlz_size_decompressed(const char *source)
{
ui32 n, r;
n = (((*source) & 2) == 2) ? 4 : 1;
r = fast_read(source + 1 + n, n);
r = r & (0xffffffff >> ((4 - n)*8));
return r;
}
size_t qlz_size_compressed(const char *source)
{
ui32 n, r;
n = (((*source) & 2) == 2) ? 4 : 1;
r = fast_read(source + 1, n);
r = r & (0xffffffff >> ((4 - n)*8));
return r;
}
static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n)
{
// Caution if modifying memcpy_up! Overlap of dst and src must be special handled.
#ifndef X86X64
unsigned char *end = dst + n;
while(dst < end)
{
*dst = *src;
dst++;
src++;
}
#else
ui32 f = 0;
do
{
*(ui32 *)(dst + f) = *(ui32 *)(src + f);
f += MINOFFSET + 1;
}
while (f < n);
#endif
}
static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s)
{
#if QLZ_COMPRESSION_LEVEL == 1
ui32 hash;
hash = hashat(s);
state->hash[hash].offset = s;
state->hash_counter[hash] = 1;
#elif QLZ_COMPRESSION_LEVEL == 2
ui32 hash;
unsigned char c;
hash = hashat(s);
c = state->hash_counter[hash];
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s;
c++;
state->hash_counter[hash] = c;
#endif
(void)state;
(void)s;
}
#if QLZ_COMPRESSION_LEVEL <= 2
static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max)
{
while(*lh < max)
{
(*lh)++;
update_hash(state, *lh);
}
}
#endif
static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state)
{
const unsigned char *last_byte = source + size - 1;
const unsigned char *src = source;
unsigned char *cword_ptr = destination;
unsigned char *dst = destination + CWORD_LEN;
ui32 cword_val = 1U << 31;
const unsigned char *last_matchstart = last_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END;
ui32 fetch = 0;
unsigned int lits = 0;
(void) lits;
if(src <= last_matchstart)
fetch = fast_read(src, 3);
while(src <= last_matchstart)
{
if ((cword_val & 1) == 1)
{
// store uncompressed if compression ratio is too low
if (src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5))
return 0;
fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
cword_ptr = dst;
dst += CWORD_LEN;
cword_val = 1U << 31;
fetch = fast_read(src, 3);
}
#if QLZ_COMPRESSION_LEVEL == 1
{
const unsigned char *o;
ui32 hash, cached;
hash = hash_func(fetch);
cached = fetch ^ state->hash[hash].cache;
state->hash[hash].cache = fetch;
o = state->hash[hash].offset + OFFSET_BASE;
state->hash[hash].offset = CAST(src - OFFSET_BASE);
#ifdef X86X64
if ((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
{
if(cached != 0)
{
#else
if (cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
{
if (*(o + 3) != *(src + 3))
{
#endif
hash <<= 4;
cword_val = (cword_val >> 1) | (1U << 31);
fast_write((3 - 2) | hash, dst, 2);
src += 3;
dst += 2;
}
else
{
const unsigned char *old_src = src;
size_t matchlen;
hash <<= 4;
cword_val = (cword_val >> 1) | (1U << 31);
src += 4;
if(*(o + (src - old_src)) == *src)
{
src++;
if(*(o + (src - old_src)) == *src)
{
size_t q = last_byte - UNCOMPRESSED_END - (src - 5) + 1;
size_t remaining = q > 255 ? 255 : q;
src++;
while(*(o + (src - old_src)) == *src && (size_t)(src - old_src) < remaining)
src++;
}
}
matchlen = src - old_src;
if (matchlen < 18)
{
fast_write((ui32)(matchlen - 2) | hash, dst, 2);
dst += 2;
}
else
{
fast_write((ui32)(matchlen << 16) | hash, dst, 3);
dst += 3;
}
}
fetch = fast_read(src, 3);
lits = 0;
}
else
{
lits++;
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
#ifdef X86X64
fetch = fast_read(src, 3);
#else
fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16);
#endif
}
}
#elif QLZ_COMPRESSION_LEVEL >= 2
{
const unsigned char *o, *offset2;
ui32 hash, matchlen, k, m, best_k = 0;
unsigned char c;
size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1);
(void)best_k;
//hash = hashat(src);
fetch = fast_read(src, 3);
hash = hash_func(fetch);
c = state->hash_counter[hash];
offset2 = state->hash[hash].offset[0];
if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0)
{
matchlen = 3;
if(*(offset2 + matchlen) == *(src + matchlen))
{
matchlen = 4;
while(*(offset2 + matchlen) == *(src + matchlen) && matchlen < remaining)
matchlen++;
}
}
else
matchlen = 0;
for(k = 1; k < QLZ_POINTERS && c > k; k++)
{
o = state->hash[hash].offset[k];
#if QLZ_COMPRESSION_LEVEL == 3
if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
#elif QLZ_COMPRESSION_LEVEL == 2
if(*(src + matchlen) == *(o + matchlen) && ((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
#endif
{
m = 3;
while(*(o + m) == *(src + m) && m < remaining)
m++;
#if QLZ_COMPRESSION_LEVEL == 3
if ((m > matchlen) || (m == matchlen && o > offset2))
#elif QLZ_COMPRESSION_LEVEL == 2
if (m > matchlen)
#endif
{
offset2 = o;
matchlen = m;
best_k = k;
}
}
}
o = offset2;
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
c++;
state->hash_counter[hash] = c;
#if QLZ_COMPRESSION_LEVEL == 3
if(matchlen > 2 && src - o < 131071)
{
ui32 u;
size_t offset = src - o;
for(u = 1; u < matchlen; u++)
{
hash = hashat(src + u);
c = state->hash_counter[hash]++;
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u;
}
cword_val = (cword_val >> 1) | (1U << 31);
src += matchlen;
if(matchlen == 3 && offset <= 63)
{
*dst = (unsigned char)(offset << 2);
dst++;
}
else if (matchlen == 3 && offset <= 16383)
{
ui32 f = (ui32)((offset << 2) | 1);
fast_write(f, dst, 2);
dst += 2;
}
else if (matchlen <= 18 && offset <= 1023)
{
ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2;
fast_write(f, dst, 2);
dst += 2;
}
else if(matchlen <= 33)
{
ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3;
fast_write(f, dst, 3);
dst += 3;
}
else
{
ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3;
fast_write(f, dst, 4);
dst += 4;
}
}
else
{
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
}
#elif QLZ_COMPRESSION_LEVEL == 2
if(matchlen > 2)
{
cword_val = (cword_val >> 1) | (1U << 31);
src += matchlen;
if (matchlen < 10)
{
ui32 f = best_k | ((matchlen - 2) << 2) | (hash << 5);
fast_write(f, dst, 2);
dst += 2;
}
else
{
ui32 f = best_k | (matchlen << 16) | (hash << 5);
fast_write(f, dst, 3);
dst += 3;
}
}
else
{
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
}
#endif
}
#endif
}
while (src <= last_byte)
{
if ((cword_val & 1) == 1)
{
fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
cword_ptr = dst;
dst += CWORD_LEN;
cword_val = 1U << 31;
}
#if QLZ_COMPRESSION_LEVEL < 3
if (src <= last_byte - 3)
{
#if QLZ_COMPRESSION_LEVEL == 1
ui32 hash, fetch;
fetch = fast_read(src, 3);
hash = hash_func(fetch);
state->hash[hash].offset = CAST(src - OFFSET_BASE);
state->hash[hash].cache = fetch;
#elif QLZ_COMPRESSION_LEVEL == 2
ui32 hash;
unsigned char c;
hash = hashat(src);
c = state->hash_counter[hash];
state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
c++;
state->hash_counter[hash] = c;
#endif
}
#endif
*dst = *src;
src++;
dst++;
cword_val = (cword_val >> 1);
}
while((cword_val & 1) != 1)
cword_val = (cword_val >> 1);
fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
// min. size must be 9 bytes so that the qlz_size functions can take 9 bytes as argument
return dst - destination < 9 ? 9 : dst - destination;
}
extern size_t qlz_size_header(const char *source);
static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history)
{
const unsigned char *src = source + qlz_size_header((const char *)source);
unsigned char *dst = destination;
const unsigned char *last_destination_byte = destination + size - 1;
ui32 cword_val = 1;
const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END;
unsigned char *last_hashed = destination - 1;
const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1;
static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0};
(void) last_source_byte;
(void) last_hashed;
(void) state;
(void) history;
for(;;)
{
ui32 fetch;
if (cword_val == 1)
{
#ifdef QLZ_MEMORY_SAFE
if(src + CWORD_LEN - 1 > last_source_byte)
return 0;
#endif
cword_val = fast_read(src, CWORD_LEN);
src += CWORD_LEN;
}
#ifdef QLZ_MEMORY_SAFE
if(src + 4 - 1 > last_source_byte)
return 0;
#endif
fetch = fast_read(src, 4);
if ((cword_val & 1) == 1)
{
ui32 matchlen;
const unsigned char *offset2;
#if QLZ_COMPRESSION_LEVEL == 1
ui32 hash;
cword_val = cword_val >> 1;
hash = (fetch >> 4) & 0xfff;
offset2 = (const unsigned char *)(size_t)state->hash[hash].offset;
if((fetch & 0xf) != 0)
{
matchlen = (fetch & 0xf) + 2;
src += 2;
}
else
{
matchlen = *(src + 2);
src += 3;
}
#elif QLZ_COMPRESSION_LEVEL == 2
ui32 hash;
unsigned char c;
cword_val = cword_val >> 1;
hash = (fetch >> 5) & 0x7ff;
c = (unsigned char)(fetch & 0x3);
offset2 = state->hash[hash].offset[c];
if((fetch & (28)) != 0)
{
matchlen = ((fetch >> 2) & 0x7) + 2;
src += 2;
}
else
{
matchlen = *(src + 2);
src += 3;
}
#elif QLZ_COMPRESSION_LEVEL == 3
ui32 offset;
cword_val = cword_val >> 1;
if ((fetch & 3) == 0)
{
offset = (fetch & 0xff) >> 2;
matchlen = 3;
src++;
}
else if ((fetch & 2) == 0)
{
offset = (fetch & 0xffff) >> 2;
matchlen = 3;
src += 2;
}
else if ((fetch & 1) == 0)
{
offset = (fetch & 0xffff) >> 6;
matchlen = ((fetch >> 2) & 15) + 3;
src += 2;
}
else if ((fetch & 127) != 3)
{
offset = (fetch >> 7) & 0x1ffff;
matchlen = ((fetch >> 2) & 0x1f) + 2;
src += 3;
}
else
{
offset = (fetch >> 15);
matchlen = ((fetch >> 7) & 255) + 3;
src += 4;
}
offset2 = dst - offset;
#endif
#ifdef QLZ_MEMORY_SAFE
if(offset2 < history || offset2 > dst - MINOFFSET - 1)
return 0;
if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1))
return 0;
#endif
memcpy_up(dst, offset2, matchlen);
dst += matchlen;
#if QLZ_COMPRESSION_LEVEL <= 2
update_hash_upto(state, &last_hashed, dst - matchlen);
last_hashed = dst - 1;
#endif
}
else
{
if (dst < last_matchstart)
{
unsigned int n = bitlut[cword_val & 0xf];
#ifdef X86X64
*(ui32 *)dst = *(ui32 *)src;
#else
memcpy_up(dst, src, 4);
#endif
cword_val = cword_val >> n;
dst += n;
src += n;
#if QLZ_COMPRESSION_LEVEL <= 2
update_hash_upto(state, &last_hashed, dst - 3);
#endif
}
else
{
while(dst <= last_destination_byte)
{
if (cword_val == 1)
{
src += CWORD_LEN;
cword_val = 1U << 31;
}
#ifdef QLZ_MEMORY_SAFE
if(src >= last_source_byte + 1)
return 0;
#endif
*dst = *src;
dst++;
src++;
cword_val = cword_val >> 1;
}
#if QLZ_COMPRESSION_LEVEL <= 2
update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant
#endif
return size;
}
}
}
}
size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state)
{
size_t r;
ui32 compressed;
size_t base;
if(size == 0 || size > 0xffffffff - 400)
return 0;
if(size < 216)
base = 3;
else
base = 9;
#if QLZ_STREAMING_BUFFER > 0
if (state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER)
#endif
{
reset_table_compress(state);
r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, state);
#if QLZ_STREAMING_BUFFER > 0
reset_table_compress(state);
#endif
if(r == base)
{
memcpy(destination + base, source, size);
r = size + base;
compressed = 0;
}
else
{
compressed = 1;
}
state->stream_counter = 0;
}
#if QLZ_STREAMING_BUFFER > 0
else
{
unsigned char *src = state->stream_buffer + state->stream_counter;
memcpy(src, source, size);
r = base + qlz_compress_core(src, (unsigned char*)destination + base, size, state);
if(r == base)
{
memcpy(destination + base, src, size);
r = size + base;
compressed = 0;
reset_table_compress(state);
}
else
{
compressed = 1;
}
state->stream_counter += size;
}
#endif
if(base == 3)
{
*destination = (unsigned char)(0 | compressed);
*(destination + 1) = (unsigned char)r;
*(destination + 2) = (unsigned char)size;
}
else
{
*destination = (unsigned char)(2 | compressed);
fast_write((ui32)r, destination + 1, 4);
fast_write((ui32)size, destination + 5, 4);
}
*destination |= (QLZ_COMPRESSION_LEVEL << 2);
*destination |= (1 << 6);
*destination |= ((QLZ_STREAMING_BUFFER == 0 ? 0 : (QLZ_STREAMING_BUFFER == 100000 ? 1 : (QLZ_STREAMING_BUFFER == 1000000 ? 2 : 3))) << 4);
// 76543210
// 01SSLLHC
return r;
}
size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state)
{
size_t dsiz = qlz_size_decompressed(source);
#if QLZ_STREAMING_BUFFER > 0
if (state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER)
#endif
{
if((*source & 1) == 1)
{
reset_table_decompress(state);
dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination);
}
else
{
memcpy(destination, source + qlz_size_header(source), dsiz);
}
state->stream_counter = 0;
reset_table_decompress(state);
}
#if QLZ_STREAMING_BUFFER > 0
else
{
unsigned char *dst = state->stream_buffer + state->stream_counter;
if((*source & 1) == 1)
{
dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer);
}
else
{
memcpy(dst, source + qlz_size_header(source), dsiz);
reset_table_decompress(state);
}
memcpy(destination, dst, dsiz);
state->stream_counter += dsiz;
}
#endif
return dsiz;
}

6
test/generationTest.cpp
View File

@ -34,7 +34,7 @@ test_vector_t swap_elements(test_vector_t vector, int per, int max_distance) {
}
bool test_vector(const std::string_view& name, const test_vector_t& vector) {
generation_estimator gen{};
GenerationEstimator gen{};
size_t last_value{0}, last_gen{0}, index{0};
for(auto [id, exp] : vector) {
@ -50,7 +50,7 @@ bool test_vector(const std::string_view& name, const test_vector_t& vector) {
}
template <size_t N>
bool test_vector(generation_estimator& generator, const std::array<uint16_t, N>& packet_ids, const std::array<uint16_t, N>& expected) {
bool test_vector(GenerationEstimator& generator, const std::array<uint16_t, N>& packet_ids, const std::array<uint16_t, N>& expected) {
for(size_t index = 0; index < N; index++) {
auto result = generator.visit_packet(packet_ids[index]);
if(result != expected[index]) {
@ -67,7 +67,7 @@ bool test_vector(generation_estimator& generator, const std::array<uint16_t, N>&
}
int main() {
generation_estimator gen{};
GenerationEstimator gen{};
{
test_vector("00 loss", generate_test_vector(0x30000, 0));