Changed some stuff

This commit is contained in:
WolverinDEV 2020-03-09 18:28:44 +01:00
parent 2033db5ed6
commit 9533fe8920
5 changed files with 412 additions and 98 deletions

View File

@ -290,7 +290,7 @@ if(BUILD_TESTS)
add_executable(PorpertyTest test/PropertyTest.cpp ${SOURCE_FILES})
target_link_libraries(PorpertyTest ${TEST_LIBRARIES})
add_executable(BBTest test/BBTest.cpp ${SOURCE_FILES} src/query/command_unused.h)
add_executable(BBTest test/BBTest.cpp ${SOURCE_FILES} src/query/command_unused.h src/misc/ip_router.cpp src/misc/ip_router.h)
target_link_libraries(BBTest ${TEST_LIBRARIES})
add_executable(LinkedTest test/LinkedTest.cpp ${SOURCE_FILES})
@ -304,5 +304,7 @@ if(BUILD_TESTS)
add_executable(RW-Lock-Test test/rw_lock.cpp src/lock/rw_mutex.cpp)
target_link_libraries(GenerationTest ${TEST_LIBRARIES})
add_executable(IP-Router-Test test/ip_router.cpp src/misc/ip_router.cpp)
endif()
endif()

View File

@ -76,11 +76,12 @@ namespace ts {
struct converter<class> { \
static constexpr bool supported = true; \
\
static constexpr std::string(*to_string)(const std::any&) = [](const std::any& val) { \
return std::to_string(std::any_cast<class>(val)); \
}; \
static constexpr class(*from_string_view)(const std::string_view&) = [](const std::string_view& val) { \
return ((class(*)(const std::string_view&)) ts::converter<size_type>::from_string_view)(val); \
static constexpr std::string(*to_string)(const std::any&) = [](const std::any& val) { \
return std::to_string((size_type) std::any_cast<class>(val)); \
}; \
\
static constexpr class(*from_string_view)(const std::string_view&) = [](const std::string_view& val) { \
return ((class(*)(const std::string_view&)) ts::converter<size_type>::from_string_view)(val); \
}; \
}; \
}

View File

@ -10,31 +10,26 @@
using namespace ts::network;
constexpr static ip_rounter::route_entry generate_empty_end_node(void*) {
ip_rounter::route_entry result{};
constexpr static ip_router::route_entry generate_empty_end_node(void*) {
ip_router::route_entry result{};
for(auto& ptr : result.data)
ptr = nullptr;
return result;
}
template <size_t N>
constexpr std::array<ip_rounter::route_entry, N> generate_default_table() noexcept {
std::array<ip_rounter::route_entry, N> result{};
constexpr std::array<ip_router::route_entry, N> generate_default_table() noexcept {
std::array<ip_router::route_entry, N> result{};
for(ip_rounter::route_entry& entry : result)
entry.use_count = ip_rounter::route_entry::const_flag_mask | 0xFFU;
for(auto& end_ptr : result[0].data)
end_ptr = nullptr;
for(size_t index{1}; index < result.size(); index++)
for(auto& ptr : result[index].data)
ptr = &result[index - 1];
for(size_t index{0}; index < result.size(); index++) {
result[index].use_count = ip_router::route_entry::const_flag_mask | 0xFFU;
result[index].deep = index + 1;
}
return result;
}
std::array<ip_rounter::route_entry, 16> ip_rounter::recursive_ends = generate_default_table<16>();
std::array<ip_router::route_entry, 16> ip_router::recursive_ends = generate_default_table<16>();
struct sockaddr_storage_info {
size_t address_offset{0};
@ -88,33 +83,35 @@ inline void address_to_chunks(uint8_t* chunks, const sockaddr_storage &address)
}
}
ip_rounter::ip_rounter() {
ip_router::ip_router() {
for(auto& data : this->root_entry.data)
data = &ip_rounter::recursive_ends[14];
this->root_entry.use_count = ip_rounter::route_entry::const_flag_mask | 0xFFU;
data = &ip_router::recursive_ends[14];
this->root_entry.deep = 1;
this->root_entry.use_count = ip_router::route_entry::const_flag_mask | 0xFFU;
}
inline void delete_route_entry(ip_rounter::route_entry* entry, size_t level) {
inline void delete_route_entry(ip_router::route_entry* entry, size_t level) {
level -= entry->deep;
if(level != 0) {
for(auto& data : entry->data) {
auto e = (ip_rounter::route_entry*) data;
auto e = (ip_router::route_entry*) data;
if(e->is_const_entry()) continue;
delete_route_entry(e, level - 1);
delete_route_entry(e, level);
}
}
delete entry;
}
ip_rounter::~ip_rounter() {
ip_router::~ip_router() {
for(auto& entry : this->unused_nodes)
delete entry;
for(auto& data : this->root_entry.data) {
auto entry = (ip_rounter::route_entry*) data;
auto entry = (ip_router::route_entry*) data;
if(entry->is_const_entry()) continue;
delete_route_entry(entry, 14);
delete_route_entry(entry, 16 - this->root_entry.deep);
}
}
@ -122,22 +119,30 @@ ip_rounter::~ip_rounter() {
* Because we're only reading memory here, and that even quite fast we do not need to lock the register lock.
* Even if a block gets changed, it will not be deleted immediately. So we should finish reading first before that memory get freed.
*/
void* ip_rounter::resolve(const sockaddr_storage &address) const {
void* ip_router::resolve(const sockaddr_storage &address) const {
uint8_t address_chunks[16];
address_to_chunks(address_chunks, address);
const ip_rounter::route_entry* current_chunk = &this->root_entry;
const ip_router::route_entry* current_chunk = &this->root_entry;
std::lock_guard lock{this->entry_lock};
//std::shared_lock lock{this->entry_lock};
#pragma GCC unroll 15
for(size_t index{0}; index < 15; index++)
current_chunk = (ip_rounter::route_entry*) current_chunk->data[address_chunks[index]];
size_t byte_index{0};
while(true) {
byte_index += current_chunk->deep;
if(byte_index == 16) break;
assert(byte_index < 16);
current_chunk = (ip_router::route_entry*) current_chunk->data[address_chunks[byte_index - 1]];
};
if(memcmp(address_chunks, current_chunk->previous_chunks, 15) != 0)
return nullptr; /* route does not match */
return current_chunk->data[address_chunks[15]];
}
bool ip_rounter::register_route(const sockaddr_storage &address, void *target, void ** old_target) {
bool ip_router::register_route(const sockaddr_storage &address, void *target, void ** old_target) {
uint8_t address_chunks[16];
address_to_chunks(address_chunks, address);
@ -145,20 +150,61 @@ bool ip_rounter::register_route(const sockaddr_storage &address, void *target, v
if(!old_target)
old_target = &_temp_old_target;
ip_rounter::route_entry* current_chunk = &this->root_entry;
ip_router::route_entry* current_chunk = &this->root_entry;
std::lock_guard rlock{this->register_lock};
for(size_t index{0}; index < 15; index++) {
auto& next_chunk = (ip_rounter::route_entry*&) current_chunk->data[address_chunks[index]];
if(next_chunk->is_const_entry()) {
assert(next_chunk == &ip_rounter::recursive_ends[15 - index - 1]);
size_t byte_index{0};
while(true) {
byte_index += current_chunk->deep;
if(byte_index == 16) break;
assert(byte_index < 16);
auto allocated_entry = this->create_8bit_entry(15 - index - 1);
/* for the first iteration no previous_chunks check for "current_chunk" is needed because it will always match! */
auto& next_chunk = (ip_router::route_entry*&) current_chunk->data[address_chunks[byte_index - 1]];
if(next_chunk->is_const_entry()) {
/* perfect, lets allocate our own end and we're done */
//assert(next_chunk == &ip_rounter::recursive_ends[15 - index - 1]);
auto allocated_entry = this->create_8bit_entry(byte_index, true);
if(!allocated_entry) return false;
memcpy(allocated_entry->previous_chunks, address_chunks, 15);
/* no lock needed here, just a pointer exchange */
next_chunk = allocated_entry;
current_chunk->use_count++;
current_chunk = next_chunk;
break; /* end chunk now */
} else if(next_chunk->deep > 1) {
ssize_t unmatch_index{-1};
for(size_t i{0}; i < next_chunk->deep - 1; i++) {
if(next_chunk->previous_chunks[byte_index + i] != address_chunks[byte_index + i]) {
unmatch_index = i;
break;
}
}
if(unmatch_index >= 0) {
auto allocated_entry = this->create_8bit_entry(byte_index + unmatch_index, false);
if(!allocated_entry) return false;
allocated_entry->deep = unmatch_index + 1;
allocated_entry->use_count++;
allocated_entry->data[next_chunk->previous_chunks[byte_index + unmatch_index]] = next_chunk;
memcpy(allocated_entry->previous_chunks, address_chunks, 15);
{
std::lock_guard elock{this->entry_lock};
next_chunk->deep = next_chunk->deep - unmatch_index - 1;
next_chunk = allocated_entry;
}
current_chunk = next_chunk;
continue;
} else {
/* every bit matched we also have this nice jump */
}
}
current_chunk = next_chunk;
}
@ -167,119 +213,203 @@ bool ip_rounter::register_route(const sockaddr_storage &address, void *target, v
return true;
}
ip_rounter::route_entry *ip_rounter::create_8bit_entry(size_t level) {
ip_rounter::route_entry *result;
ip_router::route_entry *ip_router::create_8bit_entry(size_t level, bool end_entry) {
ip_router::route_entry *result;
if(this->unused_nodes.empty())
result = new ip_rounter::route_entry{};
result = new ip_router::route_entry{};
else {
result = this->unused_nodes.front();
this->unused_nodes.pop_front();
}
result->use_count = 0;
auto target = level == 0 ? nullptr : &ip_rounter::recursive_ends[level - 1];
for(auto& data : result->data)
data = target;
if(end_entry) {
/* this is an end chunk now */
result->deep = 16 - level;
for(auto& data : result->data)
data = nullptr;
} else {
assert(level <= 14);
result->deep = 1;
auto pointer = &ip_router::recursive_ends[15 - level - 1];
for(auto& data : result->data)
data = pointer;
}
return result;
}
void *ip_rounter::reset_route(const sockaddr_storage &address) {
void *ip_router::reset_route(const sockaddr_storage &address) {
uint8_t address_chunks[16];
address_to_chunks(address_chunks, address);
ip_rounter::route_entry* current_chunk = &this->root_entry;
ip_router::route_entry* current_chunk{&this->root_entry};
std::lock_guard rlock{this->register_lock};
for(size_t index{0}; index < 15; index++) {
current_chunk = (ip_rounter::route_entry *) current_chunk->data[address_chunks[index]];
if(current_chunk->is_const_entry()) return nullptr; /* route does not exists */
{
size_t byte_index{0};
while(true) {
byte_index += current_chunk->deep;
if(byte_index == 16) break;
assert(byte_index < 16);
current_chunk = (ip_router::route_entry*) current_chunk->data[address_chunks[byte_index - 1]];
if(current_chunk->is_const_entry()) return nullptr;
};
if(memcmp(address_chunks, current_chunk->previous_chunks, 15) != 0)
return nullptr; /* route does not match */
}
auto old = std::exchange(current_chunk->data[address_chunks[15]], nullptr);
if(!old) return nullptr; /* route does not exists */
auto old = current_chunk->data[address_chunks[15]];
if(!old) return nullptr;
if(--current_chunk->use_count == 0) {
size_t chunk_index{14};
do {
while(true) {
size_t byte_index{0};
current_chunk = &this->root_entry;
for(size_t index{0}; index < chunk_index; index++) {
current_chunk = (ip_rounter::route_entry *) current_chunk->data[address_chunks[index]];
if(current_chunk->is_const_entry()) return nullptr; /* route does not exists */
while(true) {
byte_index += current_chunk->deep;
if(byte_index == 16) break;
assert(byte_index < 16);
auto& next_chunk = (ip_router::route_entry*&) current_chunk->data[address_chunks[byte_index - 1]];
if(next_chunk->deep + byte_index == 16) {
assert(next_chunk->use_count == 0);
this->unused_nodes.push_back(next_chunk);
/* this is the last chunk */
next_chunk = &ip_router::recursive_ends[15 - byte_index];
if(--current_chunk->use_count > 0) goto exit_delete_loop;
}
current_chunk = next_chunk;
}
}
auto& chunk = (ip_rounter::route_entry *&) current_chunk->data[address_chunks[chunk_index]];
assert(!chunk->is_const_entry());
assert(chunk->use_count == 0); /* already tested earlier in theory */
this->unused_nodes.push_back(chunk);
chunk = &ip_rounter::recursive_ends[15 - chunk_index - 1];
if(--current_chunk->use_count > 0) break;
} while(--chunk_index > 0);
exit_delete_loop:;
}
return old;
}
void ip_rounter::cleanup_cache() {
void ip_router::cleanup_cache() {
std::lock_guard rlock{this->register_lock};
for(auto node : this->unused_nodes)
delete node;
this->unused_nodes.clear();
}
bool ip_rounter::validate_chunk_entry(const ip_rounter::route_entry* current_entry, size_t level) const {
if(current_entry->is_const_entry() && level != 15) /* level 15 is the default root node which is const as well */
return current_entry == &ip_rounter::recursive_ends[level];
bool ip_router::validate_chunk_entry(const ip_router::route_entry* current_entry, size_t level) const {
if(level == 0)
return true;
if(level == 0) return true;
if(current_entry->is_const_entry() && level != 16)
return level == current_entry->deep;
auto default_pointer = &ip_rounter::recursive_ends[level - 1];
auto default_pointer = &ip_router::recursive_ends[level - 1];
for(const auto& data_ptr : current_entry->data) {
if(data_ptr == default_pointer) continue;
if(!this->validate_chunk_entry((const ip_rounter::route_entry*) data_ptr, level - 1))
if(data_ptr == default_pointer)
continue;
if(!this->validate_chunk_entry((const ip_router::route_entry*) data_ptr, level - current_entry->deep))
return false;
}
return true;
}
bool ip_rounter::validate_tree() const {
bool ip_router::validate_tree() const {
std::lock_guard rlock{this->register_lock};
/* first lets validate all const chunks */
for(size_t index{0}; index < 16; index++) {
auto expected_pointer = index == 0 ? nullptr : &ip_rounter::recursive_ends[index - 1];
if(!ip_rounter::recursive_ends[index].is_const_entry())
if(!ip_router::recursive_ends[index].is_const_entry())
return false;
for(const auto& data_ptr : ip_rounter::recursive_ends[index].data)
if(data_ptr != expected_pointer)
if(ip_router::recursive_ends[index].deep != index + 1)
return false;
for(const auto& data_ptr : ip_router::recursive_ends[index].data)
if(data_ptr)
return false;
}
/* not lets check our tree */
return this->validate_chunk_entry(&this->root_entry, 15);
return this->validate_chunk_entry(&this->root_entry, 16);
}
size_t ip_rounter::chunk_memory(const ip_rounter::route_entry *current_entry, size_t level) const {
size_t result{sizeof(ip_rounter::route_entry)};
size_t ip_router::chunk_memory(const ip_router::route_entry *current_entry, size_t level) const {
size_t result{sizeof(ip_router::route_entry)};
level -= current_entry->deep;
if(level > 0) {
for(const auto& data_ptr : current_entry->data) {
auto entry = (const ip_rounter::route_entry*) data_ptr;
auto entry = (const ip_router::route_entry*) data_ptr;
if(entry->is_const_entry()) continue;
result += chunk_memory(entry, level - 1);
result += chunk_memory(entry, level);
}
}
return result;
}
size_t ip_rounter::used_memory() const {
return this->chunk_memory(&this->root_entry, 15);
size_t ip_router::used_memory() const {
return this->chunk_memory(&this->root_entry, 16);
}
std::string ip_router::print_as_string() const {
std::string result{};
this->print_as_string(result, "", &this->root_entry, 16);
result += "Memory used: " + std::to_string(this->used_memory() / 1024) + "kb";
return result;
}
template <typename I>
std::string n2hexstr(I w, size_t hex_len = sizeof(I)<<1) {
if(w == 0) return "0x0";
static const char* digits = "0123456789ABCDEF";
std::string rc(hex_len,'0');
for (size_t i=0, j=(hex_len-1)*4 ; i<hex_len; ++i,j-=4)
rc[i] = digits[(w >> j) & 0x0f];
size_t lz{0};
for(;lz < rc.length() && rc[lz] == '0'; lz++);
return "0x" + rc.substr(lz);
}
template <typename T>
inline std::string padded_num(T value) {
auto result = std::to_string(value);
return result.length() > 3 ? "" : std::string(3 - result.length(), '0') + result;
}
void ip_router::print_as_string(std::string& result, const std::string& indent, const ip_router::route_entry *current_entry, size_t level) const {
level -= current_entry->deep;
size_t range_begin{0};
for(size_t i = 0; i <= 0xFF; i++) {
auto entry = (const ip_router::route_entry*) current_entry->data[i];
if(level == 0 ? !entry : entry->is_const_entry()) continue;
if(i > 0) {
if(range_begin < i - 1)
result += indent + padded_num(range_begin) + ".." + padded_num(i - 1) + ": empty\n";
else if(range_begin == i - 1)
result += indent + padded_num(range_begin) + ": empty\n";
}
if(level == 0) {
result += indent + padded_num(i) + ": " + n2hexstr((uintptr_t) entry) + "\n";
} else {
result += indent + padded_num(i) + ": " + n2hexstr((uintptr_t) entry) + " (used by: " + std::to_string(entry->use_count) + ", deph: " + std::to_string(entry->deep) + ")\n";
this->print_as_string(result, indent + " ", entry, level);
}
range_begin = i + 1;
}
if(range_begin < 0xFF)
result += indent + padded_num(range_begin) + "..255: empty\n";
}

View File

@ -8,31 +8,38 @@
#include <shared_mutex>
namespace ts::network {
class ip_rounter {
class ip_router {
/* currently its not possible to change this! */
constexpr static auto bits_per_entry{8U}; /* must be a multiple of 2! */
constexpr static auto total_bits{128U};
public:
struct route_entry {
constexpr static auto const_flag_mask = 0x80000000ULL;
route_entry() noexcept = default;
~route_entry() = default;
uint32_t use_count;
struct {
uint8_t deep;
uint8_t previous_chunks[(total_bits - bits_per_entry) / 8]; /* subtract the last entry because we do not need a special check there */
uint32_t use_count; /* could be size_t as well :) */
} __attribute__((packed));
void* data[1U << (bits_per_entry + 1)];
[[nodiscard]] inline bool is_const_entry() const { return (this->use_count & const_flag_mask) > 0; }
};
static_assert(std::is_trivially_destructible<route_entry>::value);
static_assert(std::is_trivially_constructible<route_entry>::value);
ip_rounter();
~ip_rounter();
ip_router();
~ip_router();
void cleanup_cache();
[[nodiscard]] bool validate_tree() const;
[[nodiscard]] size_t used_memory() const;
[[nodiscard]] std::string print_as_string() const;
/**
* @return Whatever the route register succeeded to initialize
*/
@ -60,8 +67,9 @@ namespace ts::network {
spin_lock entry_lock{};
route_entry root_entry{};
route_entry* create_8bit_entry(size_t /* level */);
bool validate_chunk_entry(const ip_rounter::route_entry* current_entry, size_t level) const;
size_t chunk_memory(const ip_rounter::route_entry* current_entry, size_t level) const;
route_entry* create_8bit_entry(size_t /* level */, bool /* as end entry */);
bool validate_chunk_entry(const ip_router::route_entry* /* current entry */, size_t /* level */) const;
size_t chunk_memory(const ip_router::route_entry* /* current entry */, size_t /* level */) const;
void print_as_string(std::string& /* output */, const std::string& /* indent */, const ip_router::route_entry* /* current entry */, size_t /* level */) const;
};
}

173
test/ip_router.cpp Normal file
View File

@ -0,0 +1,173 @@
#include <assert.h>
#include <iostream>
#include <src/misc/net.h>
#include <netinet/in.h>
#include "src/misc/ip_router.h"
using namespace ts::network;
inline sockaddr_storage address(const std::string& address) {
sockaddr_storage result{};
memset(&result, 0, sizeof(result));
net::resolve_address(address, result);
return result;
}
inline void resolve(ip_router& router, const std::string& address_str) {
std::cout << address_str << " -> " << router.resolve(address(address_str)) << std::endl;
}
/*
IPv6 & 15000 iterations
0% misses:
Time per resolve: 608ns
50% misses:
Time per resolve: 448ns
100% misses:
Time per resolve: 30ns
IPv6 & 15000 iterations & shared mutex
0% misses:
Time per resolve: 848ns
50% misses:
Time per resolve: 602ns
100% misses:
Time per resolve: 39ns
IPv6 & 15000 iterations & spin lock
0% misses:
Time per resolve: 743ns
50% misses:
Time per resolve: 510ns
100% misses:
Time per resolve: 29ns
New system:
0% misses:
Time per resolve: 149ns
Used memory: 67.155kb
50% misses:
Time per resolve: 71ns
Used memory: 32.501kb
100% misses:
Time per resolve: 14ns
Used memory: 4kb
*/
inline void benchmark_resolve() {
const auto iterations = 500;
std::vector<sockaddr_storage> addresses{};
/* generate addresses */
{
const auto rnd = []{
return (uint32_t) ((uint8_t) ::rand());
};
for(int i = 0; i < iterations; i++) {
auto& address = addresses.emplace_back();
address.ss_family = AF_INET;
auto address4 = (sockaddr_in*) &address;
address4->sin_addr.s_addr = (rnd() << 24U) | (rnd() << 16U) | (rnd() << 8U) | (rnd() << 0U);
}
#if 0
for(int i = 0; i < iterations; i++) {
auto& address = addresses.emplace_back();
address.ss_family = AF_INET6;
auto address6 = (sockaddr_in6*) &address;
for(auto& part : address6->sin6_addr.__in6_u.__u6_addr8)
part = rand();
}
#endif
}
for(auto missPercentage : {0, 50, 100}) {
std::cout << missPercentage << "% misses:\n";
ip_router rounter{};
for(auto& address : addresses) {
if((::rand() % 100) < missPercentage) continue;
rounter.register_route(address, &address, nullptr);
}
/* resolve */
{
auto begin = std::chrono::system_clock::now();
size_t index{0};
for(auto& address : addresses) {
index++;
auto result = rounter.resolve(address);
(void) result;
//if(missPercentage == 0 && memcmp(result, &address, sizeof(address)) != 0)
// __asm__("nop");
}
auto end = std::chrono::system_clock::now();
std::cout << "Total time: " << std::chrono::ceil<std::chrono::microseconds>(end - begin).count() << "us\n";
std::cout << "Time per resolve: " << std::chrono::ceil<std::chrono::nanoseconds>(end - begin).count() / iterations << "ns\n";
std::cout << "Used memory: " << rounter.used_memory() / 1024 << "kb" << std::endl;
__asm__("nop");
if(missPercentage == 0)
;//std::cout << "Tree:\n" << rounter.print_as_string() << "\n";
}
}
/* resolve */
{
std::cout << "List iterate:\n";
auto begin = std::chrono::system_clock::now();
for(auto& address : addresses) {
for(auto& addressB : addresses)
if(memcmp(&address, &addressB, sizeof(sockaddr_storage)) == 0)
break;
}
auto end = std::chrono::system_clock::now();
std::cout << "Total time: " << std::chrono::ceil<std::chrono::microseconds>(end - begin).count() << "us\n";
std::cout << "Time per resolve: " << std::chrono::ceil<std::chrono::nanoseconds>(end - begin).count() / iterations << "ns\n";
}
}
int main() {
#if 0
ip_router rounter{};
assert(rounter.validate_tree());
resolve(rounter, "127.0.0.1");
rounter.register_route(address("127.0.0.1"), (void*) 0x127001);
assert(rounter.validate_tree());
resolve(rounter, "127.0.0.1");
rounter.register_route(address("127.0.0.2"), (void*) 0x127002);
assert(rounter.validate_tree());
resolve(rounter, "127.0.0.2");
resolve(rounter, "127.0.1.2");
rounter.register_route(address("127.0.1.2"), (void*) 0x127012);
resolve(rounter, "127.0.0.2");
resolve(rounter, "127.0.1.2");
std::cout << "Tree:\n" << rounter.print_as_string() << "\n";
rounter.reset_route(address("127.0.0.1"));
assert(rounter.validate_tree());
resolve(rounter, "127.0.0.1");
resolve(rounter, "127.0.0.2");
rounter.reset_route(address("127.0.0.2"));
assert(rounter.validate_tree());
resolve(rounter, "127.0.0.1");
resolve(rounter, "127.0.0.2");
#endif
benchmark_resolve();
return 0;
}