Variadic template data pack strucuture designed for debug/trace log (variable-sized records)

Question

I am currently writing this pack template to pack all the values (raw / fundamental + arrays of such, especially c-strings and std::string) as a helper for my rqueue - record queue currently used for debug/trace log (with remote acces, future plan includes other variable-size records gathered from small devices - PLCs).

Interface - template<class... Elements> struct pack

• static constexpr size_t count: Number of stored elements
• template<size_t I=0> using type: Type of stored element at index
• static constexpr bool trivial: Trivial pack can safely be copied by memcpy
• T value: First value (if not empty)
• pack<Next...> next: Next values (if any)
• template<size_t I> type<I>& get(): Get stored element at index
• template<size_t I> const type<I>& get() const: Get stored element at index (const version)
• size_t length(bool last = true) const:
  Length is the size in bytes of trivial/flat representation.
  Designed to compute the storage space size to store all contained strings in continuous buffer
  last: query length without (true) or with (false) '0円' terminator for strings
• size_t flatten(void *dst, size_t max, bool last = true) const:
  Flatten will store the data in continous buffer.
  return: number of bytes used (compare it with length() to see if all the data was successfully stored)
  dst: destination buffer pointer
  max: size of destination buffer
  last: query length without (true) or with (false) '0円' terminator for strings
• template<typename Byte = byte, size_t N> size_t flatten(Byte (&dst)[N], bool last = true) const: Flatten for fixed-size byte/char array

For Doxygen in code:

#ifdef FIRDA_DOXYGEN_INVOKED_
namespace firda {
/// Data Pack designed for direct data storage,
/// transfer and variable-sized records.
//:
/// Can embed fundamental types and arrays of fundamental types,
/// especially c-strings (`char[N]`).
/// Supports `std::string` as well, but `trivial` will be `false`
/// and `flatten()` has to be used for continuous storage.
template<class... Elements> struct pack
{
/// Number of stored elements
 static constexpr size_t count;
/// Type of stored element at index
 template<size_t I=0> using type;
/// Trivial pack can safely be copied by memcpy
 static constexpr bool trivial;
/// First value (if not empty)
 T value;
/// Next values (if any)
 pack<Next...> next;
/// Get stored element at index
 template<size_t I> type<I>& get();
/// Get stored element at index (const version)
 template<size_t I> const type<I>& get() const;
/// Length is the size in bytes of trivial/flat representation.
//:
/// Designed to compute the storage space size
/// to store all contained strings in continuous buffer
///\param last query length without (true) or with (false)
/// \c '0円' terminator for strings
 size_t length(bool last = true) const;
/// Flatten will store the data in continous buffer
//:
///\return number of bytes used (compare it with length()
/// to see if all the data was successfully stored)
///\param dst destination buffer pointer
///\param max size of destination buffer
///\param last query length without (true) or with (false)
/// \c '0円' terminator for strings
 size_t flatten(void *dst, size_t max, bool last = true) const;
/// Flatten for fixed-size byte/char array
///\see flatten(void*,size_t,bool)
 template<typename Byte = byte, size_t N>
 size_t flatten(Byte (&dst)[N], bool last = true) const;
};
}
#else

Important doxygen config settings:

PREDEFINED = FIRDA_DOXYGEN_INVOKED_
MULTILINE_CPP_IS_BRIEF = YES

IdeOne Example

// c-string vs. c++string
 auto hello = make_pack("hello");
 auto world = make_pack(" world!"_s);
 cout << sizeof(hello) << '/' << hello.length()
 << '+' << sizeof(world) << '/' << world.length()
 << ": " << hello << world << endl;
// flattening test
 auto again = make_pack("hello again :)"_s);
 char aflat[again.length(false)];
 again.flatten(aflat, sizeof(aflat), false);
 cout << again << endl;
 cout << aflat << endl;
// complex flattening test
 auto pack = make_pack("pi", ' ', "= "_s, 3.14159f);
 assert(pack.length() == 11);
 char flat[12];
 pack.flatten(flat, false);
 cout << sizeof(pack) << ": " << pack << endl;
 cout << sizeof(flat) << ": " << flat << flat[3] << flat+4
 << pack.get<3>() << " (" << *(float*)(flat+7) << ")" << endl;
 for (auto b : flat) cout << hex << setfill('0') << setw(2) << (word)b;
 cout << dec << endl;

Output:

6/6+4/7: hello world!
hello again :)
hello again :)
12: pi = 3.14159
12: pi = 3.14159 (3.14159)
706900203d2000ffd00f494008

The Code

//######################################################## detail forward
namespace firda { namespace detail_ { namespace pack {
template<class...> struct data;
}}}
//################################################################## pack
namespace firda {
template<class... T> class pack
 : public detail_::pack::data<T...>
{
 typedef detail_::pack::data<T...> base;
public:
 using base::base;
 using base::count;
 using base::trivial;
 using base::length;
 using base::flatten;
 template<typename Byte = byte, size_t N> enable_if_t<
 sizeof(Byte) == 1 && is_integral<Byte>::value,
 size_t> flatten(Byte (&dst)[N], bool last = true) const
 {
 return flatten(dst, N, last);
 }
};
}
#endif
//############################################################# make_pack
namespace firda {
/// Create pack<...> from arguments
template<class... Args> inline
 pack<remove_cref_t<Args>...>
 make_pack(Args&&... args)
{
 return pack<remove_cref_t<Args>...>(
 forward<Args>(args)...);
}
//================================================================ output
/// No output for empty pack
inline ostream& operator << (ostream& s, const pack<>&)
{
 return s;
}
/// Output one packed value
template<class T> inline ostream&
 operator << (ostream& s, const pack<T>& p)
{
 return s << p.value;
}
/// Output more packed values
template<class T, class... N> inline ostream&
 operator << (ostream& s, const pack<T,N...>& p)
{
 return s << p.value << p.next;
}
}
//################################################################ detail
#ifndef FIRDA_DOXYGEN_INVOKED_
namespace firda { namespace detail_ { namespace pack {
#pragma pack(push,1)
// empty data + fallback ------------------------------------------------
template<class... E> struct data
{
 static_assert(sizeof...(E) == 0, "Unmatched argument data");
 static constexpr size_t count = 0;
 static constexpr bool trivial = true;
 size_t length(bool=true)
 {
 return 0;
 }
 size_t flatten(void *dst, size_t max, bool=true)
 {
 return 0;
 }
protected:
 ~data() = default;
};
// trivial element specialization ---------------------------------------
template<class T> struct data<T>
{
 static constexpr size_t count = 1;
 template<size_t I=0> using type = enable_if_t<I==0,T>;
 static constexpr bool trivial = is_trivial<T>::value;
 static_assert(trivial, "Non-trivial element");
 T value;
 data() = default;
 data(const T& value): value(value) {}
 data(T&& value): value(forward<T>(value)) {}
 template<size_t I> type<I>& get()
 {
 static_assert(I == 0, "Index out of range");
 return value;
 }
 template<size_t I> const type<I>& get() const
 {
 static_assert(I == 0, "Index out of range");
 return value;
 }
 size_t length(bool=true) const
 {
 return sizeof(value);
 }
 size_t flatten(void *dst, size_t max, bool=true) const
 {
 size_t sz = sizeof(value);
 if (sz > max) return 0;
 memcpy(dst, &value, sz);
 return sz;
 }
protected:
 ~data() = default;
};
// array specialization -------------------------------------------------
template<class T, size_t N> struct data<T[N]>
{
 static constexpr size_t count = 1;
 template<size_t I=0> using type = enable_if_t<I==0,T[N]>;
 static constexpr bool trivial = is_trivial<T>::value;
 static_assert(trivial, "Non-trivial array");
 T value[N];
 data() = default;
 data(const T value[N])
 {
 copy(value, value+N, this->value);
 }
 template<size_t I> type<I>& get()
 {
 static_assert(I == 0, "Index out of range");
 return value;
 }
 template<size_t I> const type<I>& get() const
 {
 static_assert(I == 0, "Index out of range");
 return value;
 }
 size_t length(bool=true) const
 {
 return sizeof(value);
 }
 size_t flatten(void *dst, size_t max, bool=true) const
 {
 size_t sz = sizeof(value);
 if (sz > max) return 0;
 memcpy(dst, value, sz);
 return sz;
 }
protected:
 ~data() = default;
};
// string specialization ------------------------------------------------
template<> struct data<string>
{
 static constexpr size_t count = 1;
 template<size_t I=0> using type = enable_if_t<I==0,string>;
 static constexpr bool trivial = false;
 string value;
 data() = default;
 data(const string& value): value(value) {}
 data(string&& value): value(forward<string>(value)) {}
 template<size_t I> type<I>& get()
 {
 static_assert(I == 0, "Index out of range");
 return value;
 }
 template<size_t I> const type<I>& get() const
 {
 static_assert(I == 0, "Index out of range");
 return value;
 }
 size_t length(bool last=true) const
 {
 return value.length() + (size_t)!last;
 }
 size_t flatten(void *dst, size_t max, bool last=true) const
 {
 size_t sz = value.length() + (size_t)!last;
 if (sz > max) return 0;
 memcpy(dst, value.c_str(), sz);
 return sz;
 }
protected:
 ~data() = default;
};
// recursive specialization ---------------------------------------------
template<class T, class... Next> struct data<T, Next...> : data<T>
{
 static constexpr size_t count = 1 + sizeof...(Next);
 template<size_t I=0> using type = conditional_t<I==0,
 T, typename data<Next...>::template type<I==0?0:I-1>>;
 static constexpr size_t trivial
 = data<T>::trivial && data<Next...>::trivial;
 using data<T>::value;
 firda::pack<Next...> next;
 data() = default;
 template<class First, class... Args>
 data(First&& value, Args&&... args)
 : data<T>(forward<First>(value))
 , next(forward<Args>(args)...) {}
 using data<T>::get;
 template<size_t I> enable_if_t<I!=0, type<I>&> get()
 {
 static_assert(I < count, "Index out of range");
 return next.template get<I-1>();
 }
 template<size_t I> enable_if_t<I!=0, const type<I>&> get() const
 {
 static_assert(I < count, "Index out of range");
 return next.template get<I-1>();
 }
 size_t length(bool last=true) const
 {
 return data<T>::length(false) + next.length(last);
 }
 size_t flatten(void *dst, size_t max, bool last=true) const
 {
 size_t sz = data<T>::flatten(dst, max, false);
 return sz == 0 ? 0 : next
 .flatten(((byte*)dst) + sz, max - sz, last);
 }
protected:
 ~data() = default;
};
#pragma pack(pop)
}}}
#endif

Answer 1

Interface Enhancement

The get() is not enough for algorithmization. A way to call some (templated/universal) function (functor) is needed. For now, this was added:

template<class Action>
 bool exec(size_t pos, Action&& action = Action())
 if pos >= count; return false
 if pos == 0; action(value)
 else next.exec(pos-1, forward<Action>(action))
 return true

More functions like this are needed (e.g. for_each).

Documentation

The MULTILINE_CPP_IS_BRIEF = YES is no longer default behaviour of Doxygen and the comments shall therefore be changed:

///\brief Data Pack designed for direct data storage,
/// transfer and variable-sized records.
///
/// Can embed fundamental types and arrays of fundamental types,
/// especially c-strings (`char[N]`).
/// Supports `std::string` as well, but `trivial` will be `false`
/// and `flatten()` has to be used for continuous storage.

Tag for SFINAE

The pack needs to be identified by a tag (struct pack_tag) to allow SFINAE in variadic functions (templated overloading with enable_if_t).

template<class... Elements> class pack: public pack_tag

Current Code

#include "basics.hpp"
namespace firda
///\addtogroup format
///\{
//######################################################## detail forward
#ifndef FIRDA_DOXYGEN_INVOKED_
namespace firda.detail_.pack_
forward template<class...> struct data
namespace firda
#endif
//################################################################### tag
struct pack_tag ///< Tag for data pack (to be used in SFINAE)
//################################################################## doxy
#ifdef FIRDA_DOXYGEN_INVOKED_
///\brief Data Pack designed for direct data storage,
/// transfer and variable-sized records.
///
/// Can embed fundamental types and arrays of fundamental types,
/// especially c-strings (`char[N]`).
/// Supports `std::string` as well, but `trivial` will be `false`
/// and `flatten()` has to be used for continuous storage.
///\ingroup rqueue
template<class... Elements> class pack: public pack_tag
public:
//----------------------------------------------------------------------
/// Number of stored elements
 static constexpr size_t count = sizeof...(Elements)
/// Type of stored element at index
 template<size_t I=0> using type = ...
/// Trivial pack can safely be copied by memcpy
 static constexpr bool trivial = ...
//----------------------------------------------------------------------
/// First value (if not empty)
 T value
/// Next values (if any)
 pack<Next...> next
/// Default construct the pack
 pack() = default
/// Pass each argument to apropriate `value` constructor
 template<class... Args> pack(Args&&... args)
//----------------------------------------------------------------------
/// Get stored element at index
 template<size_t I> type<I>& get()
/// Get stored element at index (const version)
 template<size_t I> const type<I>& get() const
//----------------------------------------------------------------------
///\brief Length is the size in bytes of trivial/flat representation.
///
/// Designed to compute the storage space size
/// to store all contained strings in continuous buffer.
///\param last query length without (true) or with (false)
/// \c '0円' terminator for strings
 size_t length(bool last = true) const
//----------------------------------------------------------------------
///\brief Flatten will store the data in continous buffer
///\return number of bytes used (compare it with length()
/// to see if all the data was successfully stored)
 size_t flatten
 ( void *dst ///< destination buffer pointer
 , size_t max ///< max size of destination buffer
 , bool last ///\param last query length without (true)
 = true) const /// or with (false) \c '0円' terminator for strings
/// Flatten for fixed-size byte/char array
///\see flatten(void*,size_t,bool) const
 template<typename Byte = byte, size_t N>
 size_t flatten(Byte (&dst)[N], bool last = true) const
//----------------------------------------------------------------------
/// Execute action with element at position
///\return false if `pos` out of range
 template<class Action> bool
 exec
 ( size_t pos ///< position (index) of the element
 , Action&& action ///< action (functor) to execute on the element
 = Action() )
/// Execute action with element at position (const version)
///\return false if `pos` out of range
 template<class Action> bool
 exec
 ( size_t pos ///< position (index) of the element
 , Action&& action ///< action (functor) to execute on the element
 = Action() ) const
#else
//################################################################## pack
template<class... T> class pack
 : public pack_tag
 , public detail_::pack_::data<T...>
 typedef detail_::pack_::data<T...> base
public:
 using base: base, count, trivial, length, flatten
 template<typename Byte = byte, size_t N> enable_if_t<
 sizeof(Byte) == 1 && is_integral<Byte>::value,
 size_t> flatten(Byte (&dst)[N], bool last = true) const
 return flatten(dst, N, last)
#endif
/// Get pack type at index (const, volatile and reference removed)
template<size_t I, class... Elements> using pack_t
 = typename pack<remove_cvref_t<Elements>...>::template type<I>
//############################################################# make_pack
/// Create pack<...> from arguments (const, volatile and reference removed)
template<class... Args> inline
 pack<remove_cvref_t<Args>...>
 make_pack(Args&&... args)
 return pack<remove_cvref_t<Args>...>(
 forward<Args>(args)...)
//================================================================ output
#ifdef FIRDA_DOXYGEN_INVOKED_
/// Output packed values
template<class... Elements> inline ostream&
 operator << (ostream& s, const pack<Elements...>& p)
#else
/// No output for empty pack
inline ostream& operator << (ostream& s, const pack<>&)
 return s
/// Output one packed value
template<class T> inline ostream&
 operator << (ostream& s, const pack<T>& p)
 return s << p.value
/// Output more packed values
template<class T, class... N> inline ostream&
 operator << (ostream& s, const pack<T,N...>& p)
 return s << p.value << p.next
#endif
//################################################################ detail
#ifndef FIRDA_DOXYGEN_INVOKED_
namespace firda.detail_.pack_
#pragma pack(push,1)
// empty data + fallback ------------------------------------------------
template<class... E> struct data
 static_assert(sizeof...(E) == 0, "Unmatched arguments")
 static constexpr size_t count = 0
 static constexpr bool trivial = true
 size_t length(bool=true)
 return 0
 size_t flatten(void *dst, size_t max, bool=true)
 return 0
 template<class Action>
 bool exec(size_t pos, Action&& action = Action())
 return false
 template<class Action>
 bool exec(size_t pos, Action&& action = Action()) const
 return false
protected:
 ~data() = default
// trivial element specialization ---------------------------------------
template<class T> struct data<T>
 static constexpr size_t count = 1
 template<size_t I=0> using type = enable_if_t<I==0,T>
 static constexpr bool trivial = is_trivial<T>::value
 static_assert(trivial, "Non-trivial element")
 T value
 data() = default
 data(const T& value): value(value) {}
 data(T&& value): value(forward<T>(value)) {}
 template<size_t I> type<I>& get()
 static_assert(I == 0, "Index out of range")
 return value
 template<size_t I> const type<I>& get() const
 static_assert(I == 0, "Index out of range")
 return value
 size_t length(bool=true) const
 return sizeof(value)
 size_t flatten(void *dst, size_t max, bool=true) const
 size_t sz = sizeof(value)
 if sz > max; return 0
 memcpy(dst, &value, sz)
 return sz
 template<class Action>
 bool exec(size_t pos, Action&& action = Action())
 if pos > 0; return false
 action(value)
 return true
 template<class Action>
 bool exec(size_t pos, Action&& action = Action()) const
 if pos > 0; return false
 action(value)
 return true
protected:
 ~data() = default
// array specialization -------------------------------------------------
template<class T, size_t N> struct data<T[N]>
 static constexpr size_t count = 1
 template<size_t I=0> using type = enable_if_t<I==0,T[N]>
 static constexpr bool trivial = is_trivial<T>::value
 static_assert(trivial, "Non-trivial array")
 T value[N]
 data() = default
 data(const T value[N])
 copy(value, value+N, this->value)
 template<size_t I> type<I>& get()
 static_assert(I == 0, "Index out of range")
 return value
 template<size_t I> const type<I>& get() const
 static_assert(I == 0, "Index out of range")
 return value
 size_t length(bool=true) const
 return sizeof(value)
 size_t flatten(void *dst, size_t max, bool=true) const
 size_t sz = sizeof(value)
 if sz > max; return 0
 memcpy(dst, value, sz)
 return sz
 template<class Action>
 bool exec(size_t pos, Action&& action = Action())
 if pos > 0; return false
 action(value)
 return true
 template<class Action>
 bool exec(size_t pos, Action&& action = Action()) const
 if pos > 0; return false
 action(value)
 return true
protected:
 ~data() = default
// string specialization ------------------------------------------------
template<> struct data<string>
 static constexpr size_t count = 1
 template<size_t I=0> using type = enable_if_t<I==0,string>
 static constexpr bool trivial = false
 string value
 data() = default
 data(const string& value): value(value) {}
 data(string&& value): value(forward<string>(value)) {}
 template<size_t I> type<I>& get()
 static_assert(I == 0, "Index out of range")
 return value
 template<size_t I> const type<I>& get() const
 static_assert(I == 0, "Index out of range")
 return value
 size_t length(bool last=true) const
 return value.length() + (size_t)!last
 size_t flatten(void *dst, size_t max, bool last=true) const
 size_t sz = value.length() + (size_t)!last
 if sz > max; return 0
 memcpy(dst, value.c_str(), sz)
 return sz
 template<class Action>
 bool exec(size_t pos, Action&& action = Action())
 if pos > 0; return false
 action(value)
 return true
 template<class Action>
 bool exec(size_t pos, Action&& action = Action()) const
 if pos > 0; return false
 action(value)
 return true
protected:
 ~data() = default
// recursive specialization ---------------------------------------------
template<class T, class... Next> struct data<T, Next...> : data<T>
 static constexpr size_t count = 1 + sizeof...(Next)
 template<size_t I=0> using type = conditional_t<I==0,
 T, typename data<Next...>::template type<I==0?0:I-1>>
 static constexpr size_t trivial
 = data<T>::trivial && data<Next...>::trivial
 using data<T>::value
 firda::pack<Next...> next
 data() = default
 template<class First, class... Args>
 data(First&& value, Args&&... args)
 : data<T>(forward<First>(value))
 , next(forward<Args>(args)...) {}
 using data<T>::get
 template<size_t I> enable_if_t<I!=0, type<I>&> get()
 static_assert(I < count, "Index out of range")
 return next.template get<I-1>()
 template<size_t I> enable_if_t<I!=0, const type<I>&> get() const
 static_assert(I < count, "Index out of range")
 return next.template get<I-1>()
 size_t length(bool last=true) const
 return data<T>::length(false) + next.length(last)
 size_t flatten(void *dst, size_t max, bool last=true) const
 size_t sz = data<T>::flatten(dst, max, false)
 return sz == 0 ? 0 : next
 .flatten(((byte*)dst) + sz, max - sz, last)
 template<class Action>
 bool exec(size_t pos, Action&& action = Action())
 if pos >= count; return false
 if pos == 0; action(value)
 else next.exec(pos-1, forward<Action>(action))
 return true
 template<class Action>
 bool exec(size_t pos, Action&& action = Action()) const
 if pos >= count; return false
 if pos == 0; action(value)
 else next.exec(pos-1, forward<Action>(action))
 return true
protected:
 ~data() = default
#pragma pack(pop)
//#######################################################################
namespace firda
#endif // doxy
///\}