ThorsSQL Lib: Part 3: Layer 5 HandShake

Question

Have a working version of MySQL implementation of ThorsSQL library done.

If you want to check it out you can find the whole thing on github ThorsSQL.

This is a follow on to previous code Reviews:
Part 3: Layer 5 Req/Resp
Part 3: Layer 4
Part 3: Layer 3
Part 3: Layer 2
Part 3: Layer 1
Part 2
Part 1

The documentation for these classes is here:

Part 3 (Layer 5): The HandShake Resp and Reply

When you first connect to a MySQL server; the first action of the server is to send you a HandShake package.

The client should check the HandShake package make sure it knows the protocol suggested by the server. Then send a HandShakeResponse message to the server. If everything goes well the server will send back OK response.

If we look at the code I posted for Layer 4 you may have spotted this happening when the Connection object was initially created (added below for reference).

Connection::Connection(
 std::string const& username,
 std::string const& password,
 std::string const& database,
 std::map<std::string, std::string> const& options,
 ConectReader& pr,
 ConectWriter& pw)
 : packageReader(pr)
 , packageWriter(pw)
{
 std::unique_ptr<RespPackage> initPack = recvMessage(
 {{0x0A, [](int firstByte, ConectReader& reader
 )
 {return new RespPackageHandShake(firstByte, reader);}
 }
 });
 std::unique_ptr<RespPackageHandShake> handshake = downcastUniquePtr<RespPackageHandShake>(std::move(initPack));
 packageReader.initFromHandshake(handshake->getCapabilities(), handshake->getCharset());
 packageWriter.initFromHandshake(handshake->getCapabilities(), handshake->getCharset());
 RequPackageHandShakeResponse handshakeresp(username, password, options, database, *handshake);
 std::unique_ptr<RespPackage> ok = sendHandshakeMessage<RespPackage>(handshakeresp,
 {{0xFE, [](int firstByte, ConectReader& reader)
 {return new RespPackageAuthSwitchRequest(firstByte, reader);}
 }
 });
 if (!ok)
 {
 throw std::domain_error("Connection::Connection: Handshake failed: Unexpected Package");
 }
 if (!(ok->isOK()))
 {
 throw std::domain_error(errorMsg("Connection::Connection: Handshake failed: Got: ", (*ok)));
 }
}

So the first object we receive above (with the call recvMessage()) from the server is RespPackageHandShake.

The client then starts a Requ/Resp cycle (with the call sendHandshakeMessage()) sending a RequPackageHandShakeResponse object back to the server.

Note: Normally we use the sendMessageGetResponse() to send messages to the server, the difference between this and sendHandshakeMessage() is that former resets the package number count (at layer 2) to zero before starting the interaction with server since we are continuing a discussion started by the server we don't want to reset this count to zero (this is the one special case).

If everything goes well then we should get RespPackageOK returned. BUT the server may want to change authentication protocol in which case it will return a RespPackageAuthSwitchRequest asking the client for better authentication.

Currently this client does not support that and will throw an exception (A fun task for somebody that want to try :-) )

You will notice that the HandShakeResponse object contains all the work for encrypting the password for transport to the server using some random key data provided by the initial HandShake.

RespPackageHandShake.h

#ifndef THORS_ANVIL_MYSQL_DETAILS_PACKAGE_RESP_HAND_SHAKE_H
#define THORS_ANVIL_MYSQL_DETAILS_PACKAGE_RESP_HAND_SHAKE_H
#include "RespPackage.h"
#include <string>
#include <sstream>
#include <ostream>
#include <iomanip>
namespace ThorsAnvil
{
 namespace MySQL
 {
class ConectReader;
class RespPackageHandShake: public RespPackage
{
 std::string serverVersion;
 long connectionID;
 std::string authPluginData;
 long check;
 //--
 long statusFlag;
 long authPluginLength;
 std::string reserved;
 std::string authPluginName;
 bool isV9;
 long capabilities;
 char charset;
 public:
 RespPackageHandShake(int firstByte, ConectReader& reader);
 virtual std::ostream& print(std::ostream& s) const;
 long getCapabilities() const {return capabilities;}
 long getCharset() const {return charset;}
 std::string const& getAuthPluginName() const {return authPluginName;}
 std::string const& getAuthPluginData() const {return authPluginData;}
};
inline std::ostream& RespPackageHandShake::print(std::ostream& s) const
{
 std::stringstream reservedDecoded;
 for (char x: reserved)
 {
 reservedDecoded << "0x" << std::hex << static_cast<int>(x) << " ";
 }
 std::stringstream authPluginDataDecoded;
 for (char x: authPluginData)
 {
 authPluginDataDecoded << "0x" << std::hex << static_cast<int>(x) << " ";
 }
 return s << "RespPackageHandShake: "
 << "serverVersion(" << serverVersion << ") "
 << "connectionID(" << connectionID << ") "
 << "authPluginData(" << authPluginDataDecoded.str() << ") "
 << "check(" << check << ") "
 << "statusFlag( 0x" << std::hex << std::setw(8) << std::setfill('0') << statusFlag << ") "
 << "authPluginLength(" << authPluginLength << ") "
 << "reserved(" << reservedDecoded.str() << ") "
 << "authPluginName(" << authPluginName << ") "
 << "capabilities(" << capabilities << ") "
 << "isV9(" << isV9 << ") "
 << std::dec;
}
 }
}
#endif

RespPackageHandShake.cpp

#include "ThorMySQL.h"
#include "ConectReader.h"
#include "RespPackageHandShake.h"
#include <cassert>
using namespace ThorsAnvil::MySQL;
RespPackageHandShake::RespPackageHandShake(int firstbyte, ConectReader& reader)
 : RespPackage(reader, "HandShake")
 , serverVersion(reader.nulTerminatedString())
 , connectionID(reader.fixedLengthInteger<4>())
 , authPluginData(reader.fixedLengthString(8))
 , check(0)
 , statusFlag(0)
 , authPluginLength(0)
 , isV9(false)
 , capabilities(0)
{
 // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeV10
 assert(firstbyte = 0x0A);
 if (reader.isEmpty())
 {
 isV9 = true;
 return;
 }
 check = reader.fixedLengthInteger<1>();
 capabilities = reader.fixedLengthInteger<2>();
 if (reader.isEmpty())
 {
 return;
 }
 charset = reader.fixedLengthInteger<1>();
 statusFlag = reader.fixedLengthInteger<2>();
 long cap2 = reader.fixedLengthInteger<2>();
 authPluginLength= 0;
 capabilities = capabilities | (cap2 << 16);
 if (capabilities & CLIENT_PLUGIN_AUTH)
 {
 authPluginLength= reader.fixedLengthInteger<1>();
 }
 else
 {
 int fill = reader.fixedLengthInteger<1>();
 assert(fill == 0);
 }
 reserved = reader.fixedLengthString(10);
 std::string authPluginData2;
 if (capabilities & CLIENT_SECURE_CONNECTION)
 {
 authPluginData2 = reader.variableLengthString(std::max(13L, authPluginLength - 8L));
 }
 if (capabilities & CLIENT_PLUGIN_AUTH)
 {
 authPluginName = reader.nulTerminatedString();
 }
 // Because of the way 13L is the min size for authPluginData2 data
 // This may exceed the actual size of the auth data. So after concatenating the
 // data make sure we only use the required length `authPluginLength`. This means
 // removing the extra terminating '0円' character.
 // see: mysql-5.7/sql/auth/sql_authentication.cc 538
 // the 13th byte is "0円 byte, terminating the second part of a scramble"
 authPluginData = (authPluginData + authPluginData2).substr(0, authPluginLength-1);
}

RequPackageHandShakeResp.h

#ifndef THORS_ANVIL_MYSQL_PACKAGE_REQU_HANDSHAKE_RESPONSE_H
#define THORS_ANVIL_MYSQL_PACKAGE_REQU_HANDSHAKE_RESPONSE_H
#include "RequPackage.h"
#include "ThorSQL/SQLUtil.h"
#include <string>
#include <ostream>
namespace ThorsAnvil
{
 namespace MySQL
 {
using Options=SQL::Options;
class RespPackageHandShake;
class ConectWriter;
class RequPackageHandShakeResponse: public RequPackage
{
 std::string const& username;
 std::string authResponse;
 Options const& options;
 std::string const& database;
 std::string const& authPluginName;
 long capabilities;
 public:
 RequPackageHandShakeResponse(std::string const& username,
 std::string const& password,
 Options const& options,
 std::string const& database,
 RespPackageHandShake const& handshake);
 virtual std::ostream& print(std::ostream& s) const override;
 virtual void build(ConectWriter& writer) const override;
};
 }
}
#endif

RequPackageHandShakeResp.cpp

#include "ThorMySQL.h"
#include "RequPackageHandShakeResp.h"
#include "RespPackageHandShake.h"
#include "ThorCryptWrapper.h"
using namespace ThorsAnvil::MySQL;
RequPackageHandShakeResponse::RequPackageHandShakeResponse(std::string const& username,
 std::string const& password,
 Options const& options,
 std::string const& database,
 RespPackageHandShake const& handshake)
 : RequPackage("RequPackageHandShakeResponse", "HandShake-Response")
 , username(username)
 , options(options)
 , database(database)
 , authPluginName(handshake.getAuthPluginName())
 , capabilities(handshake.getCapabilities())
{
 if (authPluginName == "mysql_old_password")
 {
 throw std::runtime_error(
 errorMsg("ThorsAnvil::MySQL::HandshakeResponsePackage::HandshakeResponsePackage: ",
 "mysql_old_password: not supported"
 ));
 }
 else if (authPluginName == "mysql_clear_password")
 {
 throw std::runtime_error(
 errorMsg("ThorsAnvil::MySQL::HandshakeResponsePackage::HandshakeResponsePackage: ",
 "mysql_clear_password: not supported"
 ));
 }
 else if (authPluginName == "authentication_windows_client")
 {
 throw std::runtime_error(
 errorMsg("ThorsAnvil::MySQL::HandshakeResponsePackage::HandshakeResponsePackage: ",
 "authentication_windows_client: not supported"
 ));
 }
 else if (authPluginName == "sha256_password")
 {
 throw std::runtime_error(
 errorMsg("ThorsAnvil::MySQL::HandshakeResponsePackage::HandshakeResponsePackage: ",
 "sha256_password: not supported"
 ));
 }
 else if (authPluginName == "mysql_native_password")
 {
 // Requires CLIENT_SECURE_CONNECTION
 // SHA1( password ) XOR SHA1( "20-bytes random data from server" <concat> SHA1( SHA1( password ) ) )
 ThorSHADigestStore stage1;
 thorSHA1(stage1, password);
 ThorSHADigestStore stage2;
 thorSHA1(stage2, stage1);
 std::string extendedAuth = handshake.getAuthPluginData() + std::string(stage2, stage2 + SHA_DIGEST_LENGTH);
 ThorSHADigestStore extendedHash;
 thorSHA1(extendedHash, extendedAuth);
 for (int loop=0;loop < SHA_DIGEST_LENGTH;++loop)
 {
 extendedHash[loop] = extendedHash[loop] ^ stage1[loop];
 }
 authResponse = std::string(extendedHash, extendedHash + SHA_DIGEST_LENGTH);
 }
 else
 {
 throw std::runtime_error(
 errorMsg("ThorsAnvil::MySQL::HandshakeResponsePackage::HandshakeResponsePackage: ",
 "UNKNOWN authentication method(", authPluginName, "): not supported"
 ));
 }
}
void RequPackageHandShakeResponse::build(ConectWriter& writer) const
{
 // These capabilities mirror the `mysql` tool.
 // We will leave this for now but it may change
 // as the understanding of the system updates.
 unsigned long cap = CLIENT_SET_CLIENT | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA | CLIENT_CONNECT_ATTRS
 | CLIENT_PLUGIN_AUTH | CLIENT_PS_MULTI_RESULTS | CLIENT_MULTI_RESULTS | CLIENT_MULTI_STATEMENTS
 | CLIENT_SECURE_CONNECTION | CLIENT_TRANSACTIONS
 | CLIENT_INTERACTIVE | CLIENT_PROTOCOL_41
 | CLIENT_LOCAL_FILES
 | CLIENT_CONNECT_WITH_DB | CLIENT_LONG_FLAG | CLIENT_LONG_PASSWORD;
 // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse41
 // Turn off flags not supported by the server
 unsigned long localCap = capabilities & cap;
 writer.writeFixedLengthInteger<4>(localCap);
 long maxPacketSize = 0x01000000;
 writer.writeFixedLengthInteger<4>(maxPacketSize);
 unsigned char charset = 0x21;
 writer.writeFixedLengthInteger<1>(charset);
 std::string reserved(23, '0円');
 writer.writeFixedLengthString(reserved, 23);
 writer.writeNullTerminatedString(username);
 if (localCap & CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA)
 {
 writer.writeLengthEncodedInteger(authResponse.size());
 writer.writeFixedLengthString(authResponse, authResponse.size());
 }
 else if (localCap & CLIENT_SECURE_CONNECTION)
 {
 char val = authResponse.size();
 writer.writeFixedLengthInteger<1>(val);
 writer.writeFixedLengthString(authResponse, authResponse.size());
 }
 else
 {
 writer.writeNullTerminatedString(authResponse);
 }
 if (localCap & CLIENT_CONNECT_WITH_DB)
 {
 writer.writeNullTerminatedString(database);
 }
 if (localCap & CLIENT_PLUGIN_AUTH)
 {
 writer.writeNullTerminatedString(authPluginName);
 }
 // TODO Add Key Values
 // For now empty the options
 if (localCap & CLIENT_CONNECT_ATTRS)
 {
 std::size_t size = 0;
 for (auto const& loop: options)
 {
 size += loop.first.size() + loop.second.size();
 }
 writer.writeLengthEncodedInteger(size);
 for (auto const& loop: options)
 {
 writer.writeLengthEncodedString(loop.first);
 writer.writeLengthEncodedString(loop.second);
 }
 }
}
std::ostream& RequPackageHandShakeResponse::print(std::ostream& s) const
{
 std::stringstream authRespDecoded;
 for (char x: authResponse)
 { authRespDecoded << "0x" << std::hex << static_cast<unsigned int>(static_cast<unsigned char>(x)) << " ";
 }
 std::stringstream keyValDecoded;
 for (auto const& val: options)
 { keyValDecoded << "KV(" << val.first << " => " << val.second << ") ";
 }
 return s << "HandshakeResponsePackage: "
 << "username(" << username << ") "
 << "authResponse(" << authRespDecoded.str() << ") "
 << "options(" << keyValDecoded.str() << ") "
 << "database(" << database << ") "
 << "authPluginName(" << authPluginName << ") "
 << "capabilities(" << capabilities << ") ";
}

RespPackageAuthSwitchRequest.h

#ifndef THORS_ANVIL_MYSQL_DETAILS_PACKAGE_RESP_EOF_H
#define THORS_ANVIL_MYSQL_DETAILS_PACKAGE_RESP_EOF_H
#include "RespPackage.h"
#include <string>
#include <ostream>
#include <iomanip>
#include <cassert>
namespace ThorsAnvil
{
 namespace MySQL
 {
class ConectReader;
class RespPackageAuthSwitchRequest: public RespPackage
{
 public:
 std::string pluginName;
 std::string pluginData;
 RespPackageAuthSwitchRequest(int firstByte, ConectReader& reader)
 : RespPackage(reader, "AuthSwitchRequest")
 , pluginName(reader.nulTerminatedString())
 , pluginData(reader.restOfPacketString())
 {
 // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest
 assert(firstByte == 0xFE);
 }
 virtual std::ostream& print(std::ostream& s) const
 {
 s << "AuthSwitchRequest: "
 << "pluginName(" << pluginName << ") "
 << "pluginData(";
 for (auto c: pluginData)
 {
 s << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(c) << " ";
 }
 s << ") " << std::dec;
 return s;
 }
};
 }
}
#endif