这是indexloc提供的服务,不要输入任何密码
Skip to content

Support C# client for mysql protocol #84397

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions ci/defs/defs.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,12 @@ class CIFiles:
platforms=Docker.Platforms.arm_amd,
depends_on=[],
),
Docker.Config(
name="clickhouse/mysql-dotnet-client",
path="./docker/test/integration/mysql_dotnet_client",
platforms=Docker.Platforms.arm_amd,
depends_on=[],
),
]


Expand Down
12 changes: 12 additions & 0 deletions ci/docker/integration/mysql_dotnet_client/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
FROM ubuntu:18.04

RUN apt-get update \
&& apt-get install -y software-properties-common build-essential dotnet-sdk-8.0 curl

ARG ver=42.2.12
COPY Program.cs Program.cs
COPY testapp.csproj testapp.csproj
RUN dotnet add package MySql.Data
RUN dotnet add package CommandLineParser

WORKDIR /testapp
90 changes: 90 additions & 0 deletions ci/docker/integration/mysql_dotnet_client/Program.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
using System;
using MySql.Data.MySqlClient;

class Program
{
[Option('h', "host", HelpText = "Host of ClickHouse server")]
public string Host { get; set; }

[Option('p', "port", HelpText = "Port of ClickHouse server")]
public string Port { get; set; }

[Option('u', "username", HelpText = "Username")]
public string Username { get; set; }

[Option('p', "password", HelpText = "Password")]
public string Password { get; set; }
static void Main()
{
var parsed = Parser.Default.ParseArguments<Program>(args);
var options = parsed.Value;
var connectionString = $"Host={options.Host};Port={options.Port};Username={options.Username};Password={options.Password};Database=default;";

try
{
using (var connection = new MySqlConnection(connectionString))
{
Console.WriteLine("Openning connection...");
connection.Open();
Console.WriteLine("Connection openned!");

using (var cmd = new MySqlCommand("SELECT 1 as a;", connection))
using (var reader = cmd.ExecuteReader())
{
while (reader.Read())
{
Console.WriteLine(reader.GetValue(0));
}
}

string createTableSql = @"
CREATE TABLE IF NOT EXISTS test_table (
id UInt32,
name String
) ENGINE = Memory";

using (var cmd = new MySqlCommand(createTableSql, connection))
{
cmd.ExecuteNonQuery();
Console.WriteLine("Table created or exists");
}

string insertSql = "INSERT INTO test_table (id, name) VALUES (1, 'Alice'), (2, 'Bob')";

using (var cmd = new MySqlCommand(insertSql, connection))
{
int rows = cmd.ExecuteNonQuery();
Console.WriteLine($"{rows} rows inserted");
}

string selectSql = "SELECT id, name FROM test_table";

using (var cmd = new MySqlCommand(selectSql, connection))
using (var reader = cmd.ExecuteReader())
{
while (reader.Read())
{
uint id = (uint)reader.GetInt32(0);
string name = reader.GetString(1);
Console.WriteLine($"id={id}, name={name}");
}
}
}
}
catch (Exception ex)
{
Console.WriteLine("=== Exception ===");
Console.WriteLine($"Type: {ex.GetType()}");
Console.WriteLine($"Message: {ex.Message}");
Console.WriteLine($"StackTrace: {ex.StackTrace}");

if (ex.InnerException != null)
{
Console.WriteLine("--- Inner Exception ---");
Console.WriteLine($"Type: {ex.InnerException.GetType()}");
Console.WriteLine($"Message: {ex.InnerException.Message}");
Console.WriteLine($"StackTrace: {ex.InnerException.StackTrace}");
}
}
}
}
15 changes: 15 additions & 0 deletions ci/docker/integration/mysql_dotnet_client/testapp.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="CommandLineParser" Version="2.9.1" />
<PackageReference Include="MySql.Data" Version="9.4.0" />
</ItemGroup>

</Project>
1 change: 1 addition & 0 deletions ci/docker/integration/runner/dockerd-entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ export DOCKER_MYSQL_JS_CLIENT_TAG=${DOCKER_MYSQL_JS_CLIENT_TAG:=latest}
export DOCKER_MYSQL_PHP_CLIENT_TAG=${DOCKER_MYSQL_PHP_CLIENT_TAG:=latest}
export DOCKER_NGINX_DAV_TAG=${DOCKER_NGINX_DAV_TAG:=latest}
export DOCKER_POSTGRESQL_JAVA_CLIENT_TAG=${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:=latest}
export DOCKER_MYSQL_DOTNET_CLIENT_TAG=${DOCKER_MYSQL_DOTNET_CLIENT_TAG:=latest}

cd /ClickHouse/tests/integration
exec "$@"
58 changes: 58 additions & 0 deletions src/Server/MySQLHandler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include <Common/logger_useful.h>
#include <Common/re2.h>
#include <Common/setThreadName.h>
#include "IO/WriteBufferFromString.h"

#if USE_SSL
# include <Poco/Net/SSLManager.h>
Expand Down Expand Up @@ -169,6 +170,12 @@ static String killConnectionIdReplacementQuery(const String & query)
return query;
}

/// Replace "SHOW COLLATIONS" into creating temporary system mysql table.
static String showCollationsReplacementQuery(const String & /*query*/)
{
return "SELECT * FROM emulated_collations";
}


/** MySQL returns this error code, HY000, so should we.
*
Expand Down Expand Up @@ -218,11 +225,59 @@ MySQLHandler::MySQLHandler(
queries_replacements.emplace("KILL QUERY", killConnectionIdReplacementQuery);
queries_replacements.emplace("SHOW TABLE STATUS LIKE", showTableStatusReplacementQuery);
queries_replacements.emplace("SHOW VARIABLES", selectEmptyReplacementQuery);
queries_replacements.emplace("SHOW COLLATION", showCollationsReplacementQuery);
settings_replacements.emplace("SQL_SELECT_LIMIT", "limit");
settings_replacements.emplace("NET_WRITE_TIMEOUT", "send_timeout");
settings_replacements.emplace("NET_READ_TIMEOUT", "receive_timeout");
}

void MySQLHandler::setupSystemTables()
{
auto execute_clickhouse_query = [&] (const String & query)
{
auto query_context = session->makeQueryContext();
query_context->setCurrentQueryId(fmt::format("mysql:{}:{}", connection_id, toString(UUIDHelpers::generateV4())));
CurrentThread::QueryScope query_scope{query_context};

auto buf = ReadBufferFromString(query);

String output;
auto out_buf = WriteBufferFromString(output);
executeQuery(buf, out_buf, false, query_context, {}, QueryFlags{}, {});
};

{
String create_query = R"(
CREATE TEMPORARY TABLE emulated_collations
(
Collation String,
Charset String,
Id UInt64,
Default String,
Compiled String,
Sortlen UInt32,
Pad_attribute Enum('PAD SPACE' = 1, 'NO PAD' = 2)
) ENGINE = Memory;
)";

execute_clickhouse_query(create_query);
}

{
String insert_query = R"(
INSERT INTO emulated_collations VALUES
('armscii8_bin', 'armscii8', 64, 'No', 'Yes', 1, 'PAD SPACE'),
('armscii8_general_ci', 'armscii8', 32, 'Yes', 'Yes', 1, 'PAD SPACE'),
('ascii_bin', 'ascii', 65, 'No', 'Yes', 1, 'PAD SPACE'),
('ascii_general_ci', 'ascii', 11, 'Yes', 'Yes', 1, 'PAD SPACE'),
('big5_bin', 'big5', 84, 'No', 'Yes', 1, 'PAD SPACE'),
('big5_chinese_ci', 'big5', 1, 'Yes', 'Yes', 1, 'PAD SPACE'),
('binary', 'binary', 63, 'Yes', 'Yes', 1, 'NO PAD');
)";
execute_clickhouse_query(insert_query);
}
}

MySQLHandler::~MySQLHandler() = default;

void MySQLHandler::run()
Expand Down Expand Up @@ -286,6 +341,8 @@ void MySQLHandler::run()
OKPacket ok_packet(0, handshake_response.capability_flags, 0, 0, 0);
packet_endpoint->sendPacket(ok_packet);

setupSystemTables();

while (tcp_server.isOpen())
{
packet_endpoint->resetSequenceId();
Expand All @@ -305,6 +362,7 @@ void MySQLHandler::run()

if (!tcp_server.isOpen())
return;

try
{
switch (command)
Expand Down
3 changes: 3 additions & 0 deletions src/Server/MySQLHandler.h
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,9 @@ class MySQLHandler : public Poco::Net::TCPServerConnection

ProfileEvents::Event read_event;
ProfileEvents::Event write_event;

void setupSystemTables();
bool initialized_system_tables = false;
};

#if USE_SSL
Expand Down
1 change: 1 addition & 0 deletions tests/ci/integration_test_images.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
"clickhouse/python-bottle": "DOCKER_PYTHON_BOTTLE_TAG",
"clickhouse/integration-test-with-unity-catalog": "DOCKER_BASE_WITH_UNITY_CATALOG_TAG",
"clickhouse/integration-test-with-hms": "DOCKER_BASE_WITH_HMS_TAG",
"clickhouse/mysql-dotnet-client": "DOCKER_MYSQL_DOTNET_CLIENT_TAG",
}

IMAGES = list(IMAGES_ENV.keys())
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
services:
dotnet:
image: clickhouse/mysql-dotnet-client:${mysql:-latest}
# to keep container running
command: sleep infinity
42 changes: 42 additions & 0 deletions tests/integration/helpers/cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,7 @@ def __init__(
self.with_postgres = False
self.with_postgres_cluster = False
self.with_postgresql_java_client = False
self.with_mysql_dotnet_client = False
self.with_kafka = False
self.with_kafka_sasl = False
self.with_kerberized_kafka = False
Expand Down Expand Up @@ -1235,6 +1236,25 @@ def setup_postgresql_java_client_cmd(
p.join(docker_compose_yml_dir, "docker_compose_postgresql_java_client.yml"),
)

def setup_mysql_dotnet_client_cmd(
self, instance, env_variables, docker_compose_yml_dir
):
self.with_mysql_dotnet_client = True
self.base_cmd.extend(
[
"--file",
p.join(
docker_compose_yml_dir, "docker_compose_mysql_dotnet_client.yml"
),
]
)
self.base_mysql_dotnet_client_cmd = self.compose_cmd(
"--env-file",
instance.env_file,
"--file",
p.join(docker_compose_yml_dir, "docker_compose_mysql_dotnet_client.yml"),
)

def setup_kafka_cmd(self, instance, env_variables, docker_compose_yml_dir):
self.with_kafka = True
env_variables["KAFKA_HOST"] = self.kafka_host
Expand Down Expand Up @@ -1636,6 +1656,7 @@ def add_instance(
with_postgres=False,
with_postgres_cluster=False,
with_postgresql_java_client=False,
with_mysql_dotnet_client=False,
clickhouse_log_file=CLICKHOUSE_LOG_FILE,
clickhouse_error_log_file=CLICKHOUSE_ERROR_LOG_FILE,
with_mongo=False,
Expand Down Expand Up @@ -1800,6 +1821,7 @@ def add_instance(
with_postgres=with_postgres,
with_postgres_cluster=with_postgres_cluster,
with_postgresql_java_client=with_postgresql_java_client,
with_mysql_dotnet_client=with_mysql_dotnet_client,
clickhouse_start_command=clickhouse_start_command,
clickhouse_start_extra_args=extra_args,
main_config_name=main_config_name,
Expand Down Expand Up @@ -1901,6 +1923,13 @@ def add_instance(
instance, env_variables, docker_compose_yml_dir
)
)

if with_mysql_dotnet_client and not self.with_mysql_dotnet_client:
cmds.append(
self.setup_mysql_dotnet_client_cmd(
instance, env_variables, docker_compose_yml_dir
)
)

if with_odbc_drivers and not self.with_odbc_drivers:
self.with_odbc_drivers = True
Expand Down Expand Up @@ -3131,6 +3160,17 @@ def get_feature_flag_value(feature_flag):
self.up_called = True
self.wait_postgresql_java_client()

if (
self.with_mysql_dotnet_client
and self.base_mysql_dotnet_client_cmd
):
logging.debug("Setup MySQL C# Client")
subprocess_check_call(
self.base_mysql_dotnet_client_cmd + common_opts
)
self.up_called = True
self.wait_mysql_dotnet_client()

if self.with_kafka and self.base_kafka_cmd:
logging.debug("Setup Kafka")
os.mkdir(self.kafka_dir)
Expand Down Expand Up @@ -3690,6 +3730,7 @@ def __init__(
with_postgres,
with_postgres_cluster,
with_postgresql_java_client,
with_mysql_java_client,
clickhouse_start_command=CLICKHOUSE_START_COMMAND,
clickhouse_start_extra_args="",
main_config_name="config.xml",
Expand Down Expand Up @@ -3764,6 +3805,7 @@ def __init__(
self.with_postgres = with_postgres
self.with_postgres_cluster = with_postgres_cluster
self.with_postgresql_java_client = with_postgresql_java_client
self.with_mysql_dotnet_client = with_mysql_dotnet_client
self.with_kafka = with_kafka
self.with_kafka_sasl = with_kafka_sasl
self.with_kerberized_kafka = with_kerberized_kafka
Expand Down
7 changes: 7 additions & 0 deletions tests/integration/test_mysql_protocol/dotnet.reference
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Openning connection...
Connection openned!
1
Table created or exists
2 rows inserted
id=1, name=Alice
id=2, name=Bob
18 changes: 18 additions & 0 deletions tests/integration/test_mysql_protocol/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
user_configs=["configs/users.xml"],
env_variables={"UBSAN_OPTIONS": "print_stacktrace=1"},
with_mysql_client=True,
with_mysql_dotnet_client=True
)

server_port = 9001
Expand Down Expand Up @@ -899,3 +900,20 @@ def setup_java_client(started_cluster, binary: Literal["true", "false"]):
).format(
host=started_cluster.get_instance_ip("node"), port=server_port, binary=binary
)


def test_mysql_dotnet_client(started_cluster):
node = cluster.instances["node"]

with open(os.path.join(SCRIPT_DIR, "dotnet.reference")) as fp:
reference = fp.read()

res = started_cluster.exec_in_container(
started_cluster.mysql_dotnet_client_docker_id,
[
"bash",
"-c",
f"dotnet run -- --host {node.hostname} --port {server_port} --username default --password 123",
],
)
assert res == reference
Loading