这是indexloc提供的服务,不要输入任何密码
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions src/Processors/Formats/Impl/BSONEachRowRowInputFormat.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -433,14 +433,20 @@ void BSONEachRowRowInputFormat::readTuple(IColumn & column, const DataTypePtr &

assertChar(BSON_DOCUMENT_END, *in);

if (read_nested_columns != data_type_tuple->getElements().size())
const auto elements_size = data_type_tuple->getElements().size();
if (read_nested_columns != elements_size)
throw Exception(
ErrorCodes::INCORRECT_DATA,
"Cannot parse tuple column with type {} from BSON array/embedded document field, "
"the number of fields in tuple and BSON document doesn't match: {} != {}",
data_type->getName(),
data_type_tuple->getElements().size(),
elements_size,
read_nested_columns);

/// There are no nested columns to grow, so we must explicitly increment the column size.
/// Otherwise, `column.size()` will return 0 for empty tuples columns.
if (elements_size == 0)
tuple_column.addSize(1);
}

void BSONEachRowRowInputFormat::readMap(IColumn & column, const DataTypePtr & data_type, BSONType bson_type)
Expand Down
7 changes: 6 additions & 1 deletion src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -407,8 +407,13 @@ bool MsgPackVisitor::start_array(size_t size) // NOLINT

ColumnTuple & column_tuple = assert_cast<ColumnTuple &>(info_stack.top().column);
/// Push nested columns into stack in reverse order.
for (ssize_t i = nested_types.size() - 1; i >= 0; --i)
for (ssize_t i = static_cast<ssize_t>(nested_types.size()) - 1; i >= 0; --i)
info_stack.push(Info{column_tuple.getColumn(i), nested_types[i], true, std::nullopt, nullptr});

/// There are no nested columns to grow, so we must explicitly increment the column size.
/// Otherwise, `column.size()` will return 0 for empty tuples columns.
if (nested_types.empty())
column_tuple.addSize(1);
}
else
{
Expand Down
2 changes: 2 additions & 0 deletions tests/queries/0_stateless/03277_empty_tuple_formats.reference
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ RowBinary
Values
()
BSONEachRow
()
MsgPack
()
Native
()
TSV
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
1 ()
1 ()
1 ()
1 ()
1 ()
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
DROP TABLE IF EXISTS t0;
DROP TABLE IF EXISTS random_filename;

CREATE TABLE t0 (c0 Int32, c1 Tuple()) ENGINE = Memory;
CREATE TABLE random_filename (name String) ENGINE = Memory;

INSERT INTO random_filename SELECT concat('03716_test_bson_empty_tuple_', toString(generateUUIDv4()), '.bson');

INSERT INTO FUNCTION file((SELECT name FROM random_filename LIMIT 1), 'BSONEachRow', 'c0 Int32, c1 Tuple()')
SELECT 1, tuple() FROM numbers(5) SETTINGS engine_file_truncate_on_insert = 1;

INSERT INTO t0 SELECT * FROM file((SELECT name FROM random_filename LIMIT 1), 'BSONEachRow', 'c0 Int32, c1 Tuple()');

SELECT * FROM t0 ORDER BY c0;

DROP TABLE t0;
DROP TABLE random_filename;
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
1 ()
1 ()
1 ()
1 ()
1 ()
20 changes: 20 additions & 0 deletions tests/queries/0_stateless/03717_msgpack_empty_tuple_column.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
-- Tags: no-fasttest
-- no-fasttest: 'MsgPack` format is not supported

DROP TABLE IF EXISTS t0;
DROP TABLE IF EXISTS random_filename;

CREATE TABLE t0 (c0 Int32, c1 Tuple()) ENGINE = Memory;
CREATE TABLE random_filename (name String) ENGINE = Memory;

INSERT INTO random_filename SELECT concat('03716_test_msgpack_empty_tuple_', toString(generateUUIDv4()), '.msgpack');

INSERT INTO FUNCTION file((SELECT name FROM random_filename LIMIT 1), 'MsgPack', 'c0 Int32, c1 Tuple()')
SELECT 1, tuple() FROM numbers(5) SETTINGS engine_file_truncate_on_insert = 1;

INSERT INTO t0 SELECT * FROM file((SELECT name FROM random_filename LIMIT 1), 'MsgPack', 'c0 Int32, c1 Tuple()');

SELECT * FROM t0 ORDER BY c0;

DROP TABLE t0;
DROP TABLE random_filename;
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
Native
0 ()
1 ()
2 ()
3 ()
4 ()
TSV
0 ()
1 ()
2 ()
3 ()
4 ()
CSV
0 \N
1 \N
2 \N
3 \N
4 \N
TSKV
0 ()
1 ()
2 ()
3 ()
4 ()
JSON
0 ()
1 ()
2 ()
3 ()
4 ()
JSONCompact
0 ()
1 ()
2 ()
3 ()
4 ()
JSONEachRow
0 []
1 []
2 []
3 []
4 []
JSONObjectEachRow
0 []
1 []
2 []
3 []
4 []
JSONCompactEachRow
0 []
1 []
2 []
3 []
4 []
JSONColumns
0 []
1 []
2 []
3 []
4 []
JSONCompactColumns
0 []
1 []
2 []
3 []
4 []
JSONColumnsWithMetadata
0 ()
1 ()
2 ()
3 ()
4 ()
ORC
0 ()
1 ()
2 ()
3 ()
4 ()
Arrow
0 ()
1 ()
2 ()
3 ()
4 ()
Avro
0 ()
1 ()
2 ()
3 ()
4 ()
RowBinary
0 ()
1 ()
2 ()
3 ()
4 ()
Values
0 ()
1 ()
2 ()
3 ()
4 ()
BSONEachRow
0 ()
1 ()
2 ()
3 ()
4 ()
MsgPack
0 ()
1 ()
2 ()
3 ()
4 ()
Native
0 ()
1 ()
2 ()
3 ()
4 ()
TSV
0 ()
1 ()
2 ()
3 ()
4 ()
CSV
0 ()
1 ()
2 ()
3 ()
4 ()
TSKV
0 ()
1 ()
2 ()
3 ()
4 ()
JSON
0 ()
1 ()
2 ()
3 ()
4 ()
JSONCompact
0 ()
1 ()
2 ()
3 ()
4 ()
JSONEachRow
0 ()
1 ()
2 ()
3 ()
4 ()
JSONObjectEachRow
0 ()
1 ()
2 ()
3 ()
4 ()
JSONCompactEachRow
0 ()
1 ()
2 ()
3 ()
4 ()
JSONColumns
0 ()
1 ()
2 ()
3 ()
4 ()
JSONCompactColumns
0 ()
1 ()
2 ()
3 ()
4 ()
JSONColumnsWithMetadata
0 ()
1 ()
2 ()
3 ()
4 ()
ORC
0 ()
1 ()
2 ()
3 ()
4 ()
Arrow
0 ()
1 ()
2 ()
3 ()
4 ()
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/usr/bin/env bash
# Tags: no-fasttest

CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh


# Test that no formats crash or return LOGICAL_ERROR on empty tuple
# This test is on the same spirit as 03277_empty_tuple_formats.sh except it has additional column in addition to empty tuple.
# For example this logical error: Invalid number of rows in Chunk Int32(size = 5) Tuple(size = 0) column Tuple() at position 1: expected 5, got 0
# was triggered when there was another column in addition to the empty tuple. This test covers that case.

FILE=03277_$CLICKHOUSE_DATABASE

# With schema inference.
for format in Native TSV CSV TSKV JSON JSONCompact JSONEachRow JSONObjectEachRow JSONCompactEachRow JSONColumns JSONCompactColumns JSONColumnsWithMetadata ORC Arrow
do
echo $format
$CLICKHOUSE_LOCAL -q "
insert into function file('$FILE', '$format') select number, () from numbers(5) settings engine_file_truncate_on_insert=1;
select * from file('$FILE', '$format');"
done

# Picky about column names.
echo Avro
$CLICKHOUSE_LOCAL -q "
insert into function file('$FILE', 'Avro') select number as x, () as y from numbers(5) settings engine_file_truncate_on_insert=1;
select * from file('$FILE', 'Avro');"

# Without schema inference.
for format in RowBinary Values BSONEachRow MsgPack Native TSV CSV TSKV JSON JSONCompact JSONEachRow JSONObjectEachRow JSONCompactEachRow JSONColumns JSONCompactColumns JSONColumnsWithMetadata ORC Arrow
do
echo $format
$CLICKHOUSE_LOCAL -q "
insert into function file('$FILE', '$format', 'x UInt64, y Tuple()') select number as x, () as y from numbers(5) settings engine_file_truncate_on_insert=1;
select * from file('$FILE', '$format', 'x UInt64, y Tuple()');"
done

# Formats that don't support empty tuples/multiple columns.
$CLICKHOUSE_LOCAL -q "
insert into function file('$FILE', 'Parquet') select number, () from numbers(5) settings engine_file_truncate_on_insert=1; -- {serverError BAD_ARGUMENTS}
insert into function file('$FILE', 'Npy') select number, () from numbers(5) settings engine_file_truncate_on_insert=1; -- {serverError TOO_MANY_COLUMNS}
insert into function file('$FILE', 'CapnProto', 'x UInt64, y Tuple()') select number as x, () as y from numbers(5) settings engine_file_truncate_on_insert=1; -- {serverError CAPN_PROTO_BAD_CAST}
insert into function file('$FILE', 'RawBLOB') select number, () from numbers(5) settings engine_file_truncate_on_insert=1; -- {serverError NOT_IMPLEMENTED}"

rm $FILE
Loading