这是indexloc提供的服务，不要输入任何密码
Skip to content

HIVE-26447: Vectorization: wrong results when filter on repeating map key orc table #3492

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 5, 2022
Merged

HIVE-26447: Vectorization: wrong results when filter on repeating map key orc table #3492

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5e2f12c
HIVE-26447: Vectorization: wrong results when filter on repeating map…
yigress Aug 2, 2022
a3458f0
Add test
yigress Aug 3, 2022
8a5b9c2
fix index
yigress Aug 4, 2022
bd66f04
add more test cases
yigress Aug 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion ...java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFMapIndexStringScalar.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,13 @@ public int findScalarInMap(MapColumnVector mapColumnVector, int mapBatchIndex) {
byte[][] keyVector = keyColVector.vector;
int[] keyStart = keyColVector.start;
int[] keyLength = keyColVector.length;
final boolean isRepeating = keyColVector.isRepeating;
for (int i = 0; i < count; i++) {
final int keyOffset = offset + i;
final int len = isRepeating? keyLength[0]: keyLength[keyOffset];
byte[] rowKey = isRepeating? keyVector[0]: keyVector[keyOffset];
if (StringExpr.equal(key, 0, key.length,
keyVector[keyOffset], keyStart[keyOffset], keyLength[keyOffset])) {
rowKey, keyStart[keyOffset], len)) {
Copy link
Contributor

@maswin maswin May 22, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

keyStart[keyOffset] should also be assigned based on isRepeating key.

return offset + i;
}
}
Expand Down
17 changes: 17 additions & 0 deletions ql/src/test/queries/clientpositive/orc_map_key_repeating.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
SET hive.vectorized.execution.enabled=true;
SET hive.fetch.task.conversion=none;

-- test single repeating key
create temporary table foo (id int, x map<string,int>) stored as orc;
insert into foo values(1, map('ABC', 9)), (2, map('ABC', 7)), (3, map('ABC', 8)), (4, map('ABC', 9));
select id from foo where x['ABC']=9;

-- test multiple repeating keys
create temporary table bar (id int, x map<string,int>) stored as orc;
insert into bar values(1, map('A', 9, 'B', 1)), (2, map('A', 7, 'B', 2)), (3, map('A', 8, 'B', 3)), (4, map('A', 9, 'B', 4));
select id from bar where x['A']=9;

-- test mixed keys
create temporary table doo (id int, x map<string,int>) stored as orc;
insert into doo values(1, map('ABC', 9, 'B', 1)), (2, map('AB', 7)), (3, map('A', 8, 'C', 3)), (4, map('D', 7, 'ABC', 9, 'E', 4));
select id from doo where x['ABC']=9;
84 changes: 84 additions & 0 deletions ql/src/test/results/clientpositive/orc_map_key_repeating.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
PREHOOK: query: create temporary table foo (id int, x map<string,int>) stored as orc
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@foo
POSTHOOK: query: create temporary table foo (id int, x map<string,int>) stored as orc
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@foo
PREHOOK: query: insert into foo values(1, map('ABC', 9)), (2, map('ABC', 7)), (3, map('ABC', 8)), (4, map('ABC', 9))
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@foo
POSTHOOK: query: insert into foo values(1, map('ABC', 9)), (2, map('ABC', 7)), (3, map('ABC', 8)), (4, map('ABC', 9))
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@foo
POSTHOOK: Lineage: foo.id SCRIPT []
POSTHOOK: Lineage: foo.x SCRIPT []
PREHOOK: query: select id from foo where x['ABC']=9
PREHOOK: type: QUERY
PREHOOK: Input: default@foo
#### A masked pattern was here ####
POSTHOOK: query: select id from foo where x['ABC']=9
POSTHOOK: type: QUERY
POSTHOOK: Input: default@foo
#### A masked pattern was here ####
1
4
PREHOOK: query: create temporary table bar (id int, x map<string,int>) stored as orc
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@bar
POSTHOOK: query: create temporary table bar (id int, x map<string,int>) stored as orc
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@bar
PREHOOK: query: insert into bar values(1, map('A', 9, 'B', 1)), (2, map('A', 7, 'B', 2)), (3, map('A', 8, 'B', 3)), (4, map('A', 9, 'B', 4))
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@bar
POSTHOOK: query: insert into bar values(1, map('A', 9, 'B', 1)), (2, map('A', 7, 'B', 2)), (3, map('A', 8, 'B', 3)), (4, map('A', 9, 'B', 4))
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@bar
POSTHOOK: Lineage: bar.id SCRIPT []
POSTHOOK: Lineage: bar.x SCRIPT []
PREHOOK: query: select id from bar where x['A']=9
PREHOOK: type: QUERY
PREHOOK: Input: default@bar
#### A masked pattern was here ####
POSTHOOK: query: select id from bar where x['A']=9
POSTHOOK: type: QUERY
POSTHOOK: Input: default@bar
#### A masked pattern was here ####
1
4
PREHOOK: query: create temporary table doo (id int, x map<string,int>) stored as orc
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@doo
POSTHOOK: query: create temporary table doo (id int, x map<string,int>) stored as orc
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@doo
PREHOOK: query: insert into doo values(1, map('ABC', 9, 'B', 1)), (2, map('AB', 7)), (3, map('A', 8, 'C', 3)), (4, map('D', 7, 'ABC', 9, 'E', 4))
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@doo
POSTHOOK: query: insert into doo values(1, map('ABC', 9, 'B', 1)), (2, map('AB', 7)), (3, map('A', 8, 'C', 3)), (4, map('D', 7, 'ABC', 9, 'E', 4))
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@doo
POSTHOOK: Lineage: doo.id SCRIPT []
POSTHOOK: Lineage: doo.x SCRIPT []
PREHOOK: query: select id from doo where x['ABC']=9
PREHOOK: type: QUERY
PREHOOK: Input: default@doo
#### A masked pattern was here ####
POSTHOOK: query: select id from doo where x['ABC']=9
POSTHOOK: type: QUERY
POSTHOOK: Input: default@doo
#### A masked pattern was here ####
1
4