+
Skip to content

Feat/es7 #200

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 15 commits into from
Nov 8, 2023
Merged

Feat/es7 #200

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
d81a675
feat(es7): make guppy comapatible with es7
thanh-nguyen-dang Apr 21, 2023
b672558
fix(type): remove type param from mapping
thanh-nguyen-dang Jun 22, 2023
14bc2b0
fix(test): fix mock data
thanh-nguyen-dang Jun 22, 2023
60416b4
fix(test): fix array-config test
thanh-nguyen-dang Jun 22, 2023
87f408f
fix(log): add debug info
thanh-nguyen-dang Jun 22, 2023
f5367db
fix(missing): change to use missing_bucket
thanh-nguyen-dang Jun 26, 2023
8c0b904
fix(debug): add more debug info
thanh-nguyen-dang Jun 27, 2023
017fd15
fix(no_data): using missing_bucket and replace null with no_data
thanh-nguyen-dang Jun 28, 2023
78beb86
fix(totalCount): fix totalCount agg
thanh-nguyen-dang Jun 29, 2023
d53ae0f
fix(docker): correct version in docker files
thanh-nguyen-dang Jun 30, 2023
4f3afcd
fix(command): correct payload to ES
thanh-nguyen-dang Jun 30, 2023
790b1a8
fix(test): correct test case
thanh-nguyen-dang Jul 11, 2023
9d2abe8
fix devHelper
thanh-nguyen-dang Nov 6, 2023
c243f2f
Merge remote-tracking branch 'origin/master' into feat/es7
thanh-nguyen-dang Nov 8, 2023
5499916
remove es6.Dockerfile
thanh-nguyen-dang Nov 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 0 additions & 51 deletions devHelper/docker/es6.Dockerfile
View file
Open in desktop

This file was deleted.

19 changes: 12 additions & 7 deletions devHelper/docker/esearch.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@ version: "3.3"
services:
# see https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-cli-run-prod-mode
elasticsearch:
image: elasticsearch
build:
dockerfile: es6.Dockerfile
# use elasticsearch/elasticsearch:7.10.0-arm64 for ARM architecture
image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.0
ports:
- "9200:9200"
- "9300:9300"
Expand All @@ -14,9 +13,12 @@ services:
volumes:
- "esdata:/usr/share/elasticsearch/data"
environment:
"cluster.name": "localdev"
"discovery.type": "single-node"
ES_JAVA_OPTS: "-Xms1g -Xmx1g"
- cluster.name=localdev
- bootstrap.memory_lock=false
- ES_JAVA_OPTS=-Xms1g -Xmx1g
- discovery.type=single-node
- network.host=0.0.0.0
- http.port=9200
deploy:
replicas: 1
restart_policy:
Expand All @@ -26,7 +28,7 @@ services:
window: 120s

kibana:
image: docker.elastic.co/kibana/kibana-oss:6.5.4
image: docker.elastic.co/kibana/kibana-oss:7.10.0
ports:
- "5601:5601"
logging:
Expand All @@ -40,6 +42,9 @@ services:
- elasticsearch
environment:
LOGGING_QUIET: "true"
SERVER_NAME: elasticsearch
ELASTICSEARCH_URL: http://elasticsearch:9200
network.host: 0.0.0.0

volumes:
esdata:
Expand Down
95 changes: 44 additions & 51 deletions devHelper/scripts/commands.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ curl -iv -X PUT "${ESHOST}/${indexName}" \
"ngram_tokenizer": {
"type": "ngram",
"min_gram": 2,
"max_gram": 20,
"max_gram": 3,
"token_chars": [ "letter", "digit" ]
}
},
Expand All @@ -64,47 +64,44 @@ curl -iv -X PUT "${ESHOST}/${indexName}" \
}
},
"mappings": {
"subject": {
"properties": {
"subject_id": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"name": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"project": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"study": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"visits": {
"type": "nested",
"properties": {
"days_to_visit": { "type": "integer" },
"visit_label": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"follow_ups": {
"type": "nested",
"properties": {
"days_to_follow_up": { "type": "integer" },
"follow_up_label": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } }
}
"properties": {
"subject_id": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"name": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"project": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"study": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"visits": {
"type": "nested",
"properties": {
"days_to_visit": { "type": "integer" },
"visit_label": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"follow_ups": {
"type": "nested",
"properties": {
"days_to_follow_up": { "type": "integer" },
"follow_up_label": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } }
}
}
},
"gender": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"race": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"ethnicity": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"vital_status": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"file_type": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"file_format": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"auth_resource_path": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"consortium_id": { "type": "integer" },
"file_count": { "type": "integer" },
"whatever_lab_result_value": { "type": "float" },
"some_nested_array_field": {
"type": "nested",
"properties": {
"some_integer_inside_nested": { "type": "integer" },
"some_string_inside_nested": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } }
}
},
"some_integer_field": { "type": "integer" },
"some_long_field": { "type": "long" },
"sensitive": { "type": "keyword" }
}
}
},
"gender": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"race": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"ethnicity": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"vital_status": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"file_type": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"file_format": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"auth_resource_path": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } },
"file_count": { "type": "integer" },
"whatever_lab_result_value": { "type": "float" },
"some_nested_array_field": {
"type": "nested",
"properties": {
"some_integer_inside_nested": { "type": "integer" },
"some_string_inside_nested": { "type": "keyword", "fields": { "analyzed": {"type": "text", "analyzer": "ngram_analyzer", "search_analyzer": "search_analyzer", "term_vector": "with_positions_offsets"} } }
}
},
"some_integer_field": { "type": "integer" },
"some_long_field": { "type": "long" },
"sensitive": { "type": "keyword" }
}
}
}
Expand All @@ -120,13 +117,11 @@ curl -iv -X PUT "${ESHOST}/${fileIndexName}" \
}
},
"mappings": {
"file": {
"properties": {
"file_id": { "type": "keyword" },
"auth_resource_path": { "type": "keyword" },
"subject_id": { "type": "keyword" },
"sensitive": { "type": "keyword" }
}
"properties": {
"file_id": { "type": "keyword" },
"auth_resource_path": { "type": "keyword" },
"subject_id": { "type": "keyword" },
"sensitive": { "type": "keyword" }
}
}
}
Expand All @@ -142,10 +137,8 @@ curl -iv -X PUT "${ESHOST}/${configIndexName}" \
}
},
"mappings": {
"_doc": {
"properties": {
"array": { "type": "keyword" }
}
"properties": {
"array": { "type": "keyword" }
}
}
}
Expand Down
9 changes: 1 addition & 8 deletions genData/genData.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,7 @@ const getRandomString = () => (Math.random() + 1).toString(36).substring(7);
async function run() {
const mapping = await client.indices.getMapping({ index: esIndex });

let m = mapping.body[esIndex].mappings;

if (program.doc_type) {
m = m[program.doc_type];
}

const m = mapping.body[esIndex].mappings;
if (m !== undefined) {
Object.entries(m.properties).forEach(([key, value]) => {
schema.items.properties[key] = fakerType(key, value, arrayFields);
Expand Down Expand Up @@ -124,7 +119,6 @@ async function run() {
const body = sample.flatMap((d) => [{
index: {
_index: esIndex,
_type: program.doc_type || null,
},
}, d]);
const chunks = chunkArray(body, 100);
Expand Down Expand Up @@ -165,7 +159,6 @@ async function run() {
{
index: {
_index: configIndex,
_type: '_doc',
_id: esIndex,
},
},
Expand Down
1 change: 1 addition & 0 deletions guppy_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{ "indices": [ { "index": "tb", "type": "subject" }, { "index": "tb_file", "type": "file" } ], "config_index": "tb-array-config", "auth_filter_field": "auth_resource_path" }
Loading
点击 这是indexloc提供的php浏览器服务，不要输入任何密码和下载