diff --git a/.gitignore b/.gitignore index bd4c2d2..541a581 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,3 @@ .vscode .idea .DS_Store - diff --git a/Cargo.lock b/Cargo.lock index fd872b7..40a5f14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,10 +10,10 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.6" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -26,22 +26,22 @@ dependencies = [ [[package]] name = "async-std" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "async-task 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "async-task 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-deque 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "futures-timer 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "kv-log-macro 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "once_cell 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "pin-project-lite 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -50,10 +50,10 @@ dependencies = [ [[package]] name = "async-task" -version = "1.2.1" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -72,8 +72,8 @@ name = "atty" version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "hermit-abi 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -89,22 +89,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "backtrace" -version = "0.3.42" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace-sys 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "backtrace-sys" -version = "0.1.32" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -112,7 +112,7 @@ name = "base64" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "safemem 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -123,69 +123,119 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "bayard" -version = "0.6.0" +version = "0.7.0" dependencies = [ - "async-std 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "bayard-client 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "bayard-proto 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "cang-jie 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bayard-client 0.7.0", + "bayard-proto 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bayard-server 0.7.0", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ctrlc 3.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "ctrlc 3.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "grpcio 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", - "iron 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "jieba-rs 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", - "job_scheduler 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "lindera-tantivy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", - "persistent 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "prometheus 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "protobuf 2.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "raft 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "router 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", - "stringreader 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tantivy 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", - "urlencoded 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "bayard-client" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" +version = "0.7.0" dependencies = [ - "bayard-proto 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bayard-proto 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bayard-server 0.7.0", + "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "grpcio 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "protobuf 2.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 2.14.0 (registry+https://github.com/rust-lang/crates.io-index)", "raft 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "bayard-proto" -version = "0.2.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "grpcio 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", - "protobuf 2.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 2.14.0 (registry+https://github.com/rust-lang/crates.io-index)", "raft 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "bayard-rest" +version = "0.7.0" +dependencies = [ + "bayard-client 0.7.0", + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "iron 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "persistent 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "router 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", + "urlencoded 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bayard-server" +version = "0.7.0" +dependencies = [ + "async-std 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bayard-proto 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cang-jie 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "grpcio 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "jieba-rs 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lindera-tantivy 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "prometheus 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 2.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "raft 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rocksdb 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", + "stringreader 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tantivy 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bincode" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bindgen" +version = "0.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cexpr 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "clang-sys 0.29.3 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hash 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "which 3.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -209,13 +259,13 @@ dependencies = [ "iron 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "persistent 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "byteorder" -version = "1.3.2" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -223,14 +273,6 @@ name = "bytes" version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "c2-chacha" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "cang-jie" version = "0.7.0" @@ -245,13 +287,16 @@ dependencies = [ name = "cc" version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "jobserver 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "cedarwood" version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -259,6 +304,14 @@ name = "census" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "cexpr" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nom 5.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -266,12 +319,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "chrono" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-integer 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "clang-sys" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", + "libloading 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -310,7 +373,7 @@ version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -319,7 +382,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "const-random-macro 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -328,7 +391,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -339,57 +402,50 @@ dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "cron" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "chrono 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", - "error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "nom 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "crossbeam" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-deque 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-epoch 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "crossbeam-channel" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "crossbeam-deque" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-epoch 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "crossbeam-epoch" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memoffset 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -398,15 +454,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "crossbeam-utils" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -418,10 +474,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "ctrlc" -version = "3.1.3" +version = "3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "nix 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -500,18 +556,10 @@ dependencies = [ "atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "error-chain" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "backtrace 0.3.42 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "fail" version = "0.3.0" @@ -524,31 +572,31 @@ dependencies = [ [[package]] name = "failure" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "backtrace 0.3.42 (registry+https://github.com/rust-lang/crates.io-index)", - "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace 0.3.46 (registry+https://github.com/rust-lang/crates.io-index)", + "failure_derive 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "failure_derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "filetime" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -563,7 +611,7 @@ name = "fs2" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -581,7 +629,7 @@ name = "fsevent-sys" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -589,7 +637,7 @@ name = "fst" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -618,67 +666,67 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "futures" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-executor 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-channel 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-executor 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "futures-channel" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "futures-core" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "futures-executor" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "futures-io" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "futures-macro" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.15 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "futures-sink" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "futures-task" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -688,19 +736,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "futures-util" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-macro 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-channel 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-macro 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.15 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-nested 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -709,7 +757,7 @@ name = "fxhash" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -718,10 +766,15 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "wasi 0.9.0+wasi-snapshot-preview1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "grpcio" version = "0.4.7" @@ -729,9 +782,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "grpcio-sys 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "protobuf 2.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 2.14.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -741,7 +794,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", "cmake 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -756,10 +809,10 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.6" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -790,8 +843,8 @@ dependencies = [ "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", "traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -805,7 +858,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-normalization 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-normalization 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -815,7 +868,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "inotify-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -823,7 +876,7 @@ name = "inotify-sys" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -831,7 +884,7 @@ name = "iovec" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -841,9 +894,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "hyper 0.10.16 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "mime_guess 1.8.7 (registry+https://github.com/rust-lang/crates.io-index)", + "mime_guess 1.8.8 (registry+https://github.com/rust-lang/crates.io-index)", "modifier 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -859,7 +912,7 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -872,17 +925,15 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "phf 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", "phf_codegen 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "job_scheduler" -version = "1.1.0" +name = "jobserver" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "chrono 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", - "cron 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "uuid 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -927,23 +978,43 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.66" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "libloading" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "librocksdb-sys" +version = "6.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bindgen 0.53.2 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", + "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "lindera" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", "lindera-core 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "lindera-dictionary 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "lindera-fst 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "lindera-ipadic 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -952,10 +1023,10 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", "lindera-fst 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -964,7 +1035,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "lindera-core 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -973,7 +1044,7 @@ name = "lindera-fst" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "levenshtein_automata 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "utf8-ranges 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -985,7 +1056,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "lindera-core 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1021,7 +1092,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "iron 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1029,9 +1100,14 @@ name = "matches" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "memchr" -version = "2.3.0" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1039,16 +1115,16 @@ name = "memmap" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "memoffset" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1061,7 +1137,7 @@ dependencies = [ [[package]] name = "mime_guess" -version = "1.8.7" +version = "1.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1080,7 +1156,7 @@ dependencies = [ "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1105,7 +1181,7 @@ version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1130,7 +1206,7 @@ name = "murmurhash32" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1139,7 +1215,7 @@ version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1151,14 +1227,30 @@ dependencies = [ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nix" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "nom" -version = "2.1.0" +version = "5.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "notify" @@ -1166,11 +1258,11 @@ version = "4.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "filetime 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", "fsevent 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "fsevent-sys 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "inotify 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", "mio-extras 2.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "walkdir 2.3.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1196,11 +1288,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "hermit-abi 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1210,21 +1302,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "owned-read" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "rental 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", "stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "owning_ref" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "percent-encoding" version = "1.0.1" @@ -1304,22 +1400,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "proc-macro-hack" -version = "0.5.11" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "proc-macro-nested" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "proc-macro2" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1333,14 +1424,14 @@ dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "protobuf 2.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 2.14.0 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "protobuf" -version = "2.10.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1350,10 +1441,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "quote" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1363,7 +1454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "protobuf 2.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 2.14.0 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1374,7 +1465,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1387,7 +1478,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1398,7 +1489,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1416,8 +1507,8 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1433,10 +1524,10 @@ dependencies = [ [[package]] name = "rand_chacha" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1490,7 +1581,7 @@ name = "rand_jitter" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1502,7 +1593,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1530,7 +1621,7 @@ name = "rayon" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-deque 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "rayon-core 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1540,11 +1631,11 @@ name = "rayon-core" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-deque 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1562,12 +1653,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "regex" -version = "1.3.3" +version = "1.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.17 (registry+https://github.com/rust-lang/crates.io-index)", "thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1578,7 +1669,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "regex-syntax" -version = "0.6.13" +version = "0.6.17" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1590,22 +1681,12 @@ dependencies = [ ] [[package]] -name = "rental" -version = "0.5.5" +name = "rocksdb" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "rental-impl 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", - "stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rental-impl" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", + "librocksdb-sys 6.6.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1628,8 +1709,8 @@ name = "rust-stemmers" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1638,16 +1719,13 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "rustc_version" -version = "0.2.3" +name = "rustc-hash" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "ryu" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1660,55 +1738,47 @@ name = "same-file" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "winapi-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "scopeguard" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde" -version = "1.0.104" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_derive" -version = "1.0.104" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_json" -version = "1.0.48" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", - "ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "shlex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "siphasher" version = "0.2.3" @@ -1721,7 +1791,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "smallvec" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1751,11 +1821,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "syn" -version = "1.0.14" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1764,9 +1834,9 @@ name = "synstructure" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1778,17 +1848,17 @@ dependencies = [ "atomicwrites 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "bitpacking 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "census 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "chrono 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "downcast-rs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "fail 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "htmlescape 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", "levenshtein_automata 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1796,17 +1866,17 @@ dependencies = [ "memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "murmurhash32 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "notify 4.0.15 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "once_cell 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "owned-read 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "owning_ref 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "owned-read 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "owning_ref 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "rust-stemmers 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "snap 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "tantivy-fst 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1821,7 +1891,7 @@ name = "tantivy-fst" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "levenshtein_automata 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "utf8-ranges 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1850,7 +1920,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1862,7 +1932,7 @@ name = "termcolor" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "winapi-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1883,11 +1953,10 @@ dependencies = [ [[package]] name = "time" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1927,10 +1996,10 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1983,7 +2052,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1996,6 +2065,11 @@ name = "version_check" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "version_check" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "void" version = "1.0.2" @@ -2008,7 +2082,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "same-file 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2016,6 +2090,14 @@ name = "wasi" version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "which" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "winapi" version = "0.2.8" @@ -2042,7 +2124,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "winapi-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2064,33 +2146,34 @@ dependencies = [ [metadata] "checksum ahash 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "6f33b5018f120946c1dcf279194f238a9f146725593ead1c08fa47ff22b0b5d3" -"checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d" +"checksum aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" -"checksum async-std 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0bf6039b315300e057d198b9d3ab92ee029e31c759b7f1afae538145e6f18a3e" -"checksum async-task 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a9f534e76ca33eaa82bc8da5adb1b9e94a16f6fa217b78e9b400094dbbf844f9" +"checksum async-std 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "538ecb01eb64eecd772087e5b6f7540cbc917f047727339a472dafed2185b267" +"checksum async-task 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0ac2c016b079e771204030951c366db398864f5026f84a44dafb0ff20f02085d" "checksum atomicwrites 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6a2baf2feb820299c53c7ad1cc4f5914a220a1cb76d7ce321d2522a94b54651f" "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" "checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" -"checksum backtrace 0.3.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b4b1549d804b6c73f4817df2ba073709e96e426f12987127c48e6745568c350b" -"checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491" +"checksum backtrace 0.3.46 (registry+https://github.com/rust-lang/crates.io-index)" = "b1e692897359247cc6bb902933361652380af0f1b7651ae5c5013407f30e109e" +"checksum backtrace-sys 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118" "checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" "checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" -"checksum bayard-client 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d66d2a49278c09df62d530ac82ea1795dcfe9692ae11beffa0c12f3c016dc92d" -"checksum bayard-proto 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f4a20679da696641d9dd2cbdb8c3854f57dbb9608f156bb2bd453ccd370de49" +"checksum bayard-proto 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9fc7cd937431740882a7e8a2fbaa3bb0514d00185238128c8cc274f10add62a2" "checksum bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" +"checksum bindgen 0.53.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6bb26d6a69a335b8cb0e7c7e9775cd5666611dc50a37177c3f2cedcfc040e8c8" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" "checksum bitpacking 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3744aff20a3437a99ebc0bb7733e9e60c7bf590478c9b897e95b38d57e5acb68" "checksum bodyparser 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f023abfa58aad6f6bc4ae0630799e24d5ee0ab8bb2e49f651d9b1f9aa4f52f30" -"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" +"checksum byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" "checksum bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1" -"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" "checksum cang-jie 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6cba53b536ecec7977fe14dd1a14b2956cb39c24f6f49d1c752bd19431a1fb4b" "checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" "checksum cedarwood 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "963e82c7b94163808ca3a452608d260b64ba5bc7b5653b4af1af59887899f48d" "checksum census 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5927edd8345aef08578bcbb4aea7314f340d80c7f4931f99fbeb40b99d8f5060" +"checksum cexpr 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" -"checksum chrono 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "31850b4a4d6bae316f7a09e691c944c28299298837edc0a03f755618c23cbc01" +"checksum chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)" = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" +"checksum clang-sys 0.29.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fe6837df1d5cba2397b835c8530f51723267e16abbf83892e9e5af4f0e5dd10a" "checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum cmake 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "81fb25b677f8bf1eb325017cb6bb8452f87969db0fedb4f757b297bee78a7c62" @@ -2098,15 +2181,14 @@ dependencies = [ "checksum const-random 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "2f1af9ac737b2dd2d577701e59fd09ba34822f6f2ebdb30a7647405d9e55e16a" "checksum const-random-macro 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "25e4c606eb459dd29f7c57b2e0879f2b6f14ee130918c2b78ccb58a9624e6c7a" "checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" -"checksum cron 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab2c162d8a208293b4a4e126db1211571ee4990dbfa6f7fdacc267486849dab8" "checksum crossbeam 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e" -"checksum crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "acec9a3b0b3559f15aee4f90746c4e5e293b701c0f7d3925d24e01645267b68c" -"checksum crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca" -"checksum crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac" +"checksum crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061" +"checksum crossbeam-deque 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" +"checksum crossbeam-epoch 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" "checksum crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" -"checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4" +"checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" "checksum crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" -"checksum ctrlc 3.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c7dfd2d8b4c82121dfdff120f818e09fc4380b0b7e17a742081a89b94853e87f" +"checksum ctrlc 3.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7a4ba686dff9fa4c1c9636ce1010b0cf98ceb421361b0bb3d6faeec43bd217a7" "checksum downcast-rs 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "52ba6eb47c2131e784a38b726eb54c1e1484904f013e576a25354d0124161af6" "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" "checksum encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" @@ -2117,11 +2199,10 @@ dependencies = [ "checksum encoding-index-tradchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" "checksum encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" "checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -"checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8" "checksum fail 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f63eec71a3013ee912a0ecb339ff0c5fa5ed9660df04bfefa10c250b885d018c" -"checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9" -"checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08" -"checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d" +"checksum failure 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "b8529c2421efa3066a5cbd8063d2244603824daccb6936b079010bb2aa89464b" +"checksum failure_derive 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231" +"checksum filetime 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f59efc38004c988e4201d11d263b8171f49a2e7ec0bdbb71773433f271504a5e" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" "checksum fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" "checksum fsevent 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5ab7d1bd1bd33cc98b0889831b72da23c0aa4df9cec7e0702f46ecea04b35db6" @@ -2131,22 +2212,23 @@ dependencies = [ "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" -"checksum futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b6f16056ecbb57525ff698bb955162d0cd03bee84e6241c27ff75c08d8ca5987" -"checksum futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fcae98ca17d102fd8a3603727b9259fcf7fa4239b603d2142926189bc8999b86" -"checksum futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "79564c427afefab1dfb3298535b21eda083ef7935b4f0ecbfcb121f0aec10866" -"checksum futures-executor 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1e274736563f686a837a0568b478bdabfeaec2dca794b5649b04e2fe1627c231" -"checksum futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e676577d229e70952ab25f3945795ba5b16d63ca794ca9d2c860e5595d20b5ff" -"checksum futures-macro 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "52e7c56c15537adb4f76d0b7a76ad131cb4d2f4f32d3b0bcabcbe1c7c5e87764" -"checksum futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "171be33efae63c2d59e6dbba34186fe0d6394fb378069a76dfd80fdcffd43c16" -"checksum futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bae52d6b29cf440e298856fec3965ee6fa71b06aa7495178615953fd669e5f9" +"checksum futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5c329ae8753502fb44ae4fc2b622fa2a94652c41e795143765ba0927f92ab780" +"checksum futures-channel 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f0c77d04ce8edd9cb903932b608268b3fffec4163dc053b3b402bf47eac1f1a8" +"checksum futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f25592f769825e89b92358db00d26f965761e094951ac44d3663ef25b7ac464a" +"checksum futures-executor 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f674f3e1bcb15b37284a90cedf55afdba482ab061c407a9c0ebbd0f3109741ba" +"checksum futures-io 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a638959aa96152c7a4cddf50fcb1e3fede0583b27157c26e67d6f99904090dc6" +"checksum futures-macro 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7" +"checksum futures-sink 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3466821b4bc114d95b087b850a724c6f83115e929bc88f1fa98a3304a944c8a6" +"checksum futures-task 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" "checksum futures-timer 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a1de7508b218029b0f01662ed8f61b1c964b3ae99d6f25462d0f55a595109df6" -"checksum futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d66274fb76985d3c62c886d1da7ac4c0903a8c9f754e8fe0f35a6a6cc39e76" +"checksum futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "22766cf25d64306bedf0384da004d05c9974ab104fcc4528f1236181c18004c5" "checksum fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" "checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +"checksum glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" "checksum grpcio 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9ac757a85603e4f8c40a9f94be06a5ad412acab80b39b4e8895ca931b6619910" "checksum grpcio-sys 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "7b2f22fb0327f153acccedbe91894dd0fb15bb6f202d8195665cd206af0402b0" "checksum hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8e6073d0ca812575946eb5f35ff68dbe519907b25c42530389ff946dc84c6ead" -"checksum hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eff2656d88f158ce120947499e971d743c05dbcbed62e5bd2f38f1698bbc3772" +"checksum hermit-abi 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8a0d737e0f947a1864e93d33fdef4af8445a00d1ed8dc0c8ddb73139ea6abf15" "checksum htmlescape 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" "checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" "checksum humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" @@ -2157,16 +2239,18 @@ dependencies = [ "checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" "checksum iron 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c6d308ca2d884650a8bf9ed2ff4cb13fbb2207b71f64cda11dc9b892067295e8" "checksum itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" -"checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f" +"checksum itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" "checksum jieba-rs 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "99b1f03a6d2ba3069edb9866fa53035096544077abb34dcaa921ca14276887b6" -"checksum job_scheduler 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4df2f5492dd06f72121fda4c62266fd9f471af4ea8fd1589b1be9d85e76a8d13" +"checksum jobserver 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum kv-log-macro 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c54d9f465d530a752e6ebdc217e081a7a614b48cb200f6f0aee21ba6bc9aabb" "checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" "checksum lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" "checksum levenshtein_automata 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73a004f877f468548d8d0ac4977456a249d8fabbdb8416c36db163dfc8f2e8ca" -"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +"checksum libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)" = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005" +"checksum libloading 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" +"checksum librocksdb-sys 6.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4e3b727e2dd20ec2fb7ed93f23d9fd5328a0871185485ebdaff007b47d3e27e4" "checksum lindera 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5555a845a4fea0530a684186d8728614516ee8124ef66ac9f5dbfd6aa19eef0c" "checksum lindera-core 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "84cc6a12fb246b0b3eac731cc5e269dc7f6b606b4f15124de43fe90dc130a5b3" "checksum lindera-dictionary 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fe764c1bda2eab7f35a1885019fbe2c4b2af4d7adc4e613f753dbcef3612b451" @@ -2177,11 +2261,12 @@ dependencies = [ "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" "checksum logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6c9172cb4c2f6c52117e25570983edcbb322f130b1031ae5d5d6b1abe7eeb493" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -"checksum memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223" +"checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +"checksum memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" "checksum memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" -"checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9" +"checksum memoffset 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b4fc2c02a7e374099d4ee95a193111f72d2110197fe200272371758f6c3643d8" "checksum mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" -"checksum mime_guess 1.8.7 (registry+https://github.com/rust-lang/crates.io-index)" = "0d977de9ee851a0b16e932979515c0f3da82403183879811bc97d50bd9cc50f7" +"checksum mime_guess 1.8.8 (registry+https://github.com/rust-lang/crates.io-index)" = "216929a5ee4dd316b1702eedf5e74548c123d370f47841ceaac38ca154690ca3" "checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f" "checksum mio-extras 2.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19" "checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125" @@ -2190,14 +2275,16 @@ dependencies = [ "checksum murmurhash32 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d736ff882f0e85fe9689fb23db229616c4c00aee2b3ac282f666d8f20eb25d4a" "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" "checksum nix 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c722bee1037d430d0f8e687bbdbf222f27cc6e4e68d5caf630857bb2b6dbdce" -"checksum nom 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e5d4598834859fedb9a0a69d5b862a970e77982a92f544d547257a4d49469067" +"checksum nix 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363" +"checksum nom 5.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b471253da97532da4b61552249c521e01e736071f71c1a4f7ebbfbf0a06aad6" "checksum notify 4.0.15 (registry+https://github.com/rust-lang/crates.io-index)" = "80ae4a7688d1fab81c5bf19c64fc8db920be8d519ce6336ed4e7efe024724dbd" "checksum num-integer 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" "checksum num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" -"checksum num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" +"checksum num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" "checksum once_cell 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b1c601810575c99596d4afc46f78a678c80105117c379eb3650cf99b8a21ce5b" -"checksum owned-read 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05d57fab18d627fc4dffbd78d4a25a5b5b5211fda724231f001bee4cef1b2d3b" -"checksum owning_ref 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "49a4b8ea2179e6a2e27411d3bca09ca6dd630821cf6894c6c7c8467a8ee7ef13" +"checksum owned-read 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b66d1e235abcebc845cf93550b89b74f468c051496fafb433ede4104b9f71ba1" +"checksum owning_ref 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6ff55baddef9e4ad00f88b6c743a2a8062d4c6ade126c2a528644b8e444d52ce" +"checksum peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" "checksum persistent 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8e8fa0009c4f3d350281309909c618abddf10bb7e3145f28410782f6a5ec74c5" "checksum phf 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)" = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18" @@ -2209,20 +2296,20 @@ dependencies = [ "checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" "checksum plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a6a0dc3910bc8db877ffed8e457763b317cf880df4ae19109b9f77d277cf6e0" "checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" -"checksum proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" -"checksum proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" -"checksum proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548" +"checksum proc-macro-hack 0.5.15 (registry+https://github.com/rust-lang/crates.io-index)" = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63" +"checksum proc-macro-nested 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8e946095f9d3ed29ec38de908c22f95d9ac008e424c7bcae54c75a79c527c694" +"checksum proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)" = "df246d292ff63439fea9bc8c0a270bed0e390d5ebd4db4ba15aba81111b5abe3" "checksum prometheus 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5567486d5778e2c6455b1b90ff1c558f29e751fc018130fa182e15828e728af1" -"checksum protobuf 2.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6686ddd96a8dbe2687b5f2a687b2cfb520854010ec480f2d74c32e7c9873d3c5" +"checksum protobuf 2.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8e86d370532557ae7573551a1ec8235a0f8d6cb276c7c9e6aa490b511c447485" "checksum quick-error 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -"checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +"checksum quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" "checksum raft 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "43648985159f2c4c25f934182a63b0dd4c5cc06a5ae96e5d40e12c3a3785abde" "checksum rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" "checksum rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9" "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" "checksum rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" "checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -"checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" +"checksum rand_chacha 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" "checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" "checksum rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" "checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" @@ -2237,35 +2324,33 @@ dependencies = [ "checksum rayon-core 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9" "checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" -"checksum regex 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b5508c1941e4e7cb19965abef075d35a9a8b5cdf0846f30b4050e9b55dc55e87" +"checksum regex 1.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" "checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e" -"checksum regex-syntax 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e734e891f5b408a29efbf8309e656876276f49ab6a6ac208600b4419bd893d90" +"checksum regex-syntax 0.6.17 (registry+https://github.com/rust-lang/crates.io-index)" = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" "checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" -"checksum rental 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "8545debe98b2b139fb04cad8618b530e9b07c152d99a5de83c860b877d67847f" -"checksum rental-impl 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "475e68978dc5b743f2f40d8e0a8fdc83f1c5e78cbf4b8fa5e74e73beebc340de" +"checksum rocksdb 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "12069b106981c6103d3eab7dd1c86751482d0779a520b7c14954c8b586c1e643" "checksum route-recognizer 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "ea509065eb0b3c446acdd0102f0d46567dc30902dc0be91d6552035d92b0f4f8" "checksum router 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dc63b6f3b8895b0d04e816b2b1aa58fdba2d5acca3cbb8f0ab8e017347d57397" "checksum rust-stemmers 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" -"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -"checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" +"checksum rustc-hash 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +"checksum ryu 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76" "checksum safemem 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" "checksum same-file 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -"checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" -"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -"checksum serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)" = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" -"checksum serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)" = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" -"checksum serde_json 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)" = "9371ade75d4c2d6cb154141b9752cf3781ec9c05e0e5cf35060e1e70ee7b9c25" +"checksum scopeguard 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +"checksum serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)" = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399" +"checksum serde_derive 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)" = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c" +"checksum serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)" = "da07b57ee2623368351e9a0488bb0b261322a15a6e0ae53e243cbdc0f4208da9" +"checksum shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" "checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" -"checksum smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44e59e0c9fa00817912ae6e4e6e3c4fe04455e75699d06eedc7d85917ed8e8f4" +"checksum smallvec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05720e22615919e4734f6a99ceae50d00226c3c5aca406e102ebc33298214e0a" "checksum snap 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f7fb9b0bb877b35a1cc1474a3b43d9c226a2625311760cdda2cbccbc0c7a8376" "checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" "checksum stringreader 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "913e7b03d63752f6cdd2df77da36749d82669904798fe8944b9ec3d23f159905" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" -"checksum syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "af6f3550d8dff9ef7dc34d384ac6f107e5d31c8f57d9f28e0081503f547ac8f5" +"checksum syn 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03" "checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" "checksum tantivy 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e1d2fbfa82ab829208e5f03f4d2c177b8a126252ab4f80ed232e1064770efb" "checksum tantivy-fst 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "38878efb477cf2efb7d9112b12b230c27d32abdfec4bea5e66095733f2928610" @@ -2275,13 +2360,13 @@ dependencies = [ "checksum termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" "checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" -"checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" +"checksum time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" "checksum traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" "checksum typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1410f6f91f21d1612654e7cc69193b0334f909dcf2c790c4826254fbb86f8887" "checksum typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "653be63c80a3296da5551e1bfd2cca35227e13cdd08c6668903ae2f4f77aa1f6" "checksum unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" -"checksum unicode-normalization 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "b561e267b2326bb4cebfc0ef9e68355c7abe6c6f522aeac2f5bf95d56c59bdcf" +"checksum unicode-normalization 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" "checksum unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" "checksum unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f30360d7979f5e9c6e6cea48af192ea8fab4afb3cf72597154b8f08935bc9c7f" @@ -2291,13 +2376,15 @@ dependencies = [ "checksum uuid 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" +"checksum version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum walkdir 2.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" "checksum wasi 0.9.0+wasi-snapshot-preview1 (registry+https://github.com/rust-lang/crates.io-index)" = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +"checksum which 3.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -"checksum winapi-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4ccfbf554c6ad11084fb7517daca16cfdcaccbdadba4fc336f032a8b12c2ad80" +"checksum winapi-util 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "fa515c5163a99cc82bab70fd3bfdd36d827be85de63737b40fcef2ce084a436e" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" diff --git a/Cargo.toml b/Cargo.toml index f0d89e8..3523b68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,51 +1,7 @@ -[package] -name = "bayard" -version = "0.6.0" -authors = ["Minoru Osuka "] -edition = "2018" -description = "Bayard is a full-text search and indexing server written in Rust." -documentation = "https://bayard-search.github.io/bayard/" -homepage = "https://github.com/bayard-search" -repository = "https://github.com/bayard-search/bayard" -readme = "README.md" -keywords = ["search", "index", "server", "information", "retrieval"] -categories = ["database"] -license = "MIT" -exclude = ["docs", "docs_src", "etc"] - -[lib] -name = "bayard" - -[[bin]] -name = "bayard" -path = "src/main.rs" - -[dependencies] -async-std = "1.4.0" -bayard-client = "0.1.0" -bayard-proto = "0.2.0" -cang-jie = "0.7.0" -clap = "2.33.0" -crossbeam-channel = "0.4.0" -ctrlc = { version = "3.1.3", features = ["termination"] } -env_logger = "0.7.1" -futures = "0.1.29" -grpcio = { version = "0.4.7", features = [ "secure" ] } -iron = "0.6.1" -jieba-rs = "0.4.10" -job_scheduler = "1.1.0" -lindera-tantivy = "0.1.0" -log = "0.4.8" -logger = "0.4.0" -num_cpus = "1.11.1" -persistent = "0.4.0" -prometheus = "0.7.0" -protobuf = "2.10.1" -raft = "0.4.3" -rand = "0.7.2" -router = "0.6.0" -serde = { version = "1.0.104", features = ["derive"] } -serde_json = "1.0.44" -stringreader = "0.1.1" -tantivy = "0.12.0" -urlencoded = "0.6.0" +[workspace] +members = [ + "bayard-server", + "bayard-client", + "bayard-rest", + "bayard", +] diff --git a/Dockerfile b/Dockerfile index 626a164..31b3b2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG RUST_VERSION=1.39.0 +ARG RUST_VERSION=1.42.0 FROM rust:${RUST_VERSION}-slim-stretch AS builder @@ -15,23 +15,25 @@ RUN set -ex \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# Cache dependencies. -COPY ./Cargo.toml ./Cargo.lock ./ -RUN mkdir -p src \ - && echo "fn main() {}" > src/main.rs \ - && touch src/lib.rs \ - && cargo build --release - COPY . ./ RUN make build FROM debian:stretch-slim -RUN mkdir -p /data WORKDIR / + +RUN set -ex \ + && apt-get update \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /data + COPY --from=builder /repo/bin /usr/local/bin COPY --from=builder /repo/etc/* /etc/ -EXPOSE 5000 + +EXPOSE 5000 7000 + ENTRYPOINT [ "bayard" ] -CMD [ "serve" ] +CMD [ "start" ] diff --git a/Makefile b/Makefile index 03ac890..07ac42b 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,9 @@ BIN_DIR ?= $(CURDIR)/bin DOCS_DIR ?= $(CURDIR)/docs -VERSION ?= - -ifeq ($(VERSION),) - VERSION = $(shell cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name=="bayard") | .version') -endif +SERVER_VERSION ?= $(shell cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name=="bayard-server") | .version') +CLIENT_VERSION ?= $(shell cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name=="bayard-client") | .version') +REST_VERSION ?= $(shell cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name=="bayard-rest") | .version') +VERSION ?= $(shell cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name=="bayard") | .version') .DEFAULT_GOAL := build @@ -15,23 +14,45 @@ clean: format: cargo fmt -build: +build: format mkdir -p $(BIN_DIR) cargo build --release cp -p ./target/release/bayard $(BIN_DIR) + cp -p ./target/release/bayard-rest $(BIN_DIR) test: cargo test -build-docker: +tag: + git tag v$(VERSION) + git push origin v$(VERSION) + +publish: format +ifeq ($(shell cargo show --json bayard-server | jq -r '.versions[].num' | grep $(SERVER_VERSION)),) + (cd bayard-server && cargo package && cargo publish) + sleep 10 +endif +ifeq ($(shell cargo show --json bayard-client | jq -r '.versions[].num' | grep $(CLIENT_VERSION)),) + (cd bayard-client && cargo package && cargo publish) + sleep 10 +endif +ifeq ($(shell cargo show --json bayard-rest | jq -r '.versions[].num' | grep $(REST_VERSION)),) + (cd bayard-rest && cargo package && cargo publish) + sleep 10 +endif +ifeq ($(shell cargo show --json bayard-client | jq -r '.versions[].num' | grep $(VERSION)),) + (cd bayard && cargo package && cargo publish) +endif + +docker-build: docker build -t bayardsearch/bayard:latest . docker tag bayardsearch/bayard:latest bayardsearch/bayard:$(VERSION) -push-docker: +docker-push: docker push bayardsearch/bayard:latest docker push bayardsearch/bayard:$(VERSION) -clean-docker: +docker-clean: docker rmi -f $(shell docker images --filter "dangling=true" -q --no-trunc) .PHONY: docs diff --git a/README.md b/README.md index 264a0be..0d1ec26 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Join the chat at https://gitter.im/bayard-search/bayard](https://badges.gitter.im/bayard-search/bayard.svg)](https://gitter.im/bayard-search/bayard?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -Bayard is a full-text search and indexing server written in [Rust](https://www.rust-lang.org/) built on top of [Tantivy](https://github.com/tantivy-search/tantivy) that implements [The Raft Consensus Algorithm](https://raft.github.io/) ([raft-rs](https://github.com/tikv/raft-rs)) and [The gRPC](https://grpc.io/) ([grpc-rs](https://github.com/tikv/grpc-rs)). +Bayard is a full-text search and indexing server written in [Rust](https://www.rust-lang.org/) built on top of [Tantivy](https://github.com/tantivy-search/tantivy) that implements [Raft Consensus Algorithm](https://raft.github.io/) and [gRPC](https://grpc.io/). Achieves consensus across all the nodes, ensures every change made to the system is made to a quorum of nodes. Bayard makes easy for programmers to develop search applications with advanced features and high availability. diff --git a/bayard-client/Cargo.toml b/bayard-client/Cargo.toml new file mode 100644 index 0000000..65bbbfc --- /dev/null +++ b/bayard-client/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "bayard-client" +version = "0.7.0" +authors = ["Minoru Osuka "] +edition = "2018" +description = "Client library for Bayard." +documentation = "https://bayard-search.github.io/bayard/" +homepage = "https://github.com/bayard-search" +repository = "https://github.com/bayard-search/bayard/bayard-client" +readme = "README.md" +keywords = ["full-text", "search", "index", "client"] +categories = ["database"] +license = "MIT" + +[dependencies] +bincode = "1.2.1" +grpcio = { version = "0.4.7", features = [ "secure" ] } +log = "0.4.8" +protobuf = "2.14.0" +raft = "0.4.3" +rand = "0.7.2" +serde_json = "1.0.44" + +bayard-proto = "0.7.0" +bayard-server = { version = "0.7.0", path = "../bayard-server" } diff --git a/bayard-client/README.md b/bayard-client/README.md new file mode 100644 index 0000000..5587860 --- /dev/null +++ b/bayard-client/README.md @@ -0,0 +1,8 @@ +# Bayard + +[![Join the chat at https://gitter.im/bayard-search/bayard](https://badges.gitter.im/bayard-search/bayard.svg)](https://gitter.im/bayard-search/bayard?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +Bayard is a full-text search and indexing server written in [Rust](https://www.rust-lang.org/) built on top of [Tantivy](https://github.com/tantivy-search/tantivy) that implements [Raft Consensus Algorithm](https://raft.github.io/) and [gRPC](https://grpc.io/). +Achieves consensus across all the nodes, ensures every change made to the system is made to a quorum of nodes. +Bayard makes easy for programmers to develop search applications with advanced features and high availability. diff --git a/bayard-client/src/index/client.rs b/bayard-client/src/index/client.rs new file mode 100644 index 0000000..182f328 --- /dev/null +++ b/bayard-client/src/index/client.rs @@ -0,0 +1,1324 @@ +use std::collections::HashMap; +use std::io::{Error, ErrorKind}; +use std::sync::Arc; + +use bincode::deserialize; +use grpcio::{ChannelBuilder, EnvBuilder}; +use log::*; +use protobuf::RepeatedField; + +use bayard_proto::proto::commonpb::State; +use bayard_proto::proto::indexpb::{ + BulkDeleteReq, BulkSetReq, CommitReq, DeleteReq, GetReq, MergeReq, MetricsReq, RollbackReq, + SchemaReq, SearchReq, SetReq, StatusReq, +}; +use bayard_proto::proto::indexpb_grpc::IndexServiceClient; +use bayard_server::raft::config::NodeAddress; +use serde_json::json; +use serde_json::Value; + +fn create_client(address: String) -> IndexServiceClient { + let env = Arc::new(EnvBuilder::new().build()); + let ch = ChannelBuilder::new(env).connect(&address); + let client = IndexServiceClient::new(ch); + client +} + +pub struct IndexClient { + address: String, // specified server address + leader_id: u64, // leader's node id + clients: HashMap>, + addresses: HashMap, + next_index: usize, + node_id: u64, // node id + client_id: u64, +} + +impl IndexClient { + pub fn new(address: &str) -> IndexClient { + let initial_id = 0; // initial node id + + let mut addresses = HashMap::new(); + addresses.insert(initial_id, address.to_string()); + + let mut clients = HashMap::new(); + let client = create_client(address.to_string()); + clients.insert(initial_id, Arc::new(client)); + + IndexClient { + address: address.to_string(), + leader_id: initial_id, + clients, + addresses, + next_index: 0, + node_id: initial_id, + client_id: rand::random(), + } + } + + pub fn get(&mut self, id: String) -> Result { + let mut req = GetReq::new(); + req.set_client_id(self.client_id); + req.set_id(id); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.node_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.node_id), + )); + } + }; + + let reply = match client.get(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to request getting document: id={}", req.get_id()), + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.index_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + // change node id + let keys: Vec = self.addresses.keys().map(|i| i.clone()).collect(); + self.next_index = (self.next_index + 1) % self.addresses.len(); + self.node_id = keys.get(self.next_index).unwrap().clone(); + + match reply.get_state() { + State::OK => { + self.leader_id = reply.get_leader_id(); + return Ok(String::from(reply.get_doc())); + } + State::NOT_FOUND => { + self.leader_id = reply.get_leader_id(); + return Err(Error::new( + ErrorKind::NotFound, + format!("document not found: id={}", req.get_id()), + )); + } + _ => { + cnt_retry += 1; + warn!("failed to get document: id={}", req.get_id()); + } + } + } + } + + pub fn search( + &mut self, + query: &str, + from: u64, + limit: u64, + exclude_count: bool, + exclude_docs: bool, + facet_field: &str, + facet_prefixes: Vec, + ) -> Result { + let mut req = SearchReq::new(); + req.set_client_id(self.client_id); + req.set_query(query.to_string()); + req.set_from(from); + req.set_limit(limit); + req.set_exclude_count(exclude_count); + req.set_exclude_docs(exclude_docs); + req.set_facet_field(facet_field.to_string()); + req.set_facet_prefixes(RepeatedField::from_vec(facet_prefixes)); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.node_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.node_id), + )); + } + }; + + let reply = match client.search(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to request searching documents: req={:?}", req), + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.index_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + // change node id + let keys: Vec = self.addresses.keys().map(|i| i.clone()).collect(); + self.next_index = (self.next_index + 1) % self.addresses.len(); + self.node_id = keys.get(self.next_index).unwrap().clone(); + + match reply.get_state() { + State::OK => { + return Ok(String::from(reply.get_search_result())); + } + _ => { + cnt_retry += 1; + warn!("failed to search documents: req={:?}", req); + } + } + } + } + + pub fn set(&mut self, doc: String) -> Result<(), std::io::Error> { + let mut req = SetReq::new(); + req.set_client_id(self.client_id); + req.set_doc(doc); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.leader_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.leader_id), + )); + } + }; + + let reply = match client.set(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to request setting document: doc={}", req.get_doc()), + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.index_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + match reply.get_state() { + State::OK => { + self.leader_id = reply.get_leader_id(); + return Ok(()); + } + State::WRONG_LEADER => { + self.leader_id = reply.get_leader_id(); + cnt_retry += 1; + debug!("retry with a new leader: id={}", self.leader_id); + continue; + } + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to set document: doc={}", req.get_doc()), + )); + } + }; + } + } + + pub fn delete(&mut self, id: String) -> Result<(), std::io::Error> { + let mut req = DeleteReq::new(); + req.set_client_id(self.client_id); + req.set_id(id); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.leader_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.leader_id), + )); + } + }; + + let reply = match client.delete(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to request deleting document: id={}", req.get_id()), + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.index_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + match reply.get_state() { + State::OK => { + self.leader_id = reply.get_leader_id(); + return Ok(()); + } + State::WRONG_LEADER => { + self.leader_id = reply.get_leader_id(); + cnt_retry += 1; + debug!("retry with a new leader: id={}", self.leader_id); + continue; + } + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to delete document: id={}", req.get_id()), + )); + } + }; + } + } + + pub fn bulk_set(&mut self, docs: String) -> Result<(), std::io::Error> { + let mut req = BulkSetReq::new(); + req.set_client_id(self.client_id); + req.set_docs(docs); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.leader_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.leader_id), + )); + } + }; + + let reply = match client.bulk_set(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + "failed to request setting documents in bulk", + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.index_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + match reply.get_state() { + State::OK => { + self.leader_id = reply.get_leader_id(); + return Ok(()); + } + State::WRONG_LEADER => { + self.leader_id = reply.get_leader_id(); + cnt_retry += 1; + debug!("retry with a new leader: id={}", self.leader_id); + continue; + } + _ => { + return Err(Error::new( + ErrorKind::Other, + "failed to set documents in bulk", + )); + } + }; + } + } + + pub fn bulk_delete(&mut self, docs: String) -> Result<(), std::io::Error> { + let mut req = BulkDeleteReq::new(); + req.set_client_id(self.client_id); + req.set_docs(docs); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.leader_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.leader_id), + )); + } + }; + + let reply = match client.bulk_delete(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + "failed to request deleting documents in bulk", + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.index_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + match reply.get_state() { + State::OK => { + self.leader_id = reply.get_leader_id(); + return Ok(()); + } + State::WRONG_LEADER => { + self.leader_id = reply.get_leader_id(); + cnt_retry += 1; + debug!("retry with a new leader: id={}", self.leader_id); + continue; + } + _ => { + return Err(Error::new( + ErrorKind::Other, + "failed to delete documents in bulk", + )); + } + }; + } + } + + pub fn commit(&mut self) -> Result<(), std::io::Error> { + let mut req = CommitReq::new(); + req.set_client_id(self.client_id); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.leader_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.leader_id), + )); + } + }; + + let reply = match client.commit(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + "failed to request committing index", + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.index_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + match reply.get_state() { + State::OK => { + self.leader_id = reply.get_leader_id(); + return Ok(()); + } + State::WRONG_LEADER => { + self.leader_id = reply.get_leader_id(); + cnt_retry += 1; + debug!("retry with a new leader: id={}", self.leader_id); + continue; + } + _ => { + return Err(Error::new(ErrorKind::Other, "failed to commit index")); + } + }; + } + } + + pub fn rollback(&mut self) -> Result<(), std::io::Error> { + let mut req = RollbackReq::new(); + req.set_client_id(self.client_id); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.leader_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.leader_id), + )); + } + }; + + let reply = match client.rollback(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + "failed to request rolling back index", + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.index_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + match reply.get_state() { + State::OK => { + self.leader_id = reply.get_leader_id(); + return Ok(()); + } + State::WRONG_LEADER => { + self.leader_id = reply.get_leader_id(); + cnt_retry += 1; + debug!("retry with a new leader: id={}", self.leader_id); + continue; + } + _ => { + return Err(Error::new(ErrorKind::Other, "failed to rollback index")); + } + }; + } + } + + pub fn merge(&mut self) -> Result<(), std::io::Error> { + let mut req = MergeReq::new(); + req.set_client_id(self.client_id); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.leader_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.leader_id), + )); + } + }; + + let reply = match client.merge(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + "failed to request merging index", + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.index_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + match reply.get_state() { + State::OK => { + self.leader_id = reply.get_leader_id(); + return Ok(()); + } + State::WRONG_LEADER => { + self.leader_id = reply.get_leader_id(); + cnt_retry += 1; + debug!("retry with a new leader: id={}", self.leader_id); + continue; + } + _ => { + return Err(Error::new(ErrorKind::Other, "failed to merge index")); + } + }; + } + } + + pub fn schema(&mut self) -> Result { + let mut req = SchemaReq::new(); + req.set_client_id(self.client_id); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.node_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.node_id), + )); + } + }; + + let reply = match client.schema(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + "failed to request getting schema", + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.index_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + // continue to use the same node. + for (id, address) in &self.addresses.clone() { + if address == self.address.as_str() { + self.node_id = *id; + break; + } + } + + match reply.get_state() { + State::OK => { + self.leader_id = reply.get_leader_id(); + return Ok(String::from(reply.get_schema())); + } + _ => { + cnt_retry += 1; + warn!("failed to get schema"); + } + } + } + } + + pub fn metrics(&mut self) -> Result { + let mut req = MetricsReq::new(); + req.set_client_id(self.client_id); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.node_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.node_id), + )); + } + }; + + let reply = match client.metrics(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + "failed to request getting metrics", + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.index_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + // continue to use the same node. + for (id, address) in &self.addresses.clone() { + if address == self.address.as_str() { + self.node_id = *id; + break; + } + } + + match reply.get_state() { + State::OK => { + self.leader_id = reply.get_leader_id(); + return Ok(String::from(reply.get_metrics())); + } + _ => { + cnt_retry += 1; + warn!("failed to get metrics"); + } + } + } + } + + pub fn status(&mut self) -> Result { + let mut req = StatusReq::new(); + req.set_client_id(self.client_id); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.node_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.node_id), + )); + } + }; + + let reply = match client.status(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + "failed to request getting status", + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.index_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.index_address); + self.addresses + .insert(id.clone(), address.index_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.index_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + // continue to use the same node. + for (id, address) in &self.addresses.clone() { + if address == self.address.as_str() { + self.node_id = *id; + break; + } + } + + match reply.get_state() { + State::OK => { + self.leader_id = reply.get_leader_id(); + + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + + let mut nodes: Value = json!([]); + for (id, address) in address_map { + let mut node: Value = json!({}); + node.as_object_mut() + .unwrap() + .insert("id".to_string(), serde_json::to_value(&id).unwrap()); + node.as_object_mut().unwrap().insert( + "address".to_string(), + serde_json::to_value(&address).unwrap(), + ); + nodes.as_array_mut().unwrap().push(node); + } + + let mut v: Value = json!({}); + v.as_object_mut().unwrap().insert( + "status".to_string(), + serde_json::to_value(&reply.get_status()).unwrap(), + ); + v.as_object_mut().unwrap().insert( + "leader".to_string(), + serde_json::to_value(&self.leader_id).unwrap(), + ); + v.as_object_mut() + .unwrap() + .insert("nodes".to_string(), nodes); + + return Ok(v.to_string()); + } + _ => { + cnt_retry += 1; + warn!("failed to get status"); + } + } + } + } +} diff --git a/bayard-client/src/index/mod.rs b/bayard-client/src/index/mod.rs new file mode 100644 index 0000000..b9babe5 --- /dev/null +++ b/bayard-client/src/index/mod.rs @@ -0,0 +1 @@ +pub mod client; diff --git a/bayard-client/src/lib.rs b/bayard-client/src/lib.rs new file mode 100644 index 0000000..0753a50 --- /dev/null +++ b/bayard-client/src/lib.rs @@ -0,0 +1,2 @@ +pub mod index; +pub mod raft; diff --git a/bayard-client/src/raft/client.rs b/bayard-client/src/raft/client.rs new file mode 100644 index 0000000..4e219d9 --- /dev/null +++ b/bayard-client/src/raft/client.rs @@ -0,0 +1,281 @@ +use std::collections::HashMap; +use std::io::{Error, ErrorKind}; +use std::sync::Arc; + +use bincode::{deserialize, serialize}; +use grpcio::{ChannelBuilder, EnvBuilder}; +use log::*; +use raft::eraftpb::{ConfChange, ConfChangeType}; + +use bayard_proto::proto::commonpb::State; +use bayard_proto::proto::raftpb_grpc::RaftServiceClient; +use bayard_server::raft::config::NodeAddress; + +fn create_client(address: String) -> RaftServiceClient { + let env = Arc::new(EnvBuilder::new().build()); + let ch = ChannelBuilder::new(env).connect(&address); + let client = RaftServiceClient::new(ch); + client +} + +pub struct RaftClient { + leader_id: u64, // leader's index in server_ids + clients: HashMap>, + addresses: HashMap, +} + +impl RaftClient { + pub fn new(address: &str) -> RaftClient { + let initial_leader_id = 0; + + let mut addresses = HashMap::new(); + addresses.insert(initial_leader_id, address.to_string()); + + let mut clients = HashMap::new(); + let client = create_client(address.to_string()); + clients.insert(initial_leader_id, Arc::new(client)); + + RaftClient { + leader_id: initial_leader_id, + clients, + addresses, + } + } + + pub fn join( + &mut self, + id: u64, + node_address: NodeAddress, + ) -> Result, std::io::Error> { + let mut req = ConfChange::new(); + req.set_node_id(id); + req.set_change_type(ConfChangeType::AddNode); + req.set_context(serialize(&node_address).unwrap()); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.leader_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.leader_id), + )); + } + }; + + let reply = match client.change_config(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!( + "failed to join node to the cluster: id={}", + req.get_node_id() + ), + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.raft_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.raft_address); + self.addresses + .insert(id.clone(), address.raft_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.raft_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.raft_address); + self.addresses + .insert(id.clone(), address.raft_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.raft_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + match reply.get_state() { + State::OK => { + return Ok(deserialize(&reply.get_address_map()).unwrap()); + } + State::WRONG_LEADER => { + warn!( + "upddate leader id: current={}, new={}", + self.leader_id, + reply.get_leader_id() + ); + self.leader_id = reply.get_leader_id(); + cnt_retry += 1; + warn!("retry with a new leader: id={}", self.leader_id); + continue; + } + _ => { + return Err(Error::new( + ErrorKind::Other, + format!( + "failed to join node to the cluster: id={}", + req.get_node_id() + ), + )); + } + }; + } + } + + pub fn leave(&mut self, id: u64) -> Result, std::io::Error> { + let mut req = ConfChange::new(); + req.set_node_id(id); + req.set_change_type(ConfChangeType::RemoveNode); + req.set_context(vec![]); + + let max_retry = 10; + let mut cnt_retry = 0; + + loop { + if max_retry < cnt_retry { + return Err(Error::new( + ErrorKind::Other, + format!("max retry count has been exceeded: max_retry={}", max_retry), + )); + } + + let client = match self.clients.get(&self.leader_id) { + Some(c) => c, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!("failed to get client for node: id={}", self.leader_id), + )); + } + }; + + let reply = match client.change_config(&req) { + Ok(r) => r, + _ => { + return Err(Error::new( + ErrorKind::Other, + format!( + "failed to leave node from the cluster: id={}", + req.get_node_id() + ), + )); + } + }; + + // update address list and clients + if reply.get_address_map().len() > 0 { + let address_map: HashMap = + deserialize(&reply.get_address_map()).unwrap(); + // add new ids + for (id, address) in &address_map { + if let Some(grpc_address) = self.addresses.get(&id) { + if grpc_address == address.raft_address.as_str() { + debug!( + "node has not been changed: id={}, address={}", + id, grpc_address + ); + } else { + debug!("update node: id={}, address={}", id, address.raft_address); + self.addresses + .insert(id.clone(), address.raft_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.raft_address.clone())), + ); + } + } else { + debug!("add node: id={}, address={}", id, address.raft_address); + self.addresses + .insert(id.clone(), address.raft_address.clone()); + self.clients.insert( + id.clone(), + Arc::new(create_client(address.raft_address.clone())), + ); + } + } + + // remove unused ids + for (id, address) in &self.addresses.clone() { + if let Some(_) = address_map.get(&id) { + debug!("node is in use: id={}, address={}", id, address); + } else { + debug!("node is not in use: id={}, address={}", id, address); + self.addresses.remove(id); + self.clients.remove(id); + } + } + + debug!("addresses={:?}", self.addresses); + } + + match reply.get_state() { + State::OK => { + return Ok(deserialize(&reply.get_address_map()).unwrap()); + } + State::WRONG_LEADER => { + warn!( + "upddate leader id: current={}, new={}", + self.leader_id, + reply.get_leader_id() + ); + self.leader_id = reply.get_leader_id(); + cnt_retry += 1; + warn!("retry with a new leader: id={}", self.leader_id); + continue; + } + _ => { + return Err(Error::new( + ErrorKind::Other, + format!( + "failed to leave node from the cluster: id={}", + req.get_node_id() + ), + )); + } + }; + } + } + + pub fn snapshot(&mut self) -> Result<(), std::io::Error> { + // TODO + Ok(()) + } +} diff --git a/bayard-client/src/raft/mod.rs b/bayard-client/src/raft/mod.rs new file mode 100644 index 0000000..b9babe5 --- /dev/null +++ b/bayard-client/src/raft/mod.rs @@ -0,0 +1 @@ +pub mod client; diff --git a/bayard-rest/Cargo.toml b/bayard-rest/Cargo.toml new file mode 100644 index 0000000..f7842f4 --- /dev/null +++ b/bayard-rest/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "bayard-rest" +version = "0.7.0" +authors = ["Minoru Osuka "] +edition = "2018" +description = "REST API server for Bayard." +documentation = "https://bayard-search.github.io/bayard/" +homepage = "https://github.com/bayard-search" +repository = "https://github.com/bayard-search/bayard/bayard-rest" +readme = "README.md" +keywords = ["full-text", "search", "index", "server", "rest"] +categories = ["database"] +license = "MIT" + +[[bin]] +name = "bayard-rest" +path = "src/main.rs" + +[dependencies] +clap = "2.33.0" +env_logger = "0.7.1" +iron = "0.6.1" +log = "0.4.8" +logger = "0.4.0" +persistent = "0.4.0" +router = "0.6.0" +serde_json = "1.0.44" +urlencoded = "0.6.0" + +bayard-client = { version = "0.7.0", path = "../bayard-client" } diff --git a/bayard-rest/README.md b/bayard-rest/README.md new file mode 100644 index 0000000..0d1ec26 --- /dev/null +++ b/bayard-rest/README.md @@ -0,0 +1,29 @@ +# Bayard + +[![Join the chat at https://gitter.im/bayard-search/bayard](https://badges.gitter.im/bayard-search/bayard.svg)](https://gitter.im/bayard-search/bayard?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +Bayard is a full-text search and indexing server written in [Rust](https://www.rust-lang.org/) built on top of [Tantivy](https://github.com/tantivy-search/tantivy) that implements [Raft Consensus Algorithm](https://raft.github.io/) and [gRPC](https://grpc.io/). +Achieves consensus across all the nodes, ensures every change made to the system is made to a quorum of nodes. +Bayard makes easy for programmers to develop search applications with advanced features and high availability. + + +## Features + +- Full-text search/indexing +- Index replication +- Bringing up a cluster +- Command line interface is available + + +## Source code repository + +- [https://github.com/bayard-search/bayard](https://github.com/bayard-search/bayard) + +## Docker container repository + +- [https://hub.docker.com/r/bayardsearch/bayard](https://hub.docker.com/r/bayardsearch/bayard) + +## Documents + +- [https://bayard-search.github.io/bayard/](https://bayard-search.github.io/bayard/) diff --git a/bayard-rest/src/cli/mod.rs b/bayard-rest/src/cli/mod.rs new file mode 100644 index 0000000..0d2475f --- /dev/null +++ b/bayard-rest/src/cli/mod.rs @@ -0,0 +1,2 @@ +pub mod root; +pub mod start; diff --git a/bayard-rest/src/cli/root.rs b/bayard-rest/src/cli/root.rs new file mode 100644 index 0000000..01f004e --- /dev/null +++ b/bayard-rest/src/cli/root.rs @@ -0,0 +1,63 @@ +use clap::{App, AppSettings, Arg, SubCommand}; + +use crate::cli::start::run_start_cli; + +pub fn run_root_cli() -> Result<(), std::io::Error> { + let app = App::new(crate_name!()) + .setting(AppSettings::DeriveDisplayOrder) + .setting(AppSettings::SubcommandRequiredElseHelp) + .version(crate_version!()) + .author(crate_authors!()) + .about("Manage Bayard REST server") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .subcommand( + SubCommand::with_name("start") + .name("start") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Start REST server") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("HOST") + .help("Node address.") + .short("H") + .long("host") + .value_name("HOST") + .default_value("0.0.0.0") + .takes_value(true), + ) + .arg( + Arg::with_name("PORT") + .help("HTTP service port number.") + .short("p") + .long("port") + .value_name("PORT") + .default_value("8000") + .takes_value(true), + ) + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ), + ) + .get_matches(); + + let (subcommand, some_options) = app.subcommand(); + let options = some_options.unwrap(); + let run_cli = match subcommand { + "start" => run_start_cli, + _ => panic!("Subcommand {} is unknown", subcommand), + }; + + run_cli(options) +} diff --git a/bayard-rest/src/cli/start.rs b/bayard-rest/src/cli/start.rs new file mode 100644 index 0000000..8a66176 --- /dev/null +++ b/bayard-rest/src/cli/start.rs @@ -0,0 +1,57 @@ +use clap::ArgMatches; +use iron::{Chain, Iron}; +use logger::Logger; +use persistent::Write; +use router::Router; + +use bayard_client::index::client::IndexClient; + +use crate::handler::bulk_delete::bulk_delete; +use crate::handler::bulk_set::bulk_set; +use crate::handler::commit::commit; +use crate::handler::delete::delete; +use crate::handler::get::get; +use crate::handler::merge::merge; +use crate::handler::metrics::metrics; +use crate::handler::rollback::rollback; +use crate::handler::schema::schema; +use crate::handler::search::search; +use crate::handler::set::set; +use crate::handler::status::status; +use crate::handler::Client; +use crate::log::set_http_logger; + +pub fn run_start_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_http_logger(); + + let host = matches.value_of("HOST").unwrap(); + let port = matches.value_of("PORT").unwrap().parse::().unwrap(); + let server = matches.value_of("SERVER").unwrap(); + + let (logger_before, logger_after) = Logger::new(None); + + let index_client = IndexClient::new(server); + + let mut router = Router::new(); + router.get("/v1/documents/:id", get, "get"); + router.put("/v1/documents/:id", set, "set"); + router.delete("/v1/documents/:id", delete, "delete"); + router.put("/v1/documents", bulk_set, "bulk_set"); + router.delete("/v1/documents", bulk_delete, "bulk_delete"); + router.get("/v1/commit", commit, "commit"); + router.get("/v1/rollback", rollback, "rollback"); + router.get("/v1/merge", merge, "merge"); + router.get("/v1/schema", schema, "schema"); + router.post("/v1/search", search, "search"); + router.get("/v1/status", status, "status"); + router.get("/v1/metrics", metrics, "metrics"); + + let mut chain = Chain::new(router); + chain.link_before(logger_before); + chain.link(Write::::both(index_client)); + chain.link_after(logger_after); + + Iron::new(chain).http(format!("{}:{}", host, port)).unwrap(); + + Ok(()) +} diff --git a/bayard-rest/src/handler/bulk_delete.rs b/bayard-rest/src/handler/bulk_delete.rs new file mode 100644 index 0000000..3ae93b2 --- /dev/null +++ b/bayard-rest/src/handler/bulk_delete.rs @@ -0,0 +1,28 @@ +use std::io::Read; + +use iron::headers::ContentType; +use iron::prelude::*; +use iron::{status, IronError, IronResult, Request, Response}; +use persistent::Write; + +use crate::handler::Client; + +pub fn bulk_delete(req: &mut Request) -> IronResult { + let mut docs = String::new(); + req.body + .read_to_string(&mut docs) + .expect("Failed to read line"); + + let client = req.get::>().unwrap(); + let mut index_client = client.lock().unwrap(); + match index_client.bulk_delete(docs) { + Ok(()) => Ok(Response::with((ContentType::json().0, status::Ok, ""))), + Err(e) => Err(IronError::new( + e, + ( + status::InternalServerError, + "failed to delete documents in bulk", + ), + )), + } +} diff --git a/bayard-rest/src/handler/bulk_set.rs b/bayard-rest/src/handler/bulk_set.rs new file mode 100644 index 0000000..0261290 --- /dev/null +++ b/bayard-rest/src/handler/bulk_set.rs @@ -0,0 +1,28 @@ +use std::io::Read; + +use iron::headers::ContentType; +use iron::prelude::*; +use iron::{status, IronError, IronResult, Request, Response}; +use persistent::Write; + +use crate::handler::Client; + +pub fn bulk_set(req: &mut Request) -> IronResult { + let mut docs = String::new(); + req.body + .read_to_string(&mut docs) + .expect("Failed to read line"); + + let client = req.get::>().unwrap(); + let mut index_client = client.lock().unwrap(); + match index_client.bulk_set(docs) { + Ok(()) => Ok(Response::with((ContentType::json().0, status::Ok, ""))), + Err(e) => Err(IronError::new( + e, + ( + status::InternalServerError, + "failed to set documents in bulk", + ), + )), + } +} diff --git a/bayard-rest/src/handler/commit.rs b/bayard-rest/src/handler/commit.rs new file mode 100644 index 0000000..da944b3 --- /dev/null +++ b/bayard-rest/src/handler/commit.rs @@ -0,0 +1,18 @@ +use iron::headers::ContentType; +use iron::prelude::*; +use iron::{status, IronError, IronResult, Request, Response}; +use persistent::Write; + +use crate::handler::Client; + +pub fn commit(req: &mut Request) -> IronResult { + let client = req.get::>().unwrap(); + let mut index_client = client.lock().unwrap(); + match index_client.commit() { + Ok(()) => Ok(Response::with((ContentType::plaintext().0, status::Ok, ""))), + Err(e) => Err(IronError::new( + e, + (status::InternalServerError, "failed to commit index"), + )), + } +} diff --git a/bayard-rest/src/handler/delete.rs b/bayard-rest/src/handler/delete.rs new file mode 100644 index 0000000..3754e35 --- /dev/null +++ b/bayard-rest/src/handler/delete.rs @@ -0,0 +1,27 @@ +use iron::headers::ContentType; +use iron::prelude::*; +use iron::{status, IronError, IronResult, Request, Response}; +use persistent::Write; +use router::Router; + +use crate::handler::Client; + +pub fn delete(req: &mut Request) -> IronResult { + let id = req + .extensions + .get::() + .unwrap() + .find("id") + .unwrap_or("") + .to_owned(); + + let client = req.get::>().unwrap(); + let mut index_client = client.lock().unwrap(); + match index_client.delete(id) { + Ok(()) => Ok(Response::with((ContentType::json().0, status::Ok, ""))), + Err(e) => Err(IronError::new( + e, + (status::InternalServerError, "failed to delete document"), + )), + } +} diff --git a/bayard-rest/src/handler/get.rs b/bayard-rest/src/handler/get.rs new file mode 100644 index 0000000..043199c --- /dev/null +++ b/bayard-rest/src/handler/get.rs @@ -0,0 +1,27 @@ +use iron::headers::ContentType; +use iron::prelude::*; +use iron::{status, IronError, IronResult, Request, Response}; +use persistent::Write; +use router::Router; + +use crate::handler::Client; + +pub fn get(req: &mut Request) -> IronResult { + let id = req + .extensions + .get::() + .unwrap() + .find("id") + .unwrap_or("") + .to_owned(); + + let client = req.get::>().unwrap(); + let mut index_client = client.lock().unwrap(); + match index_client.get(id) { + Ok(s) => Ok(Response::with((ContentType::json().0, status::Ok, s))), + Err(e) => Err(IronError::new( + e, + (status::InternalServerError, "failed to get document"), + )), + } +} diff --git a/bayard-rest/src/handler/merge.rs b/bayard-rest/src/handler/merge.rs new file mode 100644 index 0000000..5dec8c2 --- /dev/null +++ b/bayard-rest/src/handler/merge.rs @@ -0,0 +1,18 @@ +use iron::headers::ContentType; +use iron::prelude::*; +use iron::{status, IronError, IronResult, Request, Response}; +use persistent::Write; + +use crate::handler::Client; + +pub fn merge(req: &mut Request) -> IronResult { + let client = req.get::>().unwrap(); + let mut index_client = client.lock().unwrap(); + match index_client.merge() { + Ok(()) => Ok(Response::with((ContentType::plaintext().0, status::Ok, ""))), + Err(e) => Err(IronError::new( + e, + (status::InternalServerError, "failed to merge index"), + )), + } +} diff --git a/bayard-rest/src/handler/metrics.rs b/bayard-rest/src/handler/metrics.rs new file mode 100644 index 0000000..84972c8 --- /dev/null +++ b/bayard-rest/src/handler/metrics.rs @@ -0,0 +1,18 @@ +use iron::headers::ContentType; +use iron::prelude::*; +use iron::{status, IronError, IronResult, Request, Response}; +use persistent::Write; + +use crate::handler::Client; + +pub fn metrics(req: &mut Request) -> IronResult { + let client = req.get::>().unwrap(); + let mut index_client = client.lock().unwrap(); + match index_client.metrics() { + Ok(s) => Ok(Response::with((ContentType::plaintext().0, status::Ok, s))), + Err(e) => Err(IronError::new( + e, + (status::InternalServerError, "failed to get metrics"), + )), + } +} diff --git a/bayard-rest/src/handler/mod.rs b/bayard-rest/src/handler/mod.rs new file mode 100644 index 0000000..3a7e918 --- /dev/null +++ b/bayard-rest/src/handler/mod.rs @@ -0,0 +1,22 @@ +use bayard_client::index::client::IndexClient; +use iron::typemap::Key; + +pub mod bulk_delete; +pub mod bulk_set; +pub mod commit; +pub mod delete; +pub mod get; +pub mod merge; +pub mod metrics; +pub mod rollback; +pub mod schema; +pub mod search; +pub mod set; +pub mod status; + +#[derive(Copy, Clone)] +pub struct Client; + +impl Key for Client { + type Value = IndexClient; +} diff --git a/bayard-rest/src/handler/rollback.rs b/bayard-rest/src/handler/rollback.rs new file mode 100644 index 0000000..28a92e7 --- /dev/null +++ b/bayard-rest/src/handler/rollback.rs @@ -0,0 +1,18 @@ +use iron::headers::ContentType; +use iron::prelude::*; +use iron::{status, IronError, IronResult, Request, Response}; +use persistent::Write; + +use crate::handler::Client; + +pub fn rollback(req: &mut Request) -> IronResult { + let client = req.get::>().unwrap(); + let mut index_client = client.lock().unwrap(); + match index_client.rollback() { + Ok(()) => Ok(Response::with((ContentType::plaintext().0, status::Ok, ""))), + Err(e) => Err(IronError::new( + e, + (status::InternalServerError, "failed to rollback index"), + )), + } +} diff --git a/bayard-rest/src/handler/schema.rs b/bayard-rest/src/handler/schema.rs new file mode 100644 index 0000000..3e8f2f9 --- /dev/null +++ b/bayard-rest/src/handler/schema.rs @@ -0,0 +1,18 @@ +use iron::headers::ContentType; +use iron::prelude::*; +use iron::{status, IronError, IronResult, Request, Response}; +use persistent::Write; + +use crate::handler::Client; + +pub fn schema(req: &mut Request) -> IronResult { + let client = req.get::>().unwrap(); + let mut index_client = client.lock().unwrap(); + match index_client.schema() { + Ok(s) => Ok(Response::with((ContentType::json().0, status::Ok, s))), + Err(e) => Err(IronError::new( + e, + (status::InternalServerError, "failed to get schema"), + )), + } +} diff --git a/bayard-rest/src/handler/search.rs b/bayard-rest/src/handler/search.rs new file mode 100644 index 0000000..e95e6e5 --- /dev/null +++ b/bayard-rest/src/handler/search.rs @@ -0,0 +1,61 @@ +use iron::headers::ContentType; +use iron::prelude::*; +use iron::{status, IronError, IronResult, Request, Response}; +use persistent::Write; +use urlencoded::UrlEncodedQuery; + +use crate::handler::Client; + +pub fn search(req: &mut Request) -> IronResult { + let map = req.get_ref::().unwrap().to_owned(); + let query = map.get("query").unwrap().get(0).unwrap(); + + let mut from = 0; + if map.contains_key("from") { + from = map + .get("from") + .unwrap() + .get(0) + .unwrap_or(&String::from("0")) + .parse::() + .unwrap(); + } + let mut limit = 10; + if map.contains_key("limit") { + limit = map + .get("limit") + .unwrap() + .get(0) + .unwrap_or(&String::from("10")) + .parse::() + .unwrap(); + } + let exclude_count = map.contains_key("exclude_count"); + let exclude_docs = map.contains_key("exclude_docs"); + let mut facet_field: &str = ""; + if map.contains_key("facet_field") { + facet_field = map.get("facet_field").unwrap().get(0).unwrap(); + } + let mut facet_prefixes = Vec::new(); + if map.contains_key("facet_prefix") { + facet_prefixes = map.get("facet_prefix").cloned().unwrap(); + } + + let client = req.get::>().unwrap(); + let mut index_client = client.lock().unwrap(); + match index_client.search( + query, + from, + limit, + exclude_count, + exclude_docs, + facet_field, + facet_prefixes, + ) { + Ok(s) => Ok(Response::with((ContentType::json().0, status::Ok, s))), + Err(e) => Err(IronError::new( + e, + (status::InternalServerError, "failed to search index"), + )), + } +} diff --git a/bayard-rest/src/handler/set.rs b/bayard-rest/src/handler/set.rs new file mode 100644 index 0000000..f4ef03b --- /dev/null +++ b/bayard-rest/src/handler/set.rs @@ -0,0 +1,38 @@ +use std::io::Read; + +use iron::headers::ContentType; +use iron::prelude::*; +use iron::{status, IronError, IronResult, Request, Response}; +use persistent::Write; +use router::Router; +use serde_json::Value; + +use crate::handler::Client; + +pub fn set(req: &mut Request) -> IronResult { + let id = req + .extensions + .get::() + .unwrap() + .find("id") + .unwrap_or("") + .to_owned(); + let mut body = String::new(); + req.body + .read_to_string(&mut body) + .expect("Failed to read line"); + + let mut doc_json: Value = serde_json::from_str(body.as_str()).unwrap(); + doc_json["_id"] = Value::String(id); + let doc = serde_json::to_string(&doc_json).unwrap(); + + let client = req.get::>().unwrap(); + let mut index_client = client.lock().unwrap(); + match index_client.set(doc) { + Ok(()) => Ok(Response::with((ContentType::json().0, status::Ok, ""))), + Err(e) => Err(IronError::new( + e, + (status::InternalServerError, "failed to set document"), + )), + } +} diff --git a/bayard-rest/src/handler/status.rs b/bayard-rest/src/handler/status.rs new file mode 100644 index 0000000..9545f3d --- /dev/null +++ b/bayard-rest/src/handler/status.rs @@ -0,0 +1,18 @@ +use iron::headers::ContentType; +use iron::prelude::*; +use iron::{status, IronError, IronResult, Request, Response}; +use persistent::Write; + +use crate::handler::Client; + +pub fn status(req: &mut Request) -> IronResult { + let client = req.get::>().unwrap(); + let mut index_client = client.lock().unwrap(); + match index_client.status() { + Ok(s) => Ok(Response::with((ContentType::json().0, status::Ok, s))), + Err(e) => Err(IronError::new( + e, + (status::InternalServerError, "failed to get status"), + )), + } +} diff --git a/bayard-rest/src/lib.rs b/bayard-rest/src/lib.rs new file mode 100644 index 0000000..9b3270c --- /dev/null +++ b/bayard-rest/src/lib.rs @@ -0,0 +1,6 @@ +#[macro_use] +extern crate clap; + +pub mod cli; +pub mod handler; +pub mod log; diff --git a/bayard-rest/src/log/mod.rs b/bayard-rest/src/log/mod.rs new file mode 100644 index 0000000..395f6ea --- /dev/null +++ b/bayard-rest/src/log/mod.rs @@ -0,0 +1,60 @@ +use std::env; +use std::io::Write; + +use env_logger; +use env_logger::Target; + +pub fn set_logger() { + match env::var("RUST_LOG") { + Ok(val) => { + let log_level: &str = &val; + match log_level { + "error" => { /* noop */ } + "warn" => { /* noop */ } + "info" => { /* noop */ } + "debug" => { /* noop */ } + _ => env::set_var("RUST_LOG", "info"), + } + } + Err(_e) => env::set_var("RUST_LOG", "info"), + }; + env_logger::Builder::from_default_env() + .format(|buf, record| { + let ts = buf.timestamp(); + writeln!( + buf, + "[{} {} {} {}:{}] {}", + ts, + record.level(), + record.target(), + record.file().unwrap_or("unknown"), + record.line().unwrap_or(0), + record.args(), + ) + }) + .target(Target::Stderr) + .init(); +} + +pub fn set_http_logger() { + match env::var("RUST_LOG") { + Ok(val) => { + let log_level: &str = &val; + match log_level { + "error" => { /* noop */ } + "warn" => { /* noop */ } + "info" => { /* noop */ } + "debug" => { /* noop */ } + _ => env::set_var("RUST_LOG", "info"), + } + } + Err(_e) => env::set_var("RUST_LOG", "info"), + }; + env_logger::Builder::from_default_env() + .format(|buf, record| { + let ts = buf.timestamp(); + writeln!(buf, "[{} {}] {}", ts, record.level(), record.args(),) + }) + .target(Target::Stderr) + .init(); +} diff --git a/bayard-rest/src/main.rs b/bayard-rest/src/main.rs new file mode 100644 index 0000000..ab37d82 --- /dev/null +++ b/bayard-rest/src/main.rs @@ -0,0 +1,5 @@ +use bayard_rest::cli::root::run_root_cli; + +fn main() -> Result<(), std::io::Error> { + run_root_cli() +} diff --git a/bayard-server/Cargo.toml b/bayard-server/Cargo.toml new file mode 100644 index 0000000..7fa5379 --- /dev/null +++ b/bayard-server/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "bayard-server" +version = "0.7.0" +authors = ["MinoruOsuka "] +edition = "2018" +description = "Bayard is a distributed search server." +documentation = "https://bayard-search.github.io/bayard/" +homepage = "https://github.com/bayard-search" +repository = "https://github.com/bayard-search/bayard/bayard-server" +readme = "README.md" +keywords = ["full-text", "search", "index", "server"] +categories = ["database"] +license = "MIT" + +[dependencies] +async-std = "1.4.0" +bincode = "1.2.1" +cang-jie = "0.7.0" +futures = "0.1.29" +grpcio = { version = "0.4.7", features = [ "secure" ] } +jieba-rs = "0.4.10" +lazy_static = "1.4.0" +lindera-tantivy = "0.1.0" +log = "0.4.8" +prometheus = "0.7.0" +protobuf = "2.14.0" +raft = "0.4.3" +rocksdb = "0.13.0" +serde = { version = "1.0.104", features = ["derive"] } +serde_json = "1.0.44" +stringreader = "0.1.1" +tantivy = "0.12.0" + +bayard-proto = "0.7.0" diff --git a/bayard-server/README.md b/bayard-server/README.md new file mode 100644 index 0000000..0d1ec26 --- /dev/null +++ b/bayard-server/README.md @@ -0,0 +1,29 @@ +# Bayard + +[![Join the chat at https://gitter.im/bayard-search/bayard](https://badges.gitter.im/bayard-search/bayard.svg)](https://gitter.im/bayard-search/bayard?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +Bayard is a full-text search and indexing server written in [Rust](https://www.rust-lang.org/) built on top of [Tantivy](https://github.com/tantivy-search/tantivy) that implements [Raft Consensus Algorithm](https://raft.github.io/) and [gRPC](https://grpc.io/). +Achieves consensus across all the nodes, ensures every change made to the system is made to a quorum of nodes. +Bayard makes easy for programmers to develop search applications with advanced features and high availability. + + +## Features + +- Full-text search/indexing +- Index replication +- Bringing up a cluster +- Command line interface is available + + +## Source code repository + +- [https://github.com/bayard-search/bayard](https://github.com/bayard-search/bayard) + +## Docker container repository + +- [https://hub.docker.com/r/bayardsearch/bayard](https://hub.docker.com/r/bayardsearch/bayard) + +## Documents + +- [https://bayard-search.github.io/bayard/](https://bayard-search.github.io/bayard/) diff --git a/bayard-server/src/index/mod.rs b/bayard-server/src/index/mod.rs new file mode 100644 index 0000000..221cd79 --- /dev/null +++ b/bayard-server/src/index/mod.rs @@ -0,0 +1,2 @@ +pub mod search_result; +pub mod server; diff --git a/src/util/search_result.rs b/bayard-server/src/index/search_result.rs similarity index 100% rename from src/util/search_result.rs rename to bayard-server/src/index/search_result.rs diff --git a/bayard-server/src/index/server.rs b/bayard-server/src/index/server.rs new file mode 100644 index 0000000..de65d39 --- /dev/null +++ b/bayard-server/src/index/server.rs @@ -0,0 +1,1115 @@ +use std::collections::HashMap; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::sync::mpsc::{self, Receiver, Sender}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use std::{fs, thread}; + +use async_std::task::block_on; +use futures::Future; +use grpcio::{RpcContext, UnarySink}; +use log::*; +use prometheus::{CounterVec, Encoder, HistogramVec, TextEncoder}; +use raft::storage::MemStorage; +use serde::{Deserialize, Serialize}; +use stringreader::StringReader; +use tantivy::collector::{Count, FacetCollector, MultiCollector, TopDocs}; +use tantivy::merge_policy::LogMergePolicy; +use tantivy::query::{QueryParser, TermQuery}; +use tantivy::schema::{Field, FieldType, IndexRecordOption, Schema}; +use tantivy::{Document, Index, IndexWriter, Term}; + +use bayard_proto::proto::commonpb::State; +use bayard_proto::proto::indexpb::{ + BulkDeleteReply, BulkDeleteReq, BulkSetReply, BulkSetReq, CommitReply, CommitReq, DeleteReply, + DeleteReq, GetReply, GetReq, MergeReply, MergeReq, MetricsReply, MetricsReq, RollbackReply, + RollbackReq, SchemaReply, SchemaReq, SearchReply, SearchReq, SetReply, SetReq, StatusReply, + StatusReq, +}; +use bayard_proto::proto::indexpb_grpc::IndexService; + +use crate::index::search_result::{ScoredNamedFieldDocument, SearchResult}; +use crate::raft::config; +use crate::raft::config::NodeAddress; +use crate::raft::server::RaftServer; +use crate::tokenizer::tokenizer_initializer::TokenizerInitializer; + +lazy_static! { + static ref REQUEST_COUNTER: CounterVec = register_counter_vec!( + "bayard_requests_total", + "Total number of requests.", + &["func"] + ) + .unwrap(); + static ref REQUEST_HISTOGRAM: HistogramVec = register_histogram_vec!( + "bayard_request_duration_seconds", + "The request latencies in seconds.", + &["func"] + ) + .unwrap(); + static ref APPLY_COUNTER: CounterVec = register_counter_vec!( + "bayard_applies_total", + "Total number of applies.", + &["func"] + ) + .unwrap(); + static ref APPLY_HISTOGRAM: HistogramVec = register_histogram_vec!( + "bayard_apply_duration_seconds", + "The apply latencies in seconds.", + &["func"] + ) + .unwrap(); +} + +#[derive(Clone)] +pub struct IndexServer { + id: u64, + index: Arc, + index_writer: Arc>, + sender: Sender, + seq: u64, +} + +#[derive(Serialize, Deserialize, Clone)] +pub enum Op { + Get {}, + Search {}, + Set { doc: String }, + Delete { id: String }, + BulkSet { docs: String }, + BulkDelete { docs: String }, + Commit {}, + Rollback {}, + Merge {}, + Schema {}, + Metrics {}, + Status {}, +} + +impl IndexServer { + pub fn new( + index_dir: String, + schema_file: &str, + tokenizer_file: &str, + indexer_threads: usize, + indexer_memory_size: usize, + raft_storage: MemStorage, + id: u64, + node_address: NodeAddress, + addresses: HashMap, + ) -> (IndexServer, RaftServer) { + let index_path = Path::new(&index_dir); + + let schema_content = fs::read_to_string(schema_file).unwrap(); + let schema: Schema = + serde_json::from_str(&schema_content).expect("error while reading json"); + + let index = if index_path.exists() { + Index::open_in_dir(index_path).unwrap() + } else { + fs::create_dir_all(&index_path).unwrap_or_default(); + Index::create_in_dir(index_path, schema).unwrap() + }; + + if tokenizer_file != "" { + let mut tokenizer_initializer = TokenizerInitializer::new(); + let tokenizer_content = fs::read_to_string(tokenizer_file).unwrap(); + tokenizer_initializer.configure(index.tokenizers(), tokenizer_content.as_str()); + } + + let index_writer = if indexer_threads > 0 { + index + .writer_with_num_threads(indexer_threads, indexer_memory_size) + .unwrap() + } else { + index.writer(indexer_memory_size).unwrap() + }; + index_writer.set_merge_policy(Box::new(LogMergePolicy::default())); + + let (rs, rr) = mpsc::channel(); + let (apply_s, apply_r) = mpsc::channel(); + thread::spawn(move || { + config::init_and_run(raft_storage, rr, apply_s, id, node_address, addresses); + }); + + let index_server = IndexServer { + id, + index: Arc::new(index), + index_writer: Arc::new(Mutex::new(index_writer)), + sender: rs.clone(), + seq: 0, + }; + let raft_server = RaftServer::new(rs); + + let index = index_server.index.clone(); + let index_writer = index_server.index_writer.clone(); + thread::spawn(move || { + apply_daemon(apply_r, index, index_writer); + }); + + return (index_server, raft_server); + } +} + +impl IndexService for IndexServer { + fn get(&mut self, ctx: RpcContext, req: GetReq, sink: UnarySink) { + let (s1, r1) = mpsc::channel(); + let id = self.id.clone(); + let index = Arc::clone(&self.index); + let sender = self.sender.clone(); + let op = Op::Get {}; + let seq = self.seq; + self.seq += 1; + + debug!("get: req={:?}", req); + REQUEST_COUNTER.with_label_values(&["get"]).inc(); + let timer = REQUEST_HISTOGRAM.with_label_values(&["get"]).start_timer(); + + sender + .send(config::Msg::Propose { + seq, + op, + cb: Box::new(move |leader_id: i32, addresses: Vec| { + let t = Term::from_field_text( + index.schema().get_field("_id").unwrap(), + req.get_id(), + ); + let tq = TermQuery::new(t, IndexRecordOption::Basic); + let searcher = index.reader().unwrap().searcher(); + + let (state, doc) = match searcher.search(&tq, &TopDocs::with_limit(10)) { + Ok(top_docs) => { + if top_docs.len() > 0 { + let mut doc = Document::default(); + for (_score, doc_address) in top_docs { + doc = searcher.doc(doc_address).unwrap(); + } + let named_doc = index.schema().to_named_doc(&doc); + (State::OK, serde_json::to_string(&named_doc).unwrap()) + } else { + (State::NOT_FOUND, "".to_string()) + } + } + Err(_e) => (State::IO_ERROR, "".to_string()), + }; + + let mut reply = GetReply::new(); + reply.set_state(state); + if leader_id >= 0 { + // not a leader + reply.set_leader_id(leader_id as u64); + } else { + // a leader + reply.set_leader_id(id); + } + reply.set_doc(doc); + reply.set_address_map(addresses); + s1.send(reply).expect("cb channel closed"); + }), + }) + .unwrap(); + + let reply = match r1.recv_timeout(Duration::from_secs(2)) { + Ok(r) => r, + Err(_e) => { + let mut r = GetReply::new(); + r.set_state(State::IO_ERROR); + r + } + }; + + let f = sink + .success(reply.clone()) + .map_err(move |err| error!("Failed to reply get: {:?}", err)); + ctx.spawn(f); + + timer.observe_duration(); + } + + fn search(&mut self, ctx: RpcContext, req: SearchReq, sink: UnarySink) { + let (s1, r1) = mpsc::channel(); + let id = self.id.clone(); + let index = Arc::clone(&self.index); + let sender = self.sender.clone(); + let op = Op::Search {}; + let seq = self.seq; + self.seq += 1; + + debug!("search: req={:?}", req); + REQUEST_COUNTER.with_label_values(&["search"]).inc(); + let timer = REQUEST_HISTOGRAM + .with_label_values(&["search"]) + .start_timer(); + + sender + .send(config::Msg::Propose { + seq, + op, + cb: Box::new(move |leader_id: i32, addresses: Vec| { + let default_fields: Vec = index + .schema() + .fields() + .flat_map(|(field, field_entry)| { + if let FieldType::Str(text_field_options) = field_entry.field_type() { + if text_field_options.get_indexing_options().is_some() { + return Some(field); + } + } + None + }) + .collect(); + let limit = req.get_from() + req.get_limit(); + let query_parser = QueryParser::for_index(&index, default_fields); + let query = query_parser.parse_query(req.query.as_str()).unwrap(); + let searcher = index.reader().unwrap().searcher(); + let mut multi_collector = MultiCollector::new(); + let count_handle = if req.get_exclude_count() { + None + } else { + Some(multi_collector.add_collector(Count)) + }; + let top_docs_handle = if req.get_exclude_docs() { + None + } else { + Some(multi_collector.add_collector(TopDocs::with_limit(limit as usize))) + }; + let facet_handle = if req.get_facet_field().is_empty() { + None + } else { + let mut facet_collector = FacetCollector::for_field( + index.schema().get_field(req.get_facet_field()).unwrap(), + ); + for facet_prefix in req.get_facet_prefixes() { + facet_collector.add_facet(facet_prefix); + } + Some(multi_collector.add_collector(facet_collector)) + }; + + // search index + let (state, search_result) = match searcher.search(&query, &multi_collector) { + Ok(mut multi_fruit) => { + // count + let mut count: i64 = -1; + if let Some(ch) = count_handle { + count = ch.extract(&mut multi_fruit) as i64; + } + + // docs + let mut top_docs = Vec::new(); + if let Some(tdh) = top_docs_handle { + top_docs = tdh.extract(&mut multi_fruit); + } + + // facet + let mut facet: HashMap> = HashMap::new(); + if let Some(fh) = facet_handle { + let facet_counts = fh.extract(&mut multi_fruit); + let mut facet_kv: HashMap = HashMap::new(); + for facet_prefix in req.get_facet_prefixes() { + for (facet_key, facet_value) in facet_counts.get(facet_prefix) { + facet_kv.insert(facet_key.to_string(), facet_value); + } + } + facet.insert(req.get_facet_field().to_string(), facet_kv); + } + + // docs + let mut docs: Vec = Vec::new(); + let mut doc_pos: u64 = 0; + for (score, doc_address) in top_docs { + if doc_pos >= req.get_from() { + let doc = searcher.doc(doc_address).unwrap(); + let named_doc = index.schema().to_named_doc(&doc); + let scored_doc = ScoredNamedFieldDocument { + fields: named_doc, + score, + }; + docs.push(scored_doc); + } + doc_pos += 1; + } + + // search result + let search_result = SearchResult { docs, count, facet }; + + (State::OK, serde_json::to_string(&search_result).unwrap()) + } + Err(_e) => (State::IO_ERROR, "".to_string()), + }; + + let mut reply = SearchReply::new(); + reply.set_state(state); + if leader_id >= 0 { + // not a leader + reply.set_leader_id(leader_id as u64); + } else { + // a leader + reply.set_leader_id(id); + } + reply.set_search_result(search_result); + reply.set_address_map(addresses); + s1.send(reply).expect("cb channel closed"); + }), + }) + .unwrap(); + + let reply = match r1.recv_timeout(Duration::from_secs(2)) { + Ok(r) => r, + Err(_e) => { + let mut r = SearchReply::new(); + r.set_state(State::IO_ERROR); + r + } + }; + + let f = sink + .success(reply.clone()) + .map_err(move |err| error!("Failed to reply search: {:?}", err)); + ctx.spawn(f); + + timer.observe_duration(); + } + + fn set(&mut self, ctx: RpcContext, req: SetReq, sink: UnarySink) { + let (s1, r1) = mpsc::channel(); + let id = self.id.clone(); + let sender = self.sender.clone(); + let op = Op::Set { + doc: String::from(req.get_doc()), + }; + let seq = self.seq; + self.seq += 1; + + debug!("set: req={:?}", req); + REQUEST_COUNTER.with_label_values(&["set"]).inc(); + let timer = REQUEST_HISTOGRAM.with_label_values(&["set"]).start_timer(); + + sender + .send(config::Msg::Propose { + seq, + op, + cb: Box::new(move |leader_id: i32, addresses: Vec| { + let mut reply = SetReply::new(); + if leader_id >= 0 { + // not a leader + reply.set_state(State::WRONG_LEADER); + reply.set_leader_id(leader_id as u64); + } else { + // a leader + reply.set_state(State::OK); + reply.set_leader_id(id); + } + reply.set_address_map(addresses); + // done job, wake + s1.send(reply).expect("cb channel closed"); + }), + }) + .unwrap(); + + let reply = match r1.recv_timeout(Duration::from_secs(2)) { + Ok(r) => r, + Err(_e) => { + let mut r = SetReply::new(); + r.set_state(State::IO_ERROR); + r + } + }; + + let f = sink + .success(reply.clone()) + .map_err(move |err| error!("Failed to reply set: {:?}", err)); + ctx.spawn(f); + + timer.observe_duration(); + } + + fn delete(&mut self, ctx: RpcContext, req: DeleteReq, sink: UnarySink) { + let (s1, r1) = mpsc::channel(); + let id = self.id.clone(); + let sender = self.sender.clone(); + let op = Op::Delete { + id: String::from(req.get_id()), + }; + let seq = self.seq; + self.seq += 1; + + debug!("delete: req={:?}", req); + REQUEST_COUNTER.with_label_values(&["delete"]).inc(); + let timer = REQUEST_HISTOGRAM + .with_label_values(&["delete"]) + .start_timer(); + + sender + .send(config::Msg::Propose { + seq, + op, + cb: Box::new(move |leader_id: i32, addresses: Vec| { + let mut reply = DeleteReply::new(); + if leader_id >= 0 { + // not a leader + reply.set_state(State::WRONG_LEADER); + reply.set_leader_id(leader_id as u64); + } else { + // a leader + reply.set_state(State::OK); + reply.set_leader_id(id); + } + reply.set_address_map(addresses); + // done job, wake + s1.send(reply).expect("cb channel closed"); + }), + }) + .unwrap(); + + let reply = match r1.recv_timeout(Duration::from_secs(2)) { + Ok(r) => r, + Err(_e) => { + let mut r = DeleteReply::new(); + r.set_state(State::IO_ERROR); + r + } + }; + + let f = sink + .success(reply.clone()) + .map_err(move |err| error!("Failed to reply delete: {:?}", err)); + ctx.spawn(f); + + timer.observe_duration(); + } + + fn bulk_set(&mut self, ctx: RpcContext, req: BulkSetReq, sink: UnarySink) { + let (s1, r1) = mpsc::channel(); + let id = self.id.clone(); + let sender = self.sender.clone(); + let op = Op::BulkSet { + docs: String::from(req.get_docs()), + }; + let seq = self.seq; + self.seq += 1; + + debug!("bulk_set"); + REQUEST_COUNTER.with_label_values(&["bulk_set"]).inc(); + let timer = REQUEST_HISTOGRAM + .with_label_values(&["bulk_set"]) + .start_timer(); + + sender + .send(config::Msg::Propose { + seq, + op, + cb: Box::new(move |leader_id: i32, addresses: Vec| { + let mut reply = BulkSetReply::new(); + if leader_id >= 0 { + // not a leader + reply.set_state(State::WRONG_LEADER); + reply.set_leader_id(leader_id as u64); + } else { + // a leader + reply.set_state(State::OK); + reply.set_leader_id(id); + } + reply.set_address_map(addresses); + // done job, wake + s1.send(reply).expect("cb channel closed"); + }), + }) + .unwrap(); + + let reply = match r1.recv_timeout(Duration::from_secs(2)) { + Ok(r) => r, + Err(_e) => { + let mut r = BulkSetReply::new(); + r.set_state(State::IO_ERROR); + r + } + }; + + let f = sink + .success(reply.clone()) + .map_err(move |err| error!("Failed to reply bulk set: {:?}", err)); + ctx.spawn(f); + + timer.observe_duration(); + } + + fn bulk_delete( + &mut self, + ctx: RpcContext, + req: BulkDeleteReq, + sink: UnarySink, + ) { + let (s1, r1) = mpsc::channel(); + let id = self.id.clone(); + let sender = self.sender.clone(); + let op = Op::BulkDelete { + docs: String::from(req.get_docs()), + }; + let seq = self.seq; + self.seq += 1; + + debug!("bulk_delete"); + REQUEST_COUNTER.with_label_values(&["bulk_delete"]).inc(); + let timer = REQUEST_HISTOGRAM + .with_label_values(&["bulk_delete"]) + .start_timer(); + + sender + .send(config::Msg::Propose { + seq, + op, + cb: Box::new(move |leader_id: i32, addresses: Vec| { + let mut reply = BulkDeleteReply::new(); + if leader_id >= 0 { + // not a leader + reply.set_state(State::WRONG_LEADER); + reply.set_leader_id(leader_id as u64); + } else { + // a leader + reply.set_state(State::OK); + reply.set_leader_id(id); + } + reply.set_address_map(addresses); + // done job, wake + s1.send(reply).expect("cb channel closed"); + }), + }) + .unwrap(); + + let reply = match r1.recv_timeout(Duration::from_secs(2)) { + Ok(r) => r, + Err(_e) => { + let mut r = BulkDeleteReply::new(); + r.set_state(State::IO_ERROR); + r + } + }; + + let f = sink + .success(reply.clone()) + .map_err(move |err| error!("Failed to reply bulk delete: {:?}", err)); + ctx.spawn(f); + + timer.observe_duration(); + } + + fn commit(&mut self, ctx: RpcContext, req: CommitReq, sink: UnarySink) { + let (s1, r1) = mpsc::channel(); + let id = self.id.clone(); + let sender = self.sender.clone(); + let op = Op::Commit {}; + let seq = self.seq; + self.seq += 1; + + debug!("commit: req={:?}", req); + REQUEST_COUNTER.with_label_values(&["commit"]).inc(); + let timer = REQUEST_HISTOGRAM + .with_label_values(&["commit"]) + .start_timer(); + + sender + .send(config::Msg::Propose { + seq, + op, + cb: Box::new(move |leader_id: i32, addresses: Vec| { + let mut reply = CommitReply::new(); + if leader_id >= 0 { + // not a leader + reply.set_state(State::WRONG_LEADER); + reply.set_leader_id(leader_id as u64); + } else { + // a leader + reply.set_state(State::OK); + reply.set_leader_id(id); + } + reply.set_address_map(addresses); + // done job, wake + s1.send(reply).expect("cb channel closed"); + }), + }) + .unwrap(); + + let reply = match r1.recv_timeout(Duration::from_secs(2)) { + Ok(r) => r, + Err(_e) => { + let mut r = CommitReply::new(); + r.set_state(State::IO_ERROR); + r + } + }; + + let f = sink + .success(reply.clone()) + .map_err(move |err| error!("Failed to reply commit: {:?}", err)); + ctx.spawn(f); + + timer.observe_duration(); + } + + fn rollback(&mut self, ctx: RpcContext, req: RollbackReq, sink: UnarySink) { + let (s1, r1) = mpsc::channel(); + let id = self.id.clone(); + let sender = self.sender.clone(); + let op = Op::Rollback {}; + let seq = self.seq; + self.seq += 1; + + debug!("rollback: req={:?}", req); + REQUEST_COUNTER.with_label_values(&["rollback"]).inc(); + let timer = REQUEST_HISTOGRAM + .with_label_values(&["rollback"]) + .start_timer(); + + sender + .send(config::Msg::Propose { + seq, + op, + cb: Box::new(move |leader_id: i32, addresses: Vec| { + let mut reply = RollbackReply::new(); + if leader_id >= 0 { + // not a leader + reply.set_state(State::WRONG_LEADER); + reply.set_leader_id(leader_id as u64); + } else { + // a leader + reply.set_state(State::OK); + reply.set_leader_id(id); + } + reply.set_address_map(addresses); + // done job, wake + s1.send(reply).expect("cb channel closed"); + }), + }) + .unwrap(); + + let reply = match r1.recv_timeout(Duration::from_secs(2)) { + Ok(r) => r, + Err(_e) => { + let mut r = RollbackReply::new(); + r.set_state(State::IO_ERROR); + r + } + }; + + let f = sink + .success(reply.clone()) + .map_err(move |err| error!("Failed to reply commit: {:?}", err)); + ctx.spawn(f); + + timer.observe_duration(); + } + + fn merge(&mut self, ctx: RpcContext, req: MergeReq, sink: UnarySink) { + let (s1, r1) = mpsc::channel(); + let id = self.id.clone(); + let sender = self.sender.clone(); + let op = Op::Merge {}; + let seq = self.seq; + self.seq += 1; + + debug!("merge: req={:?}", req); + REQUEST_COUNTER.with_label_values(&["merge"]).inc(); + let timer = REQUEST_HISTOGRAM + .with_label_values(&["merge"]) + .start_timer(); + + sender + .send(config::Msg::Propose { + seq, + op, + cb: Box::new(move |leader_id: i32, addresses: Vec| { + let mut reply = MergeReply::new(); + if leader_id >= 0 { + // not a leader + reply.set_state(State::WRONG_LEADER); + reply.set_leader_id(leader_id as u64); + } else { + // a leader + reply.set_state(State::OK); + reply.set_leader_id(id); + } + reply.set_address_map(addresses); + // done job, wake + s1.send(reply).expect("cb channel closed"); + }), + }) + .unwrap(); + + let reply = match r1.recv_timeout(Duration::from_secs(2)) { + Ok(r) => r, + Err(_e) => { + let mut r = MergeReply::new(); + r.set_state(State::IO_ERROR); + r + } + }; + + let f = sink + .success(reply.clone()) + .map_err(move |err| error!("Failed to reply commit: {:?}", err)); + ctx.spawn(f); + + timer.observe_duration(); + } + + fn schema(&mut self, ctx: RpcContext, req: SchemaReq, sink: UnarySink) { + let (s1, r1) = mpsc::channel(); + let id = self.id.clone(); + let index = Arc::clone(&self.index); + let sender = self.sender.clone(); + let op = Op::Schema {}; + let seq = self.seq; + self.seq += 1; + + debug!("schema: req={:?}", req); + REQUEST_COUNTER.with_label_values(&["schema"]).inc(); + let timer = REQUEST_HISTOGRAM + .with_label_values(&["schema"]) + .start_timer(); + + sender + .send(config::Msg::Propose { + seq, + op, + cb: Box::new(move |leader_id: i32, addresses: Vec| { + let (state, schema) = match serde_json::to_string(&index.schema()) { + Ok(s) => (State::OK, s), + _ => (State::IO_ERROR, "".to_string()), + }; + + let mut reply = SchemaReply::new(); + reply.set_state(state); + if leader_id >= 0 { + // not a leader + reply.set_leader_id(leader_id as u64); + } else { + // a leader + reply.set_leader_id(id); + } + reply.set_schema(schema); + reply.set_address_map(addresses); + s1.send(reply).expect("cb channel closed"); + }), + }) + .unwrap(); + + let reply = match r1.recv_timeout(Duration::from_secs(2)) { + Ok(r) => r, + Err(_e) => { + let mut r = SchemaReply::new(); + r.set_state(State::IO_ERROR); + r + } + }; + + let f = sink + .success(reply.clone()) + .map_err(move |err| error!("Failed to reply schema: {:?}", err)); + ctx.spawn(f); + + timer.observe_duration(); + } + + fn metrics(&mut self, ctx: RpcContext, req: MetricsReq, sink: UnarySink) { + let (s1, r1) = mpsc::channel(); + let id = self.id.clone(); + let sender = self.sender.clone(); + let op = Op::Metrics {}; + let seq = self.seq; + self.seq += 1; + + debug!("metrics: req={:?}", req); + REQUEST_COUNTER.with_label_values(&["metrics"]).inc(); + let timer = REQUEST_HISTOGRAM + .with_label_values(&["metrics"]) + .start_timer(); + + sender + .send(config::Msg::Propose { + seq, + op, + cb: Box::new(move |leader_id: i32, addresses: Vec| { + let metric_families = prometheus::gather(); + let mut buffer = Vec::::new(); + let encoder = TextEncoder::new(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + let metrics_text = String::from_utf8(buffer.clone()).unwrap(); + + let (state, metrics_text) = (State::OK, metrics_text); + + let mut reply = MetricsReply::new(); + reply.set_state(state); + if leader_id >= 0 { + // not a leader + reply.set_leader_id(leader_id as u64); + } else { + // a leader + reply.set_leader_id(id); + } + reply.set_metrics(metrics_text); + reply.set_address_map(addresses); + s1.send(reply).expect("cb channel closed"); + }), + }) + .unwrap(); + + let reply = match r1.recv_timeout(Duration::from_secs(2)) { + Ok(r) => r, + Err(_e) => { + let mut r = MetricsReply::new(); + r.set_state(State::IO_ERROR); + r + } + }; + + let f = sink + .success(reply.clone()) + .map_err(move |err| error!("Failed to reply metrics: {:?}", err)); + ctx.spawn(f); + + timer.observe_duration(); + } + + fn status(&mut self, ctx: RpcContext, req: StatusReq, sink: UnarySink) { + let (s1, r1) = mpsc::channel(); + let id = self.id.clone(); + let _index = Arc::clone(&self.index); + let sender = self.sender.clone(); + let op = Op::Status {}; + let seq = self.seq; + self.seq += 1; + + debug!("status: req={:?}", req); + REQUEST_COUNTER.with_label_values(&["status"]).inc(); + let timer = REQUEST_HISTOGRAM + .with_label_values(&["status"]) + .start_timer(); + + sender + .send(config::Msg::Propose { + seq, + op, + cb: Box::new(move |leader_id: i32, addresses: Vec| { + // TODO: check node status. for example, index, peers and etc. + let status = "OK"; + + let (state, status) = (State::OK, status.to_string()); + + let mut reply = StatusReply::new(); + reply.set_state(state); + if leader_id >= 0 { + // not a leader + reply.set_leader_id(leader_id as u64); + } else { + // a leader + reply.set_leader_id(id); + } + reply.set_status(status); + reply.set_address_map(addresses); + s1.send(reply).expect("cb channel closed"); + }), + }) + .unwrap(); + + let reply = match r1.recv_timeout(Duration::from_secs(2)) { + Ok(r) => r, + Err(_e) => { + let mut r = StatusReply::new(); + r.set_state(State::IO_ERROR); + r + } + }; + + let f = sink + .success(reply.clone()) + .map_err(move |err| error!("Failed to reply status: {:?}", err)); + ctx.spawn(f); + + timer.observe_duration(); + } +} + +fn apply_daemon(receiver: Receiver, index: Arc, index_writer: Arc>) { + loop { + let op = match receiver.recv() { + Ok(o) => o, + _ => { + debug!("apply daemon return"); + return; + } + }; + match op { + Op::Get {} => { + // noop + } + Op::Search {} => { + // noop + } + Op::Set { doc } => { + APPLY_COUNTER.with_label_values(&["set"]).inc(); + let timer = APPLY_HISTOGRAM.with_label_values(&["set"]).start_timer(); + + let id_field = index.schema().get_field("_id").unwrap(); + + let doc = index.schema().parse_document(doc.as_str()).unwrap(); + let id = &doc.get_first(id_field).unwrap().text().unwrap().to_string(); + + let opstamp = index_writer + .lock() + .unwrap() + .delete_term(Term::from_field_text(id_field, id)); + info!("delete document: id={:?}, opstamp={}", id, opstamp); + + let opstamp = index_writer.lock().unwrap().add_document(doc); + info!("add document: id={:?}, opstamp={}", id, opstamp); + + timer.observe_duration(); + } + Op::Delete { id } => { + APPLY_COUNTER.with_label_values(&["delete"]).inc(); + let timer = APPLY_HISTOGRAM.with_label_values(&["delete"]).start_timer(); + + let id_field = index.schema().get_field("_id").unwrap(); + + let opstamp = index_writer + .lock() + .unwrap() + .delete_term(Term::from_field_text(id_field, id.as_str())); + info!("delete document: id={}, opstamp={}", id, opstamp); + + timer.observe_duration(); + } + Op::BulkSet { docs } => { + APPLY_COUNTER.with_label_values(&["bulk_set"]).inc(); + let timer = APPLY_HISTOGRAM + .with_label_values(&["bulk_set"]) + .start_timer(); + + let id_field = index.schema().get_field("_id").unwrap(); + + let mut cnt = 0; + + let mut reader = BufReader::new(StringReader::new(docs.as_str())); + let mut line = String::new(); + while reader.read_line(&mut line).unwrap() > 0 { + let doc = index.schema().parse_document(&line).unwrap(); + let id = &doc.get_first(id_field).unwrap().text().unwrap().to_string(); + + let opstamp = index_writer + .lock() + .unwrap() + .delete_term(Term::from_field_text(id_field, id)); + info!("delete document: id={:?}, opstamp={}", id, opstamp); + + let opstamp = index_writer.lock().unwrap().add_document(doc); + info!("add document: id={:?}, opstamp={}", id, opstamp); + + line.clear(); + + cnt += 1; + } + + info!("set {} documents", cnt); + + timer.observe_duration(); + } + Op::BulkDelete { docs } => { + APPLY_COUNTER.with_label_values(&["bulk_delete"]).inc(); + let timer = APPLY_HISTOGRAM + .with_label_values(&["bulk_delete"]) + .start_timer(); + + let id_field = index.schema().get_field("_id").unwrap(); + + let mut cnt = 0; + + let mut reader = BufReader::new(StringReader::new(docs.as_str())); + let mut line = String::new(); + while reader.read_line(&mut line).unwrap() > 0 { + let doc = index.schema().parse_document(&line).unwrap(); + let id = &doc.get_first(id_field).unwrap().text().unwrap().to_string(); + + let opstamp = index_writer + .lock() + .unwrap() + .delete_term(Term::from_field_text(id_field, id)); + info!("delete document: id={:?}, opstamp={}", id, opstamp); + + line.clear(); + + cnt += 1; + } + + info!("delete {} documents", cnt); + + timer.observe_duration(); + } + Op::Commit {} => { + APPLY_COUNTER.with_label_values(&["commit"]).inc(); + let timer = APPLY_HISTOGRAM.with_label_values(&["commit"]).start_timer(); + + match index_writer.lock().unwrap().commit() { + Ok(opstamp) => { + info!("commit index: opstamp={}", opstamp); + } + Err(e) => { + error!("failed to commit index: {}", e); + } + }; + + timer.observe_duration(); + } + Op::Rollback {} => { + APPLY_COUNTER.with_label_values(&["rollback"]).inc(); + let timer = APPLY_HISTOGRAM + .with_label_values(&["rollback"]) + .start_timer(); + + match index_writer.lock().unwrap().rollback() { + Ok(opstamp) => { + info!("rollback index: opstamp={}", opstamp); + } + Err(e) => { + error!("failed to rollback index: {}", e); + } + }; + + timer.observe_duration(); + } + Op::Merge {} => { + APPLY_COUNTER.with_label_values(&["rollback"]).inc(); + let timer = APPLY_HISTOGRAM + .with_label_values(&["rollback"]) + .start_timer(); + + let segments = index.searchable_segment_ids().unwrap(); + if segments.len() <= 0 { + info!("there are no segment files that can be merged"); + return; + } + + let merge_future = index_writer.lock().unwrap().merge(&segments); + match block_on(merge_future) { + Ok(segment_meta) => { + info!("merge index: segments={:?}", segment_meta); + } + Err(e) => { + error!("failed to merge index: {:?}", e); + } + }; + + timer.observe_duration(); + } + Op::Schema {} => { + // noop + } + Op::Metrics {} => { + // noop + } + Op::Status {} => { + // noop + } + } + } +} diff --git a/bayard-server/src/lib.rs b/bayard-server/src/lib.rs new file mode 100644 index 0000000..1922d3f --- /dev/null +++ b/bayard-server/src/lib.rs @@ -0,0 +1,8 @@ +#[macro_use] +extern crate lazy_static; +#[macro_use] +extern crate prometheus; + +pub mod index; +pub mod raft; +pub mod tokenizer; diff --git a/bayard-server/src/raft/config.rs b/bayard-server/src/raft/config.rs new file mode 100644 index 0000000..ddca334 --- /dev/null +++ b/bayard-server/src/raft/config.rs @@ -0,0 +1,311 @@ +use std::collections::HashMap; +use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender}; +use std::sync::Arc; +use std::thread; +use std::time::{Duration, Instant}; + +use bincode::{deserialize, serialize}; +use grpcio::{ChannelBuilder, EnvBuilder}; +use log::*; +use protobuf::Message as PMessage; +use raft::prelude::*; +use raft::storage::MemStorage; +use serde::{Deserialize, Serialize}; + +use bayard_proto::proto::raftpb::AddressState; +use bayard_proto::proto::raftpb_grpc::RaftServiceClient; + +use crate::index::server::Op; + +type ProposeCallback = Box) + Send>; // return -1 if is leader, else return leader's id, vec is serialized address map + +pub enum Msg { + Propose { + seq: u64, + op: Op, + cb: ProposeCallback, + }, + ConfigChange { + seq: u64, + change: ConfChange, + cb: ProposeCallback, + }, + Address(AddressState), + // Here we don't use Raft Message, so use dead_code to + // avoid the compiler warning. + #[allow(dead_code)] + Raft(Message), +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct NodeAddress { + pub index_address: String, + pub raft_address: String, +} + +pub fn init_and_run( + storage: MemStorage, + receiver: Receiver, + apply_sender: Sender, + id: u64, + node_address: NodeAddress, + addresses: HashMap, +) { + // create communication clients + let mut peers = vec![]; + let mut addresses = addresses; // id:address + let mut rpc_clients = HashMap::new(); + for (id, address) in &addresses { + peers.push(id.clone()); + insert_client(id.clone(), address.raft_address.as_str(), &mut rpc_clients); + } + if peers.is_empty() { + addresses.insert(id, node_address); + peers.push(id); + } + debug!("{:?}", peers); + + // Create the configuration for the Raft node. + let cfg = Config { + // The unique ID for the Raft node. + id, + // The Raft node list. + // Mostly, the peers need to be saved in the storage + peers, + // Election tick is for how long the follower may campaign again after + // it doesn't receive any message from the leader. + election_tick: 10, + // Heartbeat tick is for how long the leader needs to send + // a heartbeat to keep alive. + heartbeat_tick: 3, + // The max size limits the max size of each appended message. Mostly, 1 MB is enough. + max_size_per_msg: 1024 * 1024 * 1024, + // Max inflight msgs that the leader sends messages to follower without + // receiving ACKs. + max_inflight_msgs: 256, + // The Raft applied index. + // You need to save your applied index when you apply the committed Raft logs. + applied: 0, + // Just for log + tag: format!("[{}]", 1), + ..Default::default() + }; + + // Create the Raft node. + let mut r = RawNode::new(&cfg, storage, vec![]).unwrap(); + + // Loop forever to drive the Raft. + let mut t = Instant::now(); + let mut timeout = Duration::from_millis(100); + + // Use a HashMap to hold the `propose` callbacks. + let mut cbs = HashMap::new(); + + loop { + match receiver.recv_timeout(timeout) { + Ok(Msg::Propose { seq, op, cb }) => { + debug!("propose message"); + let leader_id = r.raft.leader_id; + if leader_id != r.raft.id { + debug!("leader is {}, i'm {}", leader_id, r.raft.id); + cb(leader_id as i32, serialize(&addresses).unwrap()); + continue; + } + let se_op = serialize(&op).unwrap(); + cbs.insert(seq, cb); + r.propose(serialize(&seq).unwrap(), se_op).unwrap(); + } + Ok(Msg::ConfigChange { seq, change, cb }) => { + debug!("config change message"); + let leader_id = r.raft.leader_id; + if r.raft.leader_id != r.raft.id { + cb(leader_id as i32, serialize(&addresses).unwrap()); + continue; + } else { + cbs.insert(seq, cb); + r.propose_conf_change(serialize(&seq).unwrap(), change) + .unwrap(); + } + } + Ok(Msg::Raft(m)) => { + debug!("raft message: to={} from={}", r.raft.id, m.from); + if let Ok(_a) = r.step(m) {}; + } + Ok(Msg::Address(address_state)) => { + debug!("address message"); + let new_addresses: HashMap = + deserialize(address_state.get_address_map()).unwrap(); + for (id, address) in &new_addresses { + let insert = match addresses.get(id) { + Some(a) => { + if a.raft_address == address.raft_address { + false + } else { + true + } + } + None => true, + }; + if insert { + insert_client(id.clone(), address.raft_address.as_str(), &mut rpc_clients); + } + } + addresses = new_addresses; + } + Err(RecvTimeoutError::Timeout) => (), + Err(RecvTimeoutError::Disconnected) => return (), + } + + let d = t.elapsed(); + if d >= timeout { + t = Instant::now(); + timeout = Duration::from_millis(100); + // We drive Raft every 100ms. + r.tick(); + } else { + timeout -= d; + } + + on_ready( + &mut r, + &mut cbs, + &mut addresses, + &mut rpc_clients, + apply_sender.clone(), + ); + } +} + +fn on_ready( + r: &mut RawNode, + cbs: &mut HashMap, + addresses: &mut HashMap, + clients: &mut HashMap>, + apply_sender: Sender, +) { + if !r.has_ready() { + return; + } + + // The Raft is ready, we can do something now. + let mut ready = r.ready(); + + let is_leader = r.raft.leader_id == r.raft.id; + if is_leader { + // If the peer is leader, the leader can send messages to other followers ASAP. + let msgs = ready.messages.drain(..); + for msg in msgs { + let client = match clients.get(&msg.get_to()) { + Some(c) => c.clone(), + None => { + continue; + } + }; + let mut address_state = AddressState::new(); + address_state.set_address_map(serialize(&addresses).unwrap()); + thread::spawn(move || { + let msg = msg; + let address_state = address_state; + if let Ok(_) = client.send_msg(&msg) {}; + if let Ok(_) = client.send_address(&address_state) {}; + }); + } + } + + if !raft::is_empty_snap(&ready.snapshot) { + // This is a snapshot, we need to apply the snapshot at first. + r.mut_store() + .wl() + .apply_snapshot(ready.snapshot.clone()) + .unwrap(); + } + + if !ready.entries.is_empty() { + // Append entries to the Raft log + r.mut_store().wl().append(&ready.entries).unwrap(); + } + + if let Some(ref hs) = ready.hs { + // Raft HardState changed, and we need to persist it. + r.mut_store().wl().set_hardstate(hs.clone()); + } + + if !is_leader { + // If not leader, the follower needs to reply the messages to + // the leader after appending Raft entries. + let msgs = ready.messages.drain(..); + for msg in msgs { + // Send messages to other peers. + let client = match clients.get(&msg.get_to()) { + Some(c) => c.clone(), + None => { + continue; + } + }; + thread::spawn(move || { + let msg = msg; + if let Ok(_) = client.send_msg(&msg) {}; + }); + } + } + + if let Some(committed_entries) = ready.committed_entries.take() { + let mut _last_apply_index = 0; + for entry in committed_entries { + // Mostly, you need to save the last apply index to resume applying + // after restart. Here we just ignore this because we use a Memory storage. + _last_apply_index = entry.get_index(); + + if entry.get_data().is_empty() { + debug!("empty entry"); + // Emtpy entry, when the peer becomes Leader it will send an empty entry. + continue; + } + + if entry.get_entry_type() == EntryType::EntryNormal { + let op: Op = deserialize(entry.get_data()).unwrap(); + let seq: u64 = deserialize(entry.get_context()).unwrap(); + match apply_sender.send(op) { + _ => {} + } + if let Some(cb) = cbs.remove(&seq) { + cb(-1, serialize(addresses).unwrap()); + } + } + + // handle EntryConfChange + if entry.get_entry_type() == EntryType::EntryConfChange { + let mut change = ConfChange::new(); + change.merge_from_bytes(entry.get_data()).unwrap(); + let seq: u64 = deserialize(entry.get_context()).unwrap(); + let id = change.get_node_id(); + + let change_type = change.get_change_type(); + if change_type == ConfChangeType::AddNode { + let address: NodeAddress = deserialize(change.get_context()).unwrap(); + insert_client(id, address.raft_address.as_str(), clients); + addresses.insert(id, address); + } else if change_type == ConfChangeType::RemoveNode { + if let Some(_client) = clients.remove(&id) { + addresses.remove(&id); + } + } + + r.apply_conf_change(&change); + if let Some(cb) = cbs.remove(&seq) { + cb(-1, serialize(addresses).unwrap()); + } + } + } + } + + // Advance the Raft + r.advance(ready); +} + +fn insert_client(id: u64, address: &str, clients: &mut HashMap>) { + let env = Arc::new(EnvBuilder::new().build()); + let ch = ChannelBuilder::new(env).connect(address); + let client = RaftServiceClient::new(ch); + clients.insert(id.clone(), Arc::new(client)); +} diff --git a/bayard-server/src/raft/mod.rs b/bayard-server/src/raft/mod.rs new file mode 100644 index 0000000..6d51ea9 --- /dev/null +++ b/bayard-server/src/raft/mod.rs @@ -0,0 +1,2 @@ +pub mod config; +pub mod server; diff --git a/bayard-server/src/raft/server.rs b/bayard-server/src/raft/server.rs new file mode 100644 index 0000000..34dd15a --- /dev/null +++ b/bayard-server/src/raft/server.rs @@ -0,0 +1,80 @@ +use std::sync::mpsc; +use std::sync::mpsc::Sender; +use std::time::Duration; + +use futures::Future; +use grpcio::{RpcContext, UnarySink}; +use log::*; +use raft::eraftpb::{ConfChange, Message, Snapshot}; + +use bayard_proto::proto::commonpb::State; +use bayard_proto::proto::raftpb::{AddressState, ChangeReply, Null}; +use bayard_proto::proto::raftpb_grpc::RaftService; + +use crate::raft::config; + +#[derive(Clone)] +pub struct RaftServer { + pub sender: Sender, + seq: u64, +} + +impl RaftServer { + pub fn new(sender: Sender) -> RaftServer { + RaftServer { sender, seq: 0 } + } +} + +impl RaftService for RaftServer { + fn snapshot(&mut self, _ctx: RpcContext, _req: Snapshot, _sink: UnarySink) { + // TODO: + } + + fn change_config(&mut self, ctx: RpcContext, req: ConfChange, sink: UnarySink) { + debug!("change config"); + let (s1, r1) = mpsc::channel(); + let sender = self.sender.clone(); + let seq = self.seq; + self.seq += 1; + sender + .send(config::Msg::ConfigChange { + seq, + change: req, + cb: Box::new(move |leader_id: i32, addresses: Vec| { + let mut reply = ChangeReply::new(); + if leader_id >= 0 { + reply.set_state(State::WRONG_LEADER); + reply.set_leader_id(leader_id as u64); + } else { + reply.set_state(State::OK); + } + reply.set_address_map(addresses); + s1.send(reply).expect("cb channel closed"); + }), + }) + .unwrap(); + let reply = match r1.recv_timeout(Duration::from_secs(2)) { + Ok(r) => r, + Err(e) => { + warn!("timeout: {:?}", e); + let mut r = ChangeReply::new(); + r.set_state(State::IO_ERROR); + r + } + }; + let f = sink + .success(reply.clone()) + .map_err(move |err| error!("Failed to reply put: {:?}", err)); + ctx.spawn(f); + } + + fn send_msg(&mut self, _ctx: RpcContext, req: Message, _sink: ::grpcio::UnarySink) { + let sender = self.sender.clone(); + sender.send(config::Msg::Raft(req)).unwrap(); + } + + fn send_address(&mut self, _ctx: RpcContext, req: AddressState, _sink: UnarySink) { + let sender = self.sender.clone(); + sender.send(config::Msg::Address(req)).unwrap(); + } +} diff --git a/src/tokenizer/alpha_num_only_filter_factory.rs b/bayard-server/src/tokenizer/alpha_num_only_filter_factory.rs similarity index 100% rename from src/tokenizer/alpha_num_only_filter_factory.rs rename to bayard-server/src/tokenizer/alpha_num_only_filter_factory.rs diff --git a/src/tokenizer/ascii_folding_filter_factory.rs b/bayard-server/src/tokenizer/ascii_folding_filter_factory.rs similarity index 100% rename from src/tokenizer/ascii_folding_filter_factory.rs rename to bayard-server/src/tokenizer/ascii_folding_filter_factory.rs diff --git a/src/tokenizer/cang_jie_tokenizer_factory.rs b/bayard-server/src/tokenizer/cang_jie_tokenizer_factory.rs similarity index 100% rename from src/tokenizer/cang_jie_tokenizer_factory.rs rename to bayard-server/src/tokenizer/cang_jie_tokenizer_factory.rs diff --git a/src/tokenizer/facet_tokenizer_factory.rs b/bayard-server/src/tokenizer/facet_tokenizer_factory.rs similarity index 100% rename from src/tokenizer/facet_tokenizer_factory.rs rename to bayard-server/src/tokenizer/facet_tokenizer_factory.rs diff --git a/src/tokenizer/lindera_tokenizer_factory.rs b/bayard-server/src/tokenizer/lindera_tokenizer_factory.rs similarity index 100% rename from src/tokenizer/lindera_tokenizer_factory.rs rename to bayard-server/src/tokenizer/lindera_tokenizer_factory.rs diff --git a/src/tokenizer/lower_case_filter_factory.rs b/bayard-server/src/tokenizer/lower_case_filter_factory.rs similarity index 100% rename from src/tokenizer/lower_case_filter_factory.rs rename to bayard-server/src/tokenizer/lower_case_filter_factory.rs diff --git a/src/tokenizer.rs b/bayard-server/src/tokenizer/mod.rs similarity index 100% rename from src/tokenizer.rs rename to bayard-server/src/tokenizer/mod.rs diff --git a/src/tokenizer/ngram_tokenizer_factory.rs b/bayard-server/src/tokenizer/ngram_tokenizer_factory.rs similarity index 100% rename from src/tokenizer/ngram_tokenizer_factory.rs rename to bayard-server/src/tokenizer/ngram_tokenizer_factory.rs diff --git a/src/tokenizer/raw_tokenizer_factory.rs b/bayard-server/src/tokenizer/raw_tokenizer_factory.rs similarity index 100% rename from src/tokenizer/raw_tokenizer_factory.rs rename to bayard-server/src/tokenizer/raw_tokenizer_factory.rs diff --git a/src/tokenizer/remove_long_filter_factory.rs b/bayard-server/src/tokenizer/remove_long_filter_factory.rs similarity index 100% rename from src/tokenizer/remove_long_filter_factory.rs rename to bayard-server/src/tokenizer/remove_long_filter_factory.rs diff --git a/src/tokenizer/simple_tokenizer_factory.rs b/bayard-server/src/tokenizer/simple_tokenizer_factory.rs similarity index 100% rename from src/tokenizer/simple_tokenizer_factory.rs rename to bayard-server/src/tokenizer/simple_tokenizer_factory.rs diff --git a/src/tokenizer/stemming_filter_factory.rs b/bayard-server/src/tokenizer/stemming_filter_factory.rs similarity index 100% rename from src/tokenizer/stemming_filter_factory.rs rename to bayard-server/src/tokenizer/stemming_filter_factory.rs diff --git a/src/tokenizer/stop_word_filter_factory.rs b/bayard-server/src/tokenizer/stop_word_filter_factory.rs similarity index 100% rename from src/tokenizer/stop_word_filter_factory.rs rename to bayard-server/src/tokenizer/stop_word_filter_factory.rs diff --git a/src/tokenizer/tokenizer_initializer.rs b/bayard-server/src/tokenizer/tokenizer_initializer.rs similarity index 100% rename from src/tokenizer/tokenizer_initializer.rs rename to bayard-server/src/tokenizer/tokenizer_initializer.rs diff --git a/bayard/Cargo.toml b/bayard/Cargo.toml new file mode 100644 index 0000000..10e1211 --- /dev/null +++ b/bayard/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "bayard" +version = "0.7.0" +authors = ["Minoru Osuka "] +edition = "2018" +description = "Bayard is a distributed search server." +documentation = "https://bayard-search.github.io/bayard/" +homepage = "https://github.com/bayard-search" +repository = "https://github.com/bayard-search/bayard/bayard" +readme = "README.md" +keywords = ["full-text", "search", "index", "server"] +categories = ["database"] +license = "MIT" + +[[bin]] +name = "bayard" +path = "src/main.rs" + +[dependencies] +clap = "2.33.0" +crossbeam-channel = "0.4.0" +ctrlc = { version = "3.1.3", features = ["termination"] } +env_logger = "0.7.1" +futures = "0.1.29" +grpcio = { version = "0.4.7", features = [ "secure" ] } +log = "0.4.8" +num_cpus = "1.11.1" +raft = "0.4.3" +serde_json = "1.0.44" + +bayard-client = { version = "0.7.0", path = "../bayard-client" } +bayard-proto = "0.7.0" +bayard-server = { version = "0.7.0", path = "../bayard-server" } diff --git a/bayard/README.md b/bayard/README.md new file mode 100644 index 0000000..0d1ec26 --- /dev/null +++ b/bayard/README.md @@ -0,0 +1,29 @@ +# Bayard + +[![Join the chat at https://gitter.im/bayard-search/bayard](https://badges.gitter.im/bayard-search/bayard.svg)](https://gitter.im/bayard-search/bayard?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +Bayard is a full-text search and indexing server written in [Rust](https://www.rust-lang.org/) built on top of [Tantivy](https://github.com/tantivy-search/tantivy) that implements [Raft Consensus Algorithm](https://raft.github.io/) and [gRPC](https://grpc.io/). +Achieves consensus across all the nodes, ensures every change made to the system is made to a quorum of nodes. +Bayard makes easy for programmers to develop search applications with advanced features and high availability. + + +## Features + +- Full-text search/indexing +- Index replication +- Bringing up a cluster +- Command line interface is available + + +## Source code repository + +- [https://github.com/bayard-search/bayard](https://github.com/bayard-search/bayard) + +## Docker container repository + +- [https://hub.docker.com/r/bayardsearch/bayard](https://hub.docker.com/r/bayardsearch/bayard) + +## Documents + +- [https://bayard-search.github.io/bayard/](https://bayard-search.github.io/bayard/) diff --git a/bayard/src/cli/bulk_delete.rs b/bayard/src/cli/bulk_delete.rs new file mode 100644 index 0000000..a28e2c9 --- /dev/null +++ b/bayard/src/cli/bulk_delete.rs @@ -0,0 +1,16 @@ +use clap::ArgMatches; + +use bayard_client::index::client::IndexClient; + +use crate::log::set_logger; + +pub fn run_bulk_delete_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_logger(); + + let server = matches.value_of("SERVER").unwrap(); + let docs = matches.value_of("DOCS").unwrap(); + + let mut index_client = IndexClient::new(server); + + index_client.bulk_delete(docs.to_string()) +} diff --git a/bayard/src/cli/bulk_set.rs b/bayard/src/cli/bulk_set.rs new file mode 100644 index 0000000..83a98cb --- /dev/null +++ b/bayard/src/cli/bulk_set.rs @@ -0,0 +1,16 @@ +use clap::ArgMatches; + +use bayard_client::index::client::IndexClient; + +use crate::log::set_logger; + +pub fn run_bulk_set_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_logger(); + + let server = matches.value_of("SERVER").unwrap(); + let docs = matches.value_of("DOCS").unwrap(); + + let mut index_client = IndexClient::new(server); + + index_client.bulk_set(docs.to_string()) +} diff --git a/bayard/src/cli/commit.rs b/bayard/src/cli/commit.rs new file mode 100644 index 0000000..ce76c57 --- /dev/null +++ b/bayard/src/cli/commit.rs @@ -0,0 +1,15 @@ +use clap::ArgMatches; + +use bayard_client::index::client::IndexClient; + +use crate::log::set_logger; + +pub fn run_commit_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_logger(); + + let server = matches.value_of("SERVER").unwrap(); + + let mut index_client = IndexClient::new(server); + + index_client.commit() +} diff --git a/bayard/src/cli/delete.rs b/bayard/src/cli/delete.rs new file mode 100644 index 0000000..2d12c24 --- /dev/null +++ b/bayard/src/cli/delete.rs @@ -0,0 +1,16 @@ +use clap::ArgMatches; + +use bayard_client::index::client::IndexClient; + +use crate::log::set_logger; + +pub fn run_delete_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_logger(); + + let server = matches.value_of("SERVER").unwrap(); + let id = matches.value_of("ID").unwrap(); + + let mut index_client = IndexClient::new(server); + + index_client.delete(id.to_string()) +} diff --git a/bayard/src/cli/get.rs b/bayard/src/cli/get.rs new file mode 100644 index 0000000..721329c --- /dev/null +++ b/bayard/src/cli/get.rs @@ -0,0 +1,25 @@ +use clap::ArgMatches; + +use bayard_client::index::client::IndexClient; + +use crate::log::set_logger; + +pub fn run_get_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_logger(); + + let server = matches.value_of("SERVER").unwrap(); + let id = matches.value_of("ID").unwrap(); + + let mut index_client = IndexClient::new(server); + + match index_client.get(id.to_string()) { + Ok(v) => { + println!("{}", v); + Ok(()) + } + Err(e) => { + println!("{}", e); + Err(e) + } + } +} diff --git a/bayard/src/cli/leave.rs b/bayard/src/cli/leave.rs new file mode 100644 index 0000000..b50140c --- /dev/null +++ b/bayard/src/cli/leave.rs @@ -0,0 +1,25 @@ +use clap::ArgMatches; + +use bayard_client::raft::client::RaftClient; + +use crate::log::set_logger; + +pub fn run_leave_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_logger(); + + let server = matches.value_of("SERVER").unwrap(); + let id = matches.value_of("ID").unwrap().parse::().unwrap(); + + let mut raft_client = RaftClient::new(server); + + match raft_client.leave(id) { + Ok(v) => { + println!("{}", serde_json::to_string(&v).unwrap()); + Ok(()) + } + Err(e) => { + println!("{}", e); + Err(e) + } + } +} diff --git a/bayard/src/cli/merge.rs b/bayard/src/cli/merge.rs new file mode 100644 index 0000000..0a6556d --- /dev/null +++ b/bayard/src/cli/merge.rs @@ -0,0 +1,15 @@ +use clap::ArgMatches; + +use bayard_client::index::client::IndexClient; + +use crate::log::set_logger; + +pub fn run_merge_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_logger(); + + let server = matches.value_of("SERVER").unwrap(); + + let mut index_client = IndexClient::new(server); + + index_client.merge() +} diff --git a/bayard/src/cli/metrics.rs b/bayard/src/cli/metrics.rs new file mode 100644 index 0000000..2e32ccc --- /dev/null +++ b/bayard/src/cli/metrics.rs @@ -0,0 +1,24 @@ +use clap::ArgMatches; + +use bayard_client::index::client::IndexClient; + +use crate::log::set_logger; + +pub fn run_metrics_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_logger(); + + let server = matches.value_of("SERVER").unwrap(); + + let mut index_client = IndexClient::new(server); + + match index_client.metrics() { + Ok(v) => { + println!("{}", v); + Ok(()) + } + Err(e) => { + println!("{}", e); + Err(e) + } + } +} diff --git a/src/cmd.rs b/bayard/src/cli/mod.rs similarity index 59% rename from src/cmd.rs rename to bayard/src/cli/mod.rs index ee94d13..9c14610 100644 --- a/src/cmd.rs +++ b/bayard/src/cli/mod.rs @@ -1,15 +1,15 @@ +pub mod bulk_delete; +pub mod bulk_set; pub mod commit; pub mod delete; -pub mod gateway; pub mod get; pub mod leave; pub mod merge; pub mod metrics; -pub mod peers; -pub mod probe; -pub mod put; pub mod rollback; -pub mod schedule; +pub mod root; pub mod schema; pub mod search; -pub mod serve; +pub mod set; +pub mod start; +pub mod status; diff --git a/bayard/src/cli/rollback.rs b/bayard/src/cli/rollback.rs new file mode 100644 index 0000000..44aeb66 --- /dev/null +++ b/bayard/src/cli/rollback.rs @@ -0,0 +1,15 @@ +use clap::ArgMatches; + +use bayard_client::index::client::IndexClient; + +use crate::log::set_logger; + +pub fn run_rollback_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_logger(); + + let server = matches.value_of("SERVER").unwrap(); + + let mut index_client = IndexClient::new(server); + + index_client.rollback() +} diff --git a/bayard/src/cli/root.rs b/bayard/src/cli/root.rs new file mode 100644 index 0000000..4cbcb72 --- /dev/null +++ b/bayard/src/cli/root.rs @@ -0,0 +1,515 @@ +use clap::{App, AppSettings, Arg, SubCommand}; +use num_cpus; + +use crate::cli::bulk_delete::run_bulk_delete_cli; +use crate::cli::bulk_set::run_bulk_set_cli; +use crate::cli::commit::run_commit_cli; +use crate::cli::delete::run_delete_cli; +use crate::cli::get::run_get_cli; +use crate::cli::leave::run_leave_cli; +use crate::cli::merge::run_merge_cli; +use crate::cli::metrics::run_metrics_cli; +use crate::cli::rollback::run_rollback_cli; +use crate::cli::schema::run_schema_cli; +use crate::cli::search::run_search_cli; +use crate::cli::set::run_set_cli; +use crate::cli::start::run_start_cli; +use crate::cli::status::run_status_cli; + +pub fn run_root_cli() -> Result<(), std::io::Error> { + let cpus = num_cpus::get().to_owned(); + let threads; + if cpus > 1 { + threads = format!("{}", cpus - 1); + } else { + threads = format!("{}", cpus); + } + + let app = App::new(crate_name!()) + .setting(AppSettings::DeriveDisplayOrder) + .setting(AppSettings::SubcommandRequiredElseHelp) + .version(crate_version!()) + .author(crate_authors!()) + .about("Manage Bayard index server") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .subcommand( + SubCommand::with_name("start") + .name("start") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Start index server") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("ID") + .help("Node ID.") + .value_name("ID") + .takes_value(true), + ) + .arg( + Arg::with_name("HOST") + .help("Node address.") + .short("H") + .long("host") + .value_name("HOST") + .default_value("0.0.0.0") + .takes_value(true), + ) + .arg( + Arg::with_name("RAFT_PORT") + .help("Raft service port number.") + .short("r") + .long("raft-port") + .value_name("RAFT_PORT") + .default_value("7000") + .takes_value(true), + ) + .arg( + Arg::with_name("INDEX_PORT") + .help("Index service port number") + .short("i") + .long("index-port") + .value_name("INDEX_PORT") + .default_value("5000") + .takes_value(true), + ) + .arg( + Arg::with_name("PEER_RAFT_ADDRESS") + .help("Raft address of a peer node running in an existing cluster.") + .short("p") + .long("peer-raft-address") + .value_name("IP:PORT") + .takes_value(true), + ) + .arg( + Arg::with_name("DATA_DIRECTORY") + .help("Data directory. Stores index, snapshots, and raft logs.") + .short("d") + .long("data-directory") + .value_name("DATA_DIRECTORY") + .default_value("./data") + .takes_value(true), + ) + .arg( + Arg::with_name("SCHEMA_FILE") + .help("Schema file. Must specify An existing file name.") + .short("s") + .long("schema-file") + .value_name("SCHEMA_FILE") + .default_value("./etc/schema.json") + .takes_value(true), + ) + .arg( + Arg::with_name("TOKENIZER_FILE") + .help("Tokenizer file. Must specify An existing file name.") + .short("T") + .long("tokenizer-file") + .value_name("TOKENIZER_FILE") + .default_value("./etc/tokenizer.json") + .takes_value(true), + ) + .arg( + Arg::with_name("INDEXER_THREADS") + .help("Number of indexer threads.") + .short("t") + .long("indexer-threads") + .value_name("INDEXER_THREADS") + .default_value(&threads) + .takes_value(true), + ) + .arg( + Arg::with_name("INDEXER_MEMORY_SIZE") + .help("Total memory size (in bytes) used by the indexer.") + .short("m") + .long("indexer-memory-size") + .value_name("INDEXER_MEMORY_SIZE") + .default_value("1000000000") + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("leave") + .name("leave") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Delete node from the cluster") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("SERVER") + .help("Raft service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:7000") + .takes_value(true), + ) + .arg( + Arg::with_name("ID") + .help("Node ID to be removed from the cluster.") + .value_name("ID") + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("get") + .name("get") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Get document from index server") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ) + .arg( + Arg::with_name("ID") + .help("A unique ID that identifies the document in the index server.") + .value_name("ID") + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("set") + .name("set") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Set document to index server") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ) + .arg( + Arg::with_name("ID") + .help("A unique ID that identifies the document in the index server.") + .value_name("ID") + .takes_value(true), + ) + .arg( + Arg::with_name("FIELDS") + .help("Fields of document to be indexed.") + .value_name("FIELDS") + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("delete") + .name("delete") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Delete document from index server") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ) + .arg( + Arg::with_name("ID") + .help("A unique ID that identifies the document in the index server.") + .value_name("ID") + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("bulk-set") + .name("bulk-set") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Set documents to index server in bulk") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ) + .arg( + Arg::with_name("DOCS") + .help("Document containing the unique ID to be indexed.") + .value_name("DOCS") + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("bulk-delete") + .name("bulk-delete") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Delete documents from index server in bulk") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ) + .arg( + Arg::with_name("DOCS") + .help("Document containing the unique ID to be indexed.") + .value_name("DOCS") + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("commit") + .name("commit") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Commit index") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("rollback") + .name("rollback") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Rollback index") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("merge") + .name("merge") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Merge index") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("schema") + .name("schema") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Shows index schema that applied") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("search") + .name("search") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Search documents from index server") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ) + .arg( + Arg::with_name("FROM") + .help("Start position of fetching results.") + .short("f") + .long("from") + .value_name("FROM") + .default_value("0") + .takes_value(true), + ) + .arg( + Arg::with_name("LIMIT") + .help("Limitation of amount that document to be returned.") + .short("l") + .long("limit") + .value_name("LIMIT") + .default_value("10") + .takes_value(true), + ) + .arg( + Arg::with_name("EXCLUDE_COUNT") + .help("A flag for exclude hit count in the search result.") + .short("c") + .long("exclude-count"), + ) + .arg( + Arg::with_name("EXCLUDE_DOCS") + .help("A flag for exclude hit documents in the search result.") + .short("d") + .long("exclude-docs"), + ) + .arg( + Arg::with_name("FACET_FIELD") + .help("Hierarchical facet field name.") + .short("F") + .long("facet-field") + .value_name("FACET_FIELD") + .default_value("") + .takes_value(true), + ) + .arg( + Arg::with_name("FACET_PREFIX") + .help("Hierarchical facet field value prefix.") + .short("V") + .long("facet-prefix") + .value_name("FACET_PREFIX") + .multiple(true) + .number_of_values(1) + .takes_value(true), + ) + .arg( + Arg::with_name("QUERY") + .help("Query string for search the index.") + .value_name("QUERY") + .required(true) + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("status") + .name("status") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Shows system status") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("metrics") + .name("metrics") + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Shows system metrics") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ), + ) + .get_matches(); + + let (subcommand, some_options) = app.subcommand(); + let options = some_options.unwrap(); + let run_cli = match subcommand { + "start" => run_start_cli, + "leave" => run_leave_cli, + "search" => run_search_cli, + "get" => run_get_cli, + "set" => run_set_cli, + "delete" => run_delete_cli, + "bulk-set" => run_bulk_set_cli, + "bulk-delete" => run_bulk_delete_cli, + "commit" => run_commit_cli, + "rollback" => run_rollback_cli, + "merge" => run_merge_cli, + "schema" => run_schema_cli, + "metrics" => run_metrics_cli, + "status" => run_status_cli, + _ => panic!("Subcommand {} is unknown", subcommand), + }; + + run_cli(options) +} diff --git a/bayard/src/cli/schema.rs b/bayard/src/cli/schema.rs new file mode 100644 index 0000000..2dbd6a0 --- /dev/null +++ b/bayard/src/cli/schema.rs @@ -0,0 +1,24 @@ +use clap::ArgMatches; + +use bayard_client::index::client::IndexClient; + +use crate::log::set_logger; + +pub fn run_schema_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_logger(); + + let server = matches.value_of("SERVER").unwrap(); + + let mut index_client = IndexClient::new(server); + + match index_client.schema() { + Ok(v) => { + println!("{}", v); + Ok(()) + } + Err(e) => { + println!("{}", e); + Err(e) + } + } +} diff --git a/src/cmd/search.rs b/bayard/src/cli/search.rs similarity index 50% rename from src/cmd/search.rs rename to bayard/src/cli/search.rs index 0378e32..c56707e 100644 --- a/src/cmd/search.rs +++ b/bayard/src/cli/search.rs @@ -1,16 +1,13 @@ -use bayard_client::client::client::{create_client, Clerk}; use clap::ArgMatches; -use crate::util::log::set_logger; +use bayard_client::index::client::IndexClient; -pub fn run_search_cli(matches: &ArgMatches) -> Result<(), String> { +use crate::log::set_logger; + +pub fn run_search_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { set_logger(); - let servers: Vec<_> = matches - .values_of("SERVERS") - .unwrap() - .map(|addr| create_client(addr)) - .collect(); + let server = matches.value_of("SERVER").unwrap(); let from = matches.value_of("FROM").unwrap().parse::().unwrap(); let limit = matches.value_of("LIMIT").unwrap().parse::().unwrap(); let exclude_count = matches.is_present("EXCLUDE_COUNT"); @@ -22,20 +19,11 @@ pub fn run_search_cli(matches: &ArgMatches) -> Result<(), String> { .map(|s| facet_prefixes.push(s.to_string())) .count(); } - // let mut facets: Vec = Vec::new(); - // if let Some(_facets) = matches.values_of("FACET") { - // _facets.map(|s| { - // facets.push(s.to_string()) - // } - // ).count(); - // } - let query = matches.value_of("QUERY").unwrap(); - let client_id = rand::random(); + let mut index_client = IndexClient::new(server); - let mut client = Clerk::new(&servers, client_id); - let value = client.search( + match index_client.search( query, from, limit, @@ -43,9 +31,14 @@ pub fn run_search_cli(matches: &ArgMatches) -> Result<(), String> { exclude_docs, facet_field, facet_prefixes, - ); - // let value = client.search(query, from, limit, exclude_count, exclude_docs, facets); - print!("{}", value); - - Ok(()) + ) { + Ok(v) => { + println!("{}", v); + Ok(()) + } + Err(e) => { + println!("{}", e); + Err(e) + } + } } diff --git a/bayard/src/cli/set.rs b/bayard/src/cli/set.rs new file mode 100644 index 0000000..c288742 --- /dev/null +++ b/bayard/src/cli/set.rs @@ -0,0 +1,22 @@ +use clap::ArgMatches; +use serde_json::Value; + +use bayard_client::index::client::IndexClient; + +use crate::log::set_logger; + +pub fn run_set_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_logger(); + + let server = matches.value_of("SERVER").unwrap(); + let id = matches.value_of("ID").unwrap(); + let fields = matches.value_of("FIELDS").unwrap(); + + let mut doc_json: Value = serde_json::from_str(fields).unwrap(); + doc_json["_id"] = Value::String(id.to_string()); + let doc = serde_json::to_string(&doc_json).unwrap(); + + let mut index_client = IndexClient::new(server); + + index_client.set(doc) +} diff --git a/bayard/src/cli/start.rs b/bayard/src/cli/start.rs new file mode 100644 index 0000000..8804f01 --- /dev/null +++ b/bayard/src/cli/start.rs @@ -0,0 +1,148 @@ +use std::collections::HashMap; +use std::path::Path; +use std::sync::Arc; + +use clap::ArgMatches; +use crossbeam_channel::select; +use futures::Future; +use grpcio::{Environment, ServerBuilder}; +use log::*; +use raft::storage::MemStorage; + +use bayard_client::raft::client::RaftClient; +use bayard_proto::proto::{indexpb_grpc, raftpb_grpc}; +use bayard_server::index::server::IndexServer; +use bayard_server::raft::config::NodeAddress; + +use crate::log::set_logger; +use crate::signal::sigterm_channel; +// use bayard_server::metrics::Metrics; + +pub fn run_start_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_logger(); + + let id = matches.value_of("ID").unwrap().parse::().unwrap(); + let host = matches.value_of("HOST").unwrap(); + let raft_port = matches + .value_of("RAFT_PORT") + .unwrap() + .parse::() + .unwrap(); + let index_port = matches + .value_of("INDEX_PORT") + .unwrap() + .parse::() + .unwrap(); + let mut peer_address = ""; + if let Some(_peer_address) = matches.value_of("PEER_RAFT_ADDRESS") { + peer_address = _peer_address; + } + let data_directory = matches.value_of("DATA_DIRECTORY").unwrap(); + let schema_file = matches.value_of("SCHEMA_FILE").unwrap(); + let tokenizer_file = matches.value_of("TOKENIZER_FILE").unwrap(); + let indexer_threads = matches + .value_of("INDEXER_THREADS") + .unwrap() + .parse::() + .unwrap(); + let indexer_memory_size = matches + .value_of("INDEXER_MEMORY_SIZE") + .unwrap() + .parse::() + .unwrap(); + + let raft_address = format!("{}:{}", host, raft_port); + let index_address = format!("{}:{}", host, index_port); + + let node_address = NodeAddress { + index_address, + raft_address, + }; + + let mut addresses = HashMap::new(); + + // change config + if peer_address != "" { + let mut client = RaftClient::new(peer_address); + match client.join(id, node_address.clone()) { + Ok(_addresses) => addresses = _addresses, + Err(e) => return Err(e), + }; + } + + let env_index = Arc::new(Environment::new(10)); + let env_raft = Arc::new(Environment::new(10)); + + let index_path = Path::new(data_directory) + .join("index") + .to_str() + .unwrap() + .to_string(); + let raft_storage = MemStorage::new(); + + // let metrics = Metrics::new(id); + + let (index, raft) = IndexServer::new( + index_path, + schema_file, + tokenizer_file, + indexer_threads, + indexer_memory_size, + raft_storage, + id, + node_address, + addresses, + // metrics, + ); + + let index_service = indexpb_grpc::create_index_service(index); + let raft_service = raftpb_grpc::create_raft_service(raft); + + let mut index_server = ServerBuilder::new(env_index) + .register_service(index_service) + .bind(host, index_port) + .build() + .unwrap(); + let mut raft_server = ServerBuilder::new(env_raft) + .register_service(raft_service) + .bind(host, raft_port) + .build() + .unwrap(); + + index_server.start(); + raft_server.start(); + + for &(ref h, p) in index_server.bind_addrs() { + info!("start index service on {}:{}", h, p); + } + + for &(ref h, p) in raft_server.bind_addrs() { + info!("start Raft service on {}:{}", h, p); + } + + // Wait for signals for termination (SIGINT, SIGTERM). + let sigterm_receiver = sigterm_channel().unwrap(); + loop { + select! { + recv(sigterm_receiver) -> _ => { + debug!("receive signal"); + break; + } + } + } + + match index_server.shutdown().wait() { + Ok(_) => { + info!("stop index service on {}:{}", host, index_port); + } + Err(e) => error!("{}", e), + } + match raft_server.shutdown().wait() { + Ok(_) => { + info!("stop Raft service on {}:{}", host, raft_port); + } + Err(e) => error!("{}", e), + } + + Ok(()) +} diff --git a/bayard/src/cli/status.rs b/bayard/src/cli/status.rs new file mode 100644 index 0000000..c744be9 --- /dev/null +++ b/bayard/src/cli/status.rs @@ -0,0 +1,24 @@ +use clap::ArgMatches; + +use bayard_client::index::client::IndexClient; + +use crate::log::set_logger; + +pub fn run_status_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { + set_logger(); + + let server = matches.value_of("SERVER").unwrap(); + + let mut index_client = IndexClient::new(server); + + match index_client.status() { + Ok(v) => { + println!("{}", v); + Ok(()) + } + Err(e) => { + println!("{}", e); + Err(e) + } + } +} diff --git a/bayard/src/lib.rs b/bayard/src/lib.rs new file mode 100644 index 0000000..0946d2f --- /dev/null +++ b/bayard/src/lib.rs @@ -0,0 +1,6 @@ +#[macro_use] +extern crate clap; + +pub mod cli; +pub mod log; +pub mod signal; diff --git a/src/util/log.rs b/bayard/src/log/mod.rs similarity index 100% rename from src/util/log.rs rename to bayard/src/log/mod.rs diff --git a/bayard/src/main.rs b/bayard/src/main.rs new file mode 100644 index 0000000..b877ef1 --- /dev/null +++ b/bayard/src/main.rs @@ -0,0 +1,5 @@ +use bayard::cli::root::run_root_cli; + +fn main() -> Result<(), std::io::Error> { + run_root_cli() +} diff --git a/src/util/signal.rs b/bayard/src/signal/mod.rs similarity index 100% rename from src/util/signal.rs rename to bayard/src/signal/mod.rs diff --git a/docs/accessing_over_http.html b/docs/accessing_over_http.html index 763f1d7..84337d2 100644 --- a/docs/accessing_over_http.html +++ b/docs/accessing_over_http.html @@ -81,7 +81,7 @@ @@ -160,7 +160,7 @@

Using Gateway

REST API

See following documents:

@@ -174,7 +174,7 @@

REST API

- @@ -192,7 +192,7 @@

REST API

- diff --git a/docs/building_bayard.html b/docs/building_bayard.html index 4443f6d..718473c 100644 --- a/docs/building_bayard.html +++ b/docs/building_bayard.html @@ -81,7 +81,7 @@ @@ -156,6 +156,14 @@

Requirementsmake >= 3.81
  • protoc >= 3.9.2
    • +

    Install protoc-gen-rust

    +
    $ cargo install protobuf-codegen
+
+
    +

    Install protoc-gen-grpc-web

    +
    $ curl -o /usr/local/bin/protoc-gen-grpc-web -L https://github.com/grpc/grpc-web/releases/download/1.0.7/protoc-gen-grpc-web-1.0.7-darwin-x86_64
+$ chmod +x /usr/local/bin/protoc-gen-grpc-web
+

    Build

    Build Bayard with the following command:

    $ make build
diff --git a/docs/cluster_mode.html b/docs/cluster_mode.html
index 4bdf796..97ecb3d 100644
--- a/docs/cluster_mode.html
+++ b/docs/cluster_mode.html
@@ -81,7 +81,7 @@
 
         
@@ -153,41 +153,70 @@ 
    Cluster modeStarting in cluster mode (3-node cluster)
    
 
    Running in standalone is not fault tolerant. If you need to improve fault tolerance, start servers in cluster mode.
 You can start servers in cluster mode with the following command:
    
-
    $ ./bin/bayard serve \
-    --id=1 \
-    --host=0.0.0.0 \
-    --port=5001 \
-    --data-directory=./data/1 \
-    --schema-file=./etc/schema.json
+
    $ ./bin/bayard start \
+               --host=0.0.0.0 \
+               --raft-port=7001 \
+               --index-port=5001 \
+               --data-directory=./data/node1 \
+               --schema-file=./etc/schema.json \
+               --tokenizer-file=./etc/tokenizer.json \
+               1
 
    
-
    $ ./bin/bayard serve \
-    --id=2 \
-    --host=0.0.0.0 \
-    --port=5002 \
-    --peers="1=0.0.0.0:5001" \
-    --data-directory=./data/2 \
-    --schema-file=./etc/schema.json
+
    $ ./bin/bayard start \
+               --host=0.0.0.0 \
+               --raft-port=7002 \
+               --index-port=5002 \
+               --peer-raft-address=0.0.0.0:7001 \
+               --data-directory=./data/node2 \
+               --schema-file=./etc/schema.json \
+               --tokenizer-file=./etc/tokenizer.json \
+               2
 
    
-
    $ ./bin/bayard serve \
-    --id=3 \
-    --host=0.0.0.0 \
-    --port=5003 \
-    --peers="1=0.0.0.0:5001,2=0.0.0.0:5002" \
-    --data-directory=./data/3 \
-    --schema-file=./etc/schema.json
+
    $ ./bin/bayard start \
+               --host=0.0.0.0 \
+               --raft-port=7003 \
+               --index-port=5003 \
+               --peer-raft-address=0.0.0.0:7001 \
+               --data-directory=./data/node3 \
+               --schema-file=./etc/schema.json \
+               --tokenizer-file=./etc/tokenizer.json \
+               3
 
    
 
    The above commands run servers on the same host, so each server must listen on a different port. This would not be necessary if each server runs on a different host.
 Recommend 3 or more odd number of servers in the cluster to avoid split-brain.
    
 When deploying to a single host, if that host goes down due to hardware failure, all of the servers in the cluster will be stopped, so recommend deploying to a different host.
    
 
    Cluster peers
    
 
    You can check the peers in the cluster with the following command:
    
-
    $ ./bin/bayard peers --server localhost:5001 | jq .
+
    $ ./bin/bayard status
+               --server=0.0.0.0:5001 | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    {
-  "1": "0.0.0.0:5001",
-  "2": "0.0.0.0:5002",
-  "3": "0.0.0.0:5003"
+  "leader": 1,
+  "nodes": [
+    {
+      "address": {
+        "index_address": "0.0.0.0:5001",
+        "raft_address": "0.0.0.0:7001"
+      },
+      "id": 1
+    },
+    {
+      "address": {
+        "index_address": "0.0.0.0:5002",
+        "raft_address": "0.0.0.0:7002"
+      },
+      "id": 2
+    },
+    {
+      "address": {
+        "index_address": "0.0.0.0:5003",
+        "raft_address": "0.0.0.0:7003"
+      },
+      "id": 3
+    }
+  ],
+  "status": "OK"
 }
 
    
 
    Remove a server from a cluster
    
@@ -195,8 +224,8 @@ 
    $ ./bin/bayard leave \
-    --servers=127.0.0.1:5001 \
-    --id=3
+               --servers=0.0.0.0:5001 \
+               3
 
    
 
                     
diff --git a/docs/configure_text_analyzers.html b/docs/configure_text_analyzers.html
index f422298..073f80e 100644
--- a/docs/configure_text_analyzers.html
+++ b/docs/configure_text_analyzers.html
@@ -81,7 +81,7 @@
 
         
diff --git a/docs/designing_schema.html b/docs/designing_schema.html
index 26e6d0d..9eae010 100644
--- a/docs/designing_schema.html
+++ b/docs/designing_schema.html
@@ -81,7 +81,7 @@
 
         
diff --git a/docs/getting_started.html b/docs/getting_started.html
index c84d977..4bfe3ad 100644
--- a/docs/getting_started.html
+++ b/docs/getting_started.html
@@ -81,7 +81,7 @@
 
         
@@ -151,20 +151,7 @@ 
    Bayard
    
                         
    Getting started
    
 
    Starting in standalone mode (Single node cluster)
    
 
    Running node in standalone mode is easy. You can start server with the following command:
    
-
    $ ./bin/bayard serve
-
    
-
    You'll see a startup message like following:
    
-
    [2019-11-27T00:30:45Z INFO bayard::server::server src/server/server.rs:119] listening on 0.0.0.0:5000
-[2019-11-27T00:30:45Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:723]  became follower at term 0
-[2019-11-27T00:30:45Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:295]  newRaft [peers: [1], term: 0, commit: 0, applied: 0, last_index: 0, last_term: 0]
-[2019-11-27T00:30:45Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:723]  became follower at term 1
-
-[...]
-
-[2019-11-27T00:30:48Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:1094]  is starting a new election at term 1
-[2019-11-27T00:30:48Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:743]  became candidate at term 2
-[2019-11-27T00:30:48Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:858]  received MsgRequestVoteResponse from 1 at term 2
-[2019-11-27T00:30:48Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:793]  became leader at term 2
+
    $ ./bin/bayard start 1
 
    
 
    Getting schema
    
 
    You can confirm current schema with the following command:
    
@@ -242,12 +229,12 @@ 
    Getting schema<
 
    
 
    Indexing document
    
 
    You can index document with the following command:
    
-
    $ ./bin/bayard put --id=1 --file=./examples/doc_1.json | jq .
-$ ./bin/bayard commit | jq .
+
    $ cat ./examples/doc_1.json | xargs -0 ./bin/bayard set 1
+$ ./bin/bayard commit
 
    
 
    Getting document
    
 
    You can get document with the following command:
    
-
    $ ./bin/bayard get --id=1 | jq .
+
    $ ./bin/bayard get 1 | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    {
@@ -277,8 +264,8 @@ 
    Getting doc
 
    
 
    Indexing documents in bulk
    
 
    You can index documents in bulk with the following command:
    
-
    $ ./bin/bayard put --bulk --file=./examples/bulk_put.jsonl | jq .
-$ ./bin/bayard commit | jq .
+
    $ cat ./examples/bulk_put.jsonl | xargs -0 ./bin/bayard bulk-set
+$ ./bin/bayard commit
 
    
 
    Searching documents
    
 
    You can search documents with the following command:
    
@@ -354,13 +341,13 @@ 
    Searc
 
    
 
    Deleting document
    
 
    You can delete document with the following command:
    
-
    $ ./bin/bayard delete --id=1 | jq .
-$ ./bin/bayard commit | jq .
+
    $ ./bin/bayard delete 1
+$ ./bin/bayard commit
 
    
 
    Deleting documents in bulk
    
 
    You can delete documents in bulk with the following command:
    
-
    $ ./bin/bayard delete --bulk --file=./examples/bulk_delete.jsonl | jq .
-$ ./bin/bayard commit | jq .
+
    $ cat ./examples/bulk_delete.jsonl | xargs -0 ./bin/bayard bulk-delete
+$ ./bin/bayard commit
 
    
 
                     
diff --git a/docs/index.html b/docs/index.html
index ddd4066..0d0436f 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -81,7 +81,7 @@
 
         
@@ -151,7 +151,7 @@ 
    Bayard
    
                         
    Bayard
    
 
    Join the chat at https://gitter.im/bayard-search/bayard
 License: MIT
    
-
    Bayard is a full-text search and indexing server written in Rust built on top of Tantivy that implements The Raft Consensus Algorithm (raft-rs) and The gRPC (grpc-rs).
    
+
    Bayard is a full-text search and indexing server written in Rust built on top of Tantivy that implements Raft Consensus Algorithm and gRPC.
    
 Achieves consensus across all the nodes, ensures every change made to the system is made to a quorum of nodes.
    
 Bayard makes easy for programmers to develop search applications with advanced features and high availability.
    
 
    Features
    
diff --git a/docs/overview.html b/docs/overview.html
index 068dac4..a6f7559 100644
--- a/docs/overview.html
+++ b/docs/overview.html
@@ -81,7 +81,7 @@
 
         
@@ -151,7 +151,7 @@ 
    Bayard
    
                         
    Bayard
    
 
    Join the chat at https://gitter.im/bayard-search/bayard
 License: MIT
    
-
    Bayard is a full-text search and indexing server written in Rust built on top of Tantivy that implements The Raft Consensus Algorithm (raft-rs) and The gRPC (grpc-rs).
    
+
    Bayard is a full-text search and indexing server written in Rust built on top of Tantivy that implements Raft Consensus Algorithm and gRPC.
    
 Achieves consensus across all the nodes, ensures every change made to the system is made to a quorum of nodes.
    
 Bayard makes easy for programmers to develop search applications with advanced features and high availability.
    
 
    Features
    
diff --git a/docs/print.html b/docs/print.html
index 66ee728..b15175d 100644
--- a/docs/print.html
+++ b/docs/print.html
@@ -83,7 +83,7 @@
 
         
@@ -153,7 +153,7 @@ 
    Bayard
    
                         
    Bayard
    
 
    Join the chat at https://gitter.im/bayard-search/bayard
 License: MIT
    
-
    Bayard is a full-text search and indexing server written in Rust built on top of Tantivy that implements The Raft Consensus Algorithm (raft-rs) and The gRPC (grpc-rs).
    
+
    Bayard is a full-text search and indexing server written in Rust built on top of Tantivy that implements Raft Consensus Algorithm and gRPC.
    
 Achieves consensus across all the nodes, ensures every change made to the system is made to a quorum of nodes.
    
 Bayard makes easy for programmers to develop search applications with advanced features and high availability.
    
 
    Features
    
@@ -183,6 +183,14 @@ 
    Requirementsmake >= 3.81
 
  • protoc >= 3.9.2
    • 
 
+
    Install protoc-gen-rust
    
+
    $ cargo install protobuf-codegen
+
+
    
+
    Install protoc-gen-grpc-web
    
+
    $ curl -o /usr/local/bin/protoc-gen-grpc-web -L https://github.com/grpc/grpc-web/releases/download/1.0.7/protoc-gen-grpc-web-1.0.7-darwin-x86_64
+$ chmod +x /usr/local/bin/protoc-gen-grpc-web
+
    
 
    Build
    
 
    Build Bayard with the following command:
    
 
    $ make build
@@ -193,20 +201,7 @@ 
    Build
    
 
    Getting started
    
 
    Starting in standalone mode (Single node cluster)
    
 
    Running node in standalone mode is easy. You can start server with the following command:
    
-
    $ ./bin/bayard serve
-
    
-
    You'll see a startup message like following:
    
-
    [2019-11-27T00:30:45Z INFO bayard::server::server src/server/server.rs:119] listening on 0.0.0.0:5000
-[2019-11-27T00:30:45Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:723]  became follower at term 0
-[2019-11-27T00:30:45Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:295]  newRaft [peers: [1], term: 0, commit: 0, applied: 0, last_index: 0, last_term: 0]
-[2019-11-27T00:30:45Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:723]  became follower at term 1
-
-[...]
-
-[2019-11-27T00:30:48Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:1094]  is starting a new election at term 1
-[2019-11-27T00:30:48Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:743]  became candidate at term 2
-[2019-11-27T00:30:48Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:858]  received MsgRequestVoteResponse from 1 at term 2
-[2019-11-27T00:30:48Z INFO raft::raft /Users/m-osuka/.cargo/registry/src/github.com-1ecc6299db9ec823/raft-0.4.3/src/raft.rs:793]  became leader at term 2
+
    $ ./bin/bayard start 1
 
    
 
    Getting schema
    
 
    You can confirm current schema with the following command:
    
@@ -284,12 +279,12 @@ 
    Getting schema<
 
    
 
    Indexing document
    
 
    You can index document with the following command:
    
-
    $ ./bin/bayard put --id=1 --file=./examples/doc_1.json | jq .
-$ ./bin/bayard commit | jq .
+
    $ cat ./examples/doc_1.json | xargs -0 ./bin/bayard set 1
+$ ./bin/bayard commit
 
    
 
    Getting document
    
 
    You can get document with the following command:
    
-
    $ ./bin/bayard get --id=1 | jq .
+
    $ ./bin/bayard get 1 | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    {
@@ -319,8 +314,8 @@ 
    Getting doc
 
    
 
    Indexing documents in bulk
    
 
    You can index documents in bulk with the following command:
    
-
    $ ./bin/bayard put --bulk --file=./examples/bulk_put.jsonl | jq .
-$ ./bin/bayard commit | jq .
+
    $ cat ./examples/bulk_put.jsonl | xargs -0 ./bin/bayard bulk-set
+$ ./bin/bayard commit
 
    
 
    Searching documents
    
 
    You can search documents with the following command:
    
@@ -396,13 +391,13 @@ 
    Searc
 
    
 
    Deleting document
    
 
    You can delete document with the following command:
    
-
    $ ./bin/bayard delete --id=1 | jq .
-$ ./bin/bayard commit | jq .
+
    $ ./bin/bayard delete 1
+$ ./bin/bayard commit
 
    
 
    Deleting documents in bulk
    
 
    You can delete documents in bulk with the following command:
    
-
    $ ./bin/bayard delete --bulk --file=./examples/bulk_delete.jsonl | jq .
-$ ./bin/bayard commit | jq .
+
    $ cat ./examples/bulk_delete.jsonl | xargs -0 ./bin/bayard bulk-delete
+$ ./bin/bayard commit
 
    
 
    Designing schema
    
 
    Schema
    
@@ -842,41 +837,70 @@ 
    Cluster modeStarting in cluster mode (3-node cluster)
    
 
    Running in standalone is not fault tolerant. If you need to improve fault tolerance, start servers in cluster mode.
 You can start servers in cluster mode with the following command:
    
-
    $ ./bin/bayard serve \
-    --id=1 \
-    --host=0.0.0.0 \
-    --port=5001 \
-    --data-directory=./data/1 \
-    --schema-file=./etc/schema.json
-
    
-
    $ ./bin/bayard serve \
-    --id=2 \
-    --host=0.0.0.0 \
-    --port=5002 \
-    --peers="1=0.0.0.0:5001" \
-    --data-directory=./data/2 \
-    --schema-file=./etc/schema.json
-
    
-
    $ ./bin/bayard serve \
-    --id=3 \
-    --host=0.0.0.0 \
-    --port=5003 \
-    --peers="1=0.0.0.0:5001,2=0.0.0.0:5002" \
-    --data-directory=./data/3 \
-    --schema-file=./etc/schema.json
+
    $ ./bin/bayard start \
+               --host=0.0.0.0 \
+               --raft-port=7001 \
+               --index-port=5001 \
+               --data-directory=./data/node1 \
+               --schema-file=./etc/schema.json \
+               --tokenizer-file=./etc/tokenizer.json \
+               1
+
    
+
    $ ./bin/bayard start \
+               --host=0.0.0.0 \
+               --raft-port=7002 \
+               --index-port=5002 \
+               --peer-raft-address=0.0.0.0:7001 \
+               --data-directory=./data/node2 \
+               --schema-file=./etc/schema.json \
+               --tokenizer-file=./etc/tokenizer.json \
+               2
+
    
+
    $ ./bin/bayard start \
+               --host=0.0.0.0 \
+               --raft-port=7003 \
+               --index-port=5003 \
+               --peer-raft-address=0.0.0.0:7001 \
+               --data-directory=./data/node3 \
+               --schema-file=./etc/schema.json \
+               --tokenizer-file=./etc/tokenizer.json \
+               3
 
    
 
    The above commands run servers on the same host, so each server must listen on a different port. This would not be necessary if each server runs on a different host.
 Recommend 3 or more odd number of servers in the cluster to avoid split-brain.
    
 When deploying to a single host, if that host goes down due to hardware failure, all of the servers in the cluster will be stopped, so recommend deploying to a different host.
    
 
    Cluster peers
    
 
    You can check the peers in the cluster with the following command:
    
-
    $ ./bin/bayard peers --server localhost:5001 | jq .
+
    $ ./bin/bayard status
+               --server=0.0.0.0:5001 | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    {
-  "1": "0.0.0.0:5001",
-  "2": "0.0.0.0:5002",
-  "3": "0.0.0.0:5003"
+  "leader": 1,
+  "nodes": [
+    {
+      "address": {
+        "index_address": "0.0.0.0:5001",
+        "raft_address": "0.0.0.0:7001"
+      },
+      "id": 1
+    },
+    {
+      "address": {
+        "index_address": "0.0.0.0:5002",
+        "raft_address": "0.0.0.0:7002"
+      },
+      "id": 2
+    },
+    {
+      "address": {
+        "index_address": "0.0.0.0:5003",
+        "raft_address": "0.0.0.0:7003"
+      },
+      "id": 3
+    }
+  ],
+  "status": "OK"
 }
 
    
 
    Remove a server from a cluster
    
@@ -884,8 +908,8 @@ 
    $ ./bin/bayard leave \
-    --servers=127.0.0.1:5001 \
-    --id=3
+               --servers=0.0.0.0:5001 \
+               3
 
    
 
    Accessing over the HTTP
    
 
    Bayard supports gRPC connections, but some users may want to use the traditional RESTful API over HTTP. Gateways are useful in such cases.
    
@@ -899,21 +923,8 @@ 
    Using Gateway
 
    REST API
    
 
    See following documents:
    
 
-
    Scheduling jobs
    
-
    You can use the Job scheduler to execute periodic commits and index merges.
    
-
    Using Job scheduler
    
-
    Start the job scheduler as follows:
    
-
    $ ./bin/bayard schedule
-
    
-
    To specify job scheduler settings:
    
-
    $ ./bin/bayard schedule --commit="0/10 * * * * * *" --merge="0 0 2 * * * *"
-
    
-
    The format of the settings is very similar to the crontab.
    
-
    sec   min   hour   day-of-month   month   day-of-week   year
-*     *     *      *              *       *             *
-
    
 
    Running on Docker
    
 
    See the available Docker container image version at the following URL:
    
 
      
@@ -931,494 +942,1140 @@ 
      Reference
      
 
+
      Command-line interface
      
+
      Several command line interfaces are available to manage Bayard.
+See the following list:
      
+
        
+
      • 
+
        bayard
        
+     Manage Bayard index server
        
+
        • 
+
      • 
+
        bayard-rest
        
+     Manage Bayard REST server
        
+
        • 
 
      
 
      bayard
      
-
      The bayard CLI manages server, cluster and index.
      
+
      DESCRIPTION
      
+
      Manage Bayard index server
      
 
      USAGE
      
-
      bayard <SUBCOMMAND>
-
      
+
      bayard <SUBCOMMAND>
      
 
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      SUBCOMMANDS
      
-
      serve       The `bayard serve` CLI starts the server.
-probe       The `bayard probe` CLI probes the server.
-peers       The `bayard peers` CLI shows the peer addresses of the cluster that the specified server is joining.
-metrics     The `bayard metrics` CLI shows the server metrics of the specified server. The metrics are output in
-            Prometheus exposition format.
-leave       The `bayard leave` CLI removes the server with the specified ID from the cluster that the specified
-            server is joining.
-put         The `bayard put` CLI puts a document with the specified ID and field. If specify an existing ID, it
-            will be overwritten with the new document.
-get         The `bayard get` CLI gets a document with the specified ID.
-delete      The `bayard delete` CLI deletes a document with the specified ID.
-commit      The `bayard commit` CLI commits updates made to the index.
-rollback    The `bayard rollback` CLI rolls back any updates made to the index to the last committed state.
-merge       The `bayard merge` CLI merges fragmented segments in the index.
-search      The `bayard search` CLI searches documents from the index.
-schema      The `bayard schema` CLI shows the index schema that the server applied.
-schedule    The `bayard schedule` CLI starts the job scheduler.
-gateway     The `bayard gateway` CLI starts a gateway for access the server over HTTP.
-help        Prints this message or the help of the given subcommand(s)
-
      
-
      EXAMPLES
      
-
      To print version information:
      
-
      $ ./bin/bayard -v
-
      
-
      bayard serve
      
-
      The bayard serve CLI starts the server.
      
+
        
+
      • 
+
        start
        
+     Start index server
        
+
        • 
+
      • 
+
        leave
        
+     Delete document from index server
        
+
        • 
+
      • 
+
        get
        
+     Get document from index server
        
+
        • 
+
      • 
+
        set
        
+     Set document to index server
        
+
        • 
+
      • 
+
        delete
        
+     Delete document from index server
        
+
        • 
+
      • 
+
        bulk-set
        
+     Set documents to index server in bulk
        
+
        • 
+
      • 
+
        bulk-delete
        
+     Delete documents from index server in bulk
        
+
        • 
+
      • 
+
        commit
        
+     Commit index
        
+
        • 
+
      • 
+
        rollback
        
+     Rollback index
        
+
        • 
+
      • 
+
        merge
        
+     Merge index
        
+
        • 
+
      • 
+
        schema
        
+     Shows index schema that applied
        
+
        • 
+
      • 
+
        search
        
+     Get document from index server
        
+
        • 
+
      • 
+
        status
        
+     Shows system status
        
+
        • 
+
      • 
+
        metrics
        
+     Shows system metrics
        
+
        • 
+
      • 
+
        help
        
+     Prints this message or the help of the given subcommand(s)
        
+
        • 
+
      
+
      bayard start
      
+
      DESCRIPTION
      
+
      Start index server.
      
 
      USAGE
      
-
      bayard serve [OPTIONS]
-
      
+
      bayard start [OPTIONS] [ID]
      
 
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -i, --id <ID>
-        Server ID. Must specify a numeric ID that is unique within the cluster. If not specified, use the default
-        ID. [default: 1]
--H, --host <HOST>
-        Host address. Must specify the host name or IP address. If not specified, use the default address. [default:
-        0.0.0.0]
--P, --port <PORT>
-        Port number. This port is used for communication via gRPC. If not specified, use the default port. [default:
-        5000]
--p, --peers <ID=IP:PORT>...
-        Server ID and addresses in an existing cluster separated by ",". If specified, the server will join the
-        cluster.
--d, --data-directory <DATA_DIRECTORY>
-        Data directory. Stores index, snapshots, and raft logs. If not specified, use the default directory.
-        [default: ./data]
--s, --schema-file <SCHEMA_FILE>
-        Schema file. Must specify An existing file name. If not specified, use the default schema file. [default:
-        ./etc/schema.json]
--t, --indexer-threads <INDEXER_THREADS>
-        Number of indexer threads. If not specified, number of CPU cores - 1 will be used. [default: 7]
-
--m, --indexer-memory-size <INDEXER_MEMORY_SIZE>
-        Total memory size (in bytes) used by the indexer. It will be split for the different thread. If not
-        specified, use the default. [default: 1000000000]
-
      
-
      EXAMPLES
      
+
        
+
      • 
+
        -H, --host <HOST>
        
+     Node address. [default: 0.0.0.0]
        
+
        • 
+
      • 
+
        -r, --raft-port <RAFT_PORT>
        
+     Raft service port number. [default: 7000]
        
+
        • 
+
      • 
+
        -i, --index-port <INDEX_PORT>
        
+     Index service port number [default: 5000]
        
+
        • 
+
      • 
+
        -p, --peer-raft-address <IP:PORT>
        
+     Raft address of a peer node running in an existing cluster.
        
+
        • 
+
      • 
+
        -d, --data-directory <DATA_DIRECTORY>
        
+     Data directory. Stores index, snapshots, and raft logs. [default: ./data]
        
+
        • 
+
      • 
+
        -s, --schema-file <SCHEMA_FILE>
        
+     Schema file. Must specify An existing file name. [default: ./etc/schema.json]
        
+
        • 
+
      • 
+
        -T, --tokenizer-file <TOKENIZER_FILE>
        
+     Tokenizer file. Must specify An existing file name. [default: ./etc/tokenizer.json]
        
+
        • 
+
      • 
+
        -t, --indexer-threads <INDEXER_THREADS>
+     Number of indexer threads. [default: 7]
        
+
        • 
+
      • 
+
        -m, --indexer-memory-size <INDEXER_MEMORY_SIZE>
        
+     Total memory size (in bytes) used by the indexer. [default: 1000000000]
        
+
        • 
+
      
+
      ARGS
      
+
        
+
      • <ID>
        
+     Node ID.
        • 
+
      
+
      EXAMPLES
      
 
      To start a server with default options:
      
-
      $ ./bin/bayard serve
+
      $ ./bin/bayard start 1
 
      
 
      To start a server with options:
      
-
      $ ./bin/bayard serve \
-      --id=1 \
-      --host=0.0.0.0 \
-      --port=5001 \
-      --data-directory=./data/1 \
-      --schema-file=./etc/schema.json
-
      
-
      bayard probe
      
-
      The bayard probe CLI probes the server.
      
-
      USAGE
      
-
      bayard probe [OPTIONS]
+
      $ ./bin/bayard start \
+               --host=0.0.0.0 \
+               --raft-port=7001 \
+               --index-port=5001 \
+               --data-directory=./data/node1 \
+               --schema-file=./etc/schema.json \
+               --tokenizer-file=./etc/tokenizer.json \
+               1
 
      
+
      bayard leave
      
+
      DESCRIPTION
      
+
      Delete node from the cluster
      
+
      USAGE
      
+
      bayard leave [OPTIONS] [ID]
      
 
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -s, --server <IP:PORT>    Server address in an existing cluster. [default: 127.0.0.1:5000]
-
      
-
      EXAMPLES
      
-
      To probe a server with default options:
      
-
      $ ./bin/bayard probe
-
      
+
        
+
      • -s, --server <IP:PORT>
        
+     Raft service address. [default: 127.0.0.1:7000]
        • 
+
      
+
      ARGS
      
+
        
+
      • <ID>
        
+     Node ID to be removed from the cluster.
        • 
+
      
+
      EXAMPLES
      
 
      To probe a server with options:
      
-
      $ ./bin/bayard probe --server=127.0.0.1:5001
+
      $ ./bin/bayard leave --servers=127.0.0.1:5001 3
 
      
-
      bayard peers
      
-
      The bayard peers CLI shows the peer addresses of the cluster that the specified server is joining.
      
+
      bayard get
      
+
      DESCRIPTION
      
+
      Get document from index server
      
 
      USAGE
      
-
      bayard peers [OPTIONS]
-
      
+
      bayard get [OPTIONS] [ID]
      
 
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -s, --server <IP:PORT>    Server address in an existing cluster. [default: 127.0.0.1:5000]
-
      
-
      EXAMPLES
      
-
      To show peers of the cluster with default options:
      
-
      $ ./bin/bayard peers
+
        
+
      • -s, --server <IP:PORT>
        
+     Index service address. [default: 127.0.0.1:5000]
        • 
+
      
+
      ARGS
      
+
        
+
      • <ID>
        
+     A unique ID that identifies the document in the index server.
        • 
+
      
+
      EXAMPLES
      
+
      To get a document with default options:
      
+
      $ ./bin/bayard get --server=192.168.11.10:5001 1
 
      
-
      To show peers of the cluster with options:
      
-
      $ ./bin/bayard peers --server=127.0.0.1:5001
+
      You'll see the result in JSON format. The result of the above command is:
      
+
      {
+  "_id": [
+    "1"
+  ],
+  "category": [
+    "/category/search/server",
+    "/language/rust"
+  ],
+  "description": [
+    "Bayard is a full text search and indexing server, written in Rust, built on top of Tantivy."
+  ],
+  "name": [
+    "Bayard"
+  ],
+  "popularity": [
+    1152
+  ],
+  "timestamp": [
+    "2019-12-19T01:41:00+00:00"
+  ],
+  "url": [
+    "https://github.com/bayard-search/bayard"
+  ]
+}
 
      
-
      bayard metrics
      
-
      The bayard metrics CLI shows the server metrics of the specified server. The metrics are output in Prometheus exposition format.
      
+
      bayard set
      
+
      DESCRIPTION
      
+
      Set document to index server
      
 
      USAGE
      
-
      bayard metrics [OPTIONS]
-
      
+
      bayard bulk-set [OPTIONS] [DOCS]
      
 
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -s, --server IP:PORT    Server address in an existing cluster. [default: 127.0.0.1:5000]
      
-
      EXAMPLES
      
-
      To show metrics with default options:
      
-
      $ ./bin/bayard metrics
-
      
-
      To show metrics with options:
      
-
      $ ./bin/bayard metrics --server=127.0.0.1:5001
+
        
+
      • -s, --server <IP:PORT>
        
+     Index service address. [default: 127.0.0.1:5000]
        • 
+
      
+
      ARGS
      
+
        
+
      • <DOCS>
        
+     Document containing the unique ID to be indexed.
        • 
+
      
+
      EXAMPLES
      
+
      To put a document:
      
+
      $ cat ./examples/doc_1.json | xargs -0 ./bin/bayard set 1
 
      
-
      bayard leave
      
-
      The bayard leave CLI removes the server with the specified ID from the cluster that the specified server is joining.
      
+
      bayard delete
      
+
      DESCRIPTION
      
+
      Delete document from index server
      
 
      USAGE
      
-
      bayard leave [OPTIONS]
-
      
+
      bayard delete [OPTIONS] [ID]
      
 
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -s, --servers <IP:PORT>...    Server addresses in an existing cluster separated by ",". If not specified, use
-                              default servers. [default: 127.0.0.1:5000]
--i, --id <ID>                 Node ID to be removed from the cluster that specified server is joining. [default: 1]
-
      
-
      EXAMPLES
      
-
      To remove a server with default options:
      
-
      $ ./bin/bayard leave
-
      
-
      To probe a server with options:
      
-
      $ ./bin/bayard leave --servers=127.0.0.1:5001 --id=3
+
        
+
      • -s, --server <IP:PORT>
        
+     Index service address. [default: 127.0.0.1:5000]
        • 
+
      
+
      ARGS
      
+
        
+
      • <ID>
        
+     A unique ID that identifies the document in the index server.
        • 
+
      
+
      EXAMPLES
      
+
      To delete a document:
      
+
      $ ./bin/bayard delete --server=0.0.0.0:5001 1
 
      
-
      bayard put
      
-
      The bayard put CLI puts a document with the specified ID and field. If specify an existing ID, it will be overwritten
-with the new document.
      
+
      bayard bulk-set
      
+
      DESCRIPTION
      
+
      Set documents to index server in bulk
      
 
      USAGE
      
-
      bayard put [OPTIONS] <DOC_ID> <FIELDS>
-
      
+
      bayard set [OPTIONS] [ARGS]
      
 
      FLAGS
      
-
      -b, --bulk       A flag indicating whether or not to put documents in bulk.
--h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -s, --servers <IP:PORT>...    Server addresses in an existing cluster separated by ",". If not specified, use
-                              default servers. [default: 127.0.0.1:5000]
--i, --id <ID>                 A unique value that identifies the document in the index. If specified, the existing
-                              document ID in the document is overwritten.
--f, --file <FILE>             File path that document(s) expressed in JSON or JSONL format.
-
      
-
      EXAMPLES
      
-
      To put a document:
      
-
      $ ./bin/bayard put --id=1 --file=./examples/doc_1.json
-
      
+
        
+
      • -s, --server <IP:PORT>
        
+     Index service address. [default: 127.0.0.1:5000]
        • 
+
      
+
      ARGS
      
+
        
+
      • 
+
        <ID>
        
+     A unique ID that identifies the document in the index server.
        
+
        • 
+
      • 
+
        <FIELDS>
        
+     Fields of document to be indexed.
        
+
        • 
+
      
+
      EXAMPLES
      
 
      To put documents in bulk:
      
-
      $ ./bin/bayard put --bulk --file=./examples/bulk_put.jsonl
+
      $ cat ./examples/bulk_put.jsonl | xargs -0 ./bin/bayard bulk-set
 
      
-
      bayard get
      
-
      The bayard get CLI gets a document with the specified ID.
      
+
      bayard bulk-delete
      
+
      DESCRIPTION
      
+
      Delete documents from index server in bulk
      
 
      USAGE
      
-
      bayard get [OPTIONS] <DOC_ID>
-
      
+
      bayard set [OPTIONS] [ARGS]
      
 
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -s, --servers <IP:PORT>...    Server addresses in an existing cluster separated by ",". If not specified, use
-                              default servers. [default: 127.0.0.1:5000]
--i, --id <ID>                 A unique value that identifies the document in the index.
-
      
-
      EXAMPLES
      
-
      To get a document with default options:
      
-
      $ ./bin/bayard get --id=1
+
        
+
      • -s, --server <IP:PORT>
        
+     Index service address. [default: 127.0.0.1:5000]
        • 
+
      
+
      ARGS
      
+
        
+
      • 
+
        <ID>
        
+     A unique ID that identifies the document in the index server.
        
+
        • 
+
      • 
+
        <FIELDS>
        
+     Fields of document to be indexed.
        
+
        • 
+
      
+
      EXAMPLES
      
+
      To delete documents in bulk:
      
+
      $ cat ./examples/bulk_delete.jsonl | xargs -0 ./bin/bayard bulk-delete
 
      
 
      bayard commit
      
-
      The bayard commit CLI commits updates made to the index.
      
+
      DESCRIPTION
      
+
      Commit index
      
 
      USAGE
      
-
      bayard commit [OPTIONS]
-
      
+
      bayard commit [OPTIONS]
      
 
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -s, --servers <IP:PORT>...    Server addresses in an existing cluster separated by ",". If not specified, use
-                              default servers. [default: 127.0.0.1:5000]
-
      
-
      EXAMPLES
      
-
      To commit an index with default options:
      
-
      $ ./bin/bayard commit
-
      
-
      To commit an index with options:
      
-
      $ ./bin/bayard commit --servers=127.0.0.1:5001
+
        
+
      • -s, --server <IP:PORT>
        
+     Index service address. [default: 127.0.0.1:5000]
        • 
+
      
+
      EXAMPLES
      
+
      To commit an index with options:
      
+
      $ ./bin/bayard commit --servers=127.0.0.1:5001
 
      
 
      bayard rollback
      
-
      The bayard rollback CLI rolls back any updates made to the index to the last committed state.
      
+
      DESCRIPTION
      
+
      Rollback index
      
 
      USAGE
      
-
      bayard rollback [OPTIONS]
-
      
+
      bayard commit [OPTIONS]
      
 
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -s, --servers <IP:PORT>...    Server addresses in an existing cluster separated by ",". If not specified, use
-                              default servers. [default: 127.0.0.1:5000]
-
      
-
      EXAMPLES
      
-
      To rollback an index with default options:
      
-
      $ ./bin/bayard rollback
-
      
+
        
+
      • -s, --server <IP:PORT>
        
+     Index service address. [default: 127.0.0.1:5000]
        • 
+
      
+
      EXAMPLES
      
 
      To rollback an index with options:
      
-
      $ ./bin/bayard rollback --servers=127.0.0.1:5001
+
      $ ./bin/bayard rollback --servers=127.0.0.1:5001
 
      
 
      bayard merge
      
-
      The bayard merge CLI merges fragmented segments in the index.
      
+
      DESCRIPTION
      
+
      Merge index
      
 
      USAGE
      
-
      bayard merge [OPTIONS]
-
      
+
      bayard merge [OPTIONS]
      
 
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -s, --servers <IP:PORT>...    Server addresses in an existing cluster separated by ",". If not specified, use
-                              default servers. [default: 127.0.0.1:5000]
-
      
-
      EXAMPLES
      
-
      To merge segments in the index with default options:
      
-
      $ ./bin/bayard merge
-
      
-
      To merge segments in the index with options:
      
-
      $ ./bin/bayard merge --servers=127.0.0.1:5001
+
        
+
      • -s, --server <IP:PORT>
        
+     Index service address. [default: 127.0.0.1:5000]
        • 
+
      
+
      EXAMPLES
      
+
      To merge an index with options:
      
+
      $ ./bin/bayard merge --servers=127.0.0.1:5001
 
      
-
      bayard search
      
-
      The bayard search CLI searches documents from the index.
      
+
      bayard schema
      
+
      DESCRIPTION
      
+
      Show index schema
      
 
      USAGE
      
-
      bayard search [OPTIONS] <QUERY>
-
      
+
      bayard schema [OPTIONS]
      
 
      FLAGS
      
-
      -c, --exclude-count    A flag indicating whether or not to exclude hit count in the search results.
--d, --exclude-docs     A flag indicating whether or not to exclude hit documents in the search results
--h, --help             Prints help information.
--v, --version          Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -s, --servers <IP:PORT>...              Server addresses in an existing cluster separated by ",". If not specified,
-                                        use default servers. [default: 127.0.0.1:5000]
--f, --from <FROM>                       Start position of fetching results. If not specified, use default value.
-                                        [default: 0]
--l, --limit <LIMIT>                     Limitation of amount that document to be returned. If not specified, use
-                                        default value. [default: 10]
--F, --facet-field <FACET_FIELD>         Hierarchical facet field name. [default: ]
--V, --facet-prefix <FACET_PREFIX>...    Hierarchical facet field value prefix.
-
      
-
      ARGS
      
-
      <QUERY>    Query string to search the index.
-
      
-
      EXAMPLES
      
-
      To search documents from the index with default options:
      
-
      $ ./bin/bayard search text:"rust"
+
        
+
      • -s, --server <IP:PORT>
        
+     Index service address. [default: 127.0.0.1:5000]
        • 
+
      
+
      EXAMPLES
      
+
      To show an index schema with options:
      
+
      $ ./bin/bayard schema --servers=127.0.0.1:5001
 
      
-
      To search documents from the index with options:
      
-
      $ ./bin/bayard search --servers=127.0.0.1:5001 --from=10 --limit=20 text:"rust"
+
      You'll see the result in JSON format. The result of the above command is:
      
+
      [
+  {
+    "name": "_id",
+    "type": "text",
+    "options": {
+      "indexing": {
+        "record": "basic",
+        "tokenizer": "raw"
+      },
+      "stored": true
+    }
+  },
+  {
+    "name": "url",
+    "type": "text",
+    "options": {
+      "indexing": {
+        "record": "freq",
+        "tokenizer": "default"
+      },
+      "stored": true
+    }
+  },
+  {
+    "name": "name",
+    "type": "text",
+    "options": {
+      "indexing": {
+        "record": "position",
+        "tokenizer": "en_stem"
+      },
+      "stored": true
+    }
+  },
+  {
+    "name": "description",
+    "type": "text",
+    "options": {
+      "indexing": {
+        "record": "position",
+        "tokenizer": "en_stem"
+      },
+      "stored": true
+    }
+  },
+  {
+    "name": "popularity",
+    "type": "u64",
+    "options": {
+      "indexed": true,
+      "fast": "single",
+      "stored": true
+    }
+  },
+  {
+    "name": "category",
+    "type": "hierarchical_facet"
+  },
+  {
+    "name": "timestamp",
+    "type": "date",
+    "options": {
+      "indexed": true,
+      "fast": "single",
+      "stored": true
+    }
+  }
+]
 
      
-
      bayard schema
      
-
      The bayard schema CLI shows the index schema that the server applied.
      
+
      bayard search
      
+
      DESCRIPTION
      
+
      Search documents from index server
      
 
      USAGE
      
-
      bayard schema [OPTIONS]
-
      
+
      bayard search [FLAGS] [OPTIONS] 
      
 
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -c, --exclude-count
        
+     A flag indicating whether or not to exclude hit count in the search results.
        
+
        • 
+
      • 
+
        -d, --exclude-docs
        
+     A flag indicating whether or not to exclude hit documents in the search results
        
+
        • 
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -s, --servers <IP:PORT>...    Server addresses in an existing cluster separated by ",". If not specified, use
-                              default servers. [default: 127.0.0.1:5000]
-
      
-
      EXAMPLES
      
-
      To show the index schema with default options:
      
-
      $ ./bin/bayard schema
+
        
+
      • 
+
        -s, --server <IP:PORT>
        
+     Index service address. [default: 127.0.0.1:5000]
        
+
        • 
+
      • 
+
        -f, --from <FROM>
        
+     Start position of fetching results. [default: 0]
        
+
        • 
+
      • 
+
        -l, --limit <LIMIT>
        
+     Limitation of amount that document to be returned. [default: 10]
        
+
        • 
+
      • 
+
        -F, --facet-field <FACET_FIELD>
        
+     Hierarchical facet field name. [default: ]
        
+
        • 
+
      • 
+
        -V, --facet-prefix <FACET_PREFIX>...
        
+     Hierarchical facet field value prefix.
        
+
        • 
+
      
+
      ARGS
      
+
        
+
      • <QUERY>
        
+     Query string to search the index.
        • 
+
      
+
      EXAMPLES
      
+
      To search documents from the index with options:
      
+
      $ ./bin/bayard search \
+               --server=0.0.0.0:5001 \
+               --facet-field=category \
+               --facet-prefix=/category/search \
+               --facet-prefix=/language \
+               description:rust | jq .
 
      
-
      To show the index schema with options:
      
-
      $ ./bin/bayard schema --servers=127.0.0.1:5001
+
      You'll see the result in JSON format. The result of the above command is:
      
+
      {
+  "count": 2,
+  "docs": [
+    {
+      "fields": {
+        "_id": [
+          "8"
+        ],
+        "category": [
+          "/category/search/library",
+          "/language/rust"
+        ],
+        "description": [
+          "Tantivy is a full-text search engine library inspired by Apache Lucene and written in Rust."
+        ],
+        "name": [
+          "Tantivy"
+        ],
+        "popularity": [
+          3142
+        ],
+        "timestamp": [
+          "2019-12-19T01:07:00+00:00"
+        ],
+        "url": [
+          "https://github.com/tantivy-search/tantivy"
+        ]
+      },
+      "score": 1.5722498
+    },
+    {
+      "fields": {
+        "_id": [
+          "1"
+        ],
+        "category": [
+          "/category/search/server",
+          "/language/rust"
+        ],
+        "description": [
+          "Bayard is a full text search and indexing server, written in Rust, built on top of Tantivy."
+        ],
+        "name": [
+          "Bayard"
+        ],
+        "popularity": [
+          1152
+        ],
+        "timestamp": [
+          "2019-12-19T01:41:00+00:00"
+        ],
+        "url": [
+          "https://github.com/bayard-search/bayard"
+        ]
+      },
+      "score": 1.5331805
+    }
+  ],
+  "facet": {
+    "category": {
+      "/language/rust": 2,
+      "/category/search/library": 1,
+      "/category/search/server": 1
+    }
+  }
+}
 
      
-
      bayard schedule
      
-
      The bayard schedule CLI starts the job scheduler.
      
+
      bayard status
      
+
      DESCRIPTION
      
+
      Show system status
      
 
      USAGE
      
-
      bayard schedule [OPTIONS]
-
      
+
      bayard status [OPTIONS]
      
 
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -s, --servers <IP:PORT>...        Server addresses in an existing cluster separated by ",". If not specified, use
-                                  default servers. [default: 127.0.0.1:5000]
--c, --commit <COMMIT_SCHEDULE>    Schedule for automatic commit in a cron-like format. If not specified, use default
-                                  schedule. [default: 0/10 * * * * * *]
--m, --merge <MERGE_SCHEDULE>      Schedule for automatic merge in a cron-like format. If not specified, use default
-                                  schedule. [default: 0 0 2 * * * *]
-
      
-
      SCHEDULE FORMAT
      
-
      The scheduling format is as follows:
      
-
      sec   min   hour   day-of-month   month   day-of-week   year
-*     *     *      *              *       *             *
-
      
-
      EXAMPLES
      
-
      To start job scheduler with default options:
      
-
      $ ./bin/bayard schedule
+
        
+
      • -s, --server <IP:PORT>
        
+     Index service address. [default: 127.0.0.1:5000]
        • 
+
      
+
      EXAMPLES
      
+
      To show an index schema with options:
      
+
      $ ./bin/bayard status --server=0.0.0.0:5001 | jq .
 
      
-
      To start job scheduler with options:
      
-
      $ ./bin/bayard schedule --commit="0/10 * * * * * *" --merge="0 0 2 * * * *"
+
      You'll see the result in JSON format. The result of the above command is:
      
+
      {
+  "leader": 1,
+  "nodes": [
+    {
+      "address": {
+        "index_address": "0.0.0.0:5001",
+        "raft_address": "0.0.0.0:7001"
+      },
+      "id": 1
+    },
+    {
+      "address": {
+        "index_address": "0.0.0.0:5002",
+        "raft_address": "0.0.0.0:7002"
+      },
+      "id": 2
+    },
+    {
+      "address": {
+        "index_address": "0.0.0.0:5003",
+        "raft_address": "0.0.0.0:7003"
+      },
+      "id": 3
+    }
+  ],
+  "status": "OK"
+}
 
      
-
      bayard gateway
      
-
      The bayard gateway CLI starts a gateway for access the server over HTTP.
      
+
      bayard metrics
      
+
      DESCRIPTION
      
+
      Shows system metrics
      
 
      USAGE
      
-
      bayard gateway [OPTIONS]
-
      
+
      bayard metrics [OPTIONS]
      
 
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
 
      OPTIONS
      
-
      -H, --host <HOST>             Host address. Must specify the host name or IP address. If not specified, use the
-                              default address. [default: 0.0.0.0]
--P, --port <PORT>             Port number. This port is used for communication via HTTP. If not specified, use the
-                              default port. [default: 8000]
--s, --servers <IP:PORT>...    Server addresses in an existing cluster separated by ",". If not specified, use
-                              default servers. [default: 127.0.0.1:5000]
+
        
+
      • -s, --server <IP:PORT>
        
+     Index service address. [default: 127.0.0.1:5000]
        • 
+
      
+
      EXAMPLES
      
+
      To show metrics with default options:
      
+
      $ ./bin/bayard metrics
 
      
+
      To show metrics with options:
      
+
      $ ./bin/bayard metrics --server=127.0.0.1:5001
+
      
+
      You'll see the result in Prometheus exposition format. The result of the above command is:
      
+
      # HELP bayard_applies_total Total number of applies.
+# TYPE bayard_applies_total counter
+bayard_applies_total{func="bulk_set"} 1
+bayard_applies_total{func="commit"} 1
+# HELP bayard_apply_duration_seconds The apply latencies in seconds.
+# TYPE bayard_apply_duration_seconds histogram
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.005"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.01"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.025"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.05"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.1"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.25"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.5"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="1"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="2.5"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="5"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="10"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="+Inf"} 1
+bayard_apply_duration_seconds_sum{func="bulk_set"} 0.001098082
+bayard_apply_duration_seconds_count{func="bulk_set"} 1
+bayard_apply_duration_seconds_bucket{func="commit",le="0.005"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="0.01"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="0.025"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="0.05"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="0.1"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="0.25"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="0.5"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="1"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="2.5"} 1
+bayard_apply_duration_seconds_bucket{func="commit",le="5"} 1
+bayard_apply_duration_seconds_bucket{func="commit",le="10"} 1
+bayard_apply_duration_seconds_bucket{func="commit",le="+Inf"} 1
+bayard_apply_duration_seconds_sum{func="commit"} 1.727736793
+bayard_apply_duration_seconds_count{func="commit"} 1
+
      
+
      bayard-rest
      
+
      DESCRIPTION
      
+
      Manage Bayard REST server
      
+
      USAGE
      
+
      bayard-rest <SUBCOMMAND>
      
+
      FLAGS
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
+
      SUBCOMMANDS
      
+
        
+
      • start
        
+     Start REST server
        • 
+
      
+
      bayard-rest start
      
+
      DESCRIPTION
      
+
      Start REST server
      
+
      USAGE
      
+
      bayard-rest start [OPTIONS]
      
+
      FLAGS
      
+
        
+
      • 
+
        -h, --help
        
+     Prints help information.
        
+
        • 
+
      • 
+
        -v, --version
        
+     Prints version information.
        
+
        • 
+
      
+
      OPTIONS
      
+
        
+
      • 
+
        -H, --host <HOST>
        
+     Node address. [default: 0.0.0.0]
        
+
        • 
+
      • 
+
        -p, --port <PORT>
        
+     HTTP service port number. [default: 8000]
        
+
        • 
+
      • 
+
        -s, --server <IP:PORT>
        
+     Index service address. [default: 127.0.0.1:5000]
        
+
        • 
+
      
 
      EXAMPLES
      
-
      To start gateway with default options:
      
-
      $ ./bin/bayard gateway
-
      
-
      To start gateway with options:
      
-
      $ ./bin/bayard gateway --servers=127.0.0.1:5001,127.0.0.1:5002,127.0.0.1:5003
+
      To start a server with default options:
      
+
      $ bayard start --host=192.168.1.22 --port=8080 --server=192.168.1.12:5001
 
      
 
      REST API
      
-
      API list
      
-
-
      The REST API can be used by starting the gateway with the “bayard gateway” CLI.
      
-See the next page for details on bayard gateway:
      
-
-
      Probe API
      
-
      Probe API probes the server.
      
+
      The REST API can be used by starting the gateway with the bayard-rest CLI.
      
+Several APIs are available to manage Bayard over the HTTP.
+See the following list:
      
+
        
+
      • 
+
        Get document API
        
+     Get API gets a document with the specified ID.
        
+
        • 
+
      • 
+
        Set document API
        
+     Set document API puts a document with the specified ID and field. If specify an existing ID, it will be overwritten with the new document.
        
+
        • 
+
      • 
+
        Delete document API
        
+     Delete document API deletes a document with the specified ID.
        
+
        • 
+
      • 
+
        Bulk set documents API
        
+     Bulk set API sets documents in bulk with the specified ID and field. If specify an existing ID, it will be overwritten with the new document.
        
+
        • 
+
      • 
+
        Bulk delete documents API
        
+     Bulk delete documents API deletes documents in bulk with the specified ID.
        
+
        • 
+
      • 
+
        Commit API
        
+     Commit API commits updates made to the index.
        
+
        • 
+
      • 
+
        rollback
        
+     TODO
        
+
        • 
+
      • 
+
        merge
        
+     TODO
        
+
        • 
+
      • 
+
        schema
        
+     TODO
        
+
        • 
+
      • 
+
        search
        
+     TODO
        
+
        • 
+
      • 
+
        status
        
+     TODO
        
+
        • 
+
      • 
+
        metrics
        
+     TODO
        
+
        • 
+
      
+
      Get document API
      
+
      Get jdocument API gets a document with the specified ID.
      
 
      Request
      
-
      GET /probe
+
      GET /v1/documents/<ID>
 
      
+
      Path parameters
      
+
        
+
      • <ID>
        
+     A unique value that identifies the document in the index.
        • 
+
      
 
      Examples
      
-
      To probe a server:
      
-
      $ curl -X GET 'http://localhost:8000/probe'
-
      
-
      Peers API
      
-
      Peers API shows the peer addresses of the cluster that the specified server is joining.
      
-
      Request
      
-
      GET /peers
-
      
-
      Examples
      
-
      To show peers of the cluster:
      
-
      $ curl -X GET 'http://localhost:8000/peers'
-
      
-
      Metrics API
      
-
      Metrics API shows the server metrics of the specified server. The metrics are output in Prometheus exposition format.
      
-
      Request
      
-
      GET /metrics
+
      To get a document:
      
+
      $ curl -X GET 'http://localhost:8000/v1/documents/1' | jq .
 
      
-
      Examples
      
-
      To show metrics:
      
-
      $ curl -X GET 'http://localhost:8000/metrics'
+
      You'll see the result in JSON format. The result of the above command is:
      
+
      {
+  "_id": [
+    "1"
+  ],
+  "category": [
+    "/category/search/server",
+    "/language/rust"
+  ],
+  "description": [
+    "Bayard is a full text search and indexing server, written in Rust, built on top of Tantivy."
+  ],
+  "name": [
+    "Bayard"
+  ],
+  "popularity": [
+    1152
+  ],
+  "timestamp": [
+    "2019-12-19T01:41:00+00:00"
+  ],
+  "url": [
+    "https://github.com/bayard-search/bayard"
+  ]
+}
 
      
-
      Put API
      
-
      Put API puts a document with the specified ID and field. If specify an existing ID, it will be overwritten with the new document.
      
-
      Request
      
-
      PUT /index/docs/<DOC_ID>
-PUT /index/docs
+
      Set document API
      
+
      Set document API sets a document with the specified ID and field. If specify an existing ID, it will be overwritten with the new document.
      
+
      Request
      
+
      PUT /v1/documents/<ID>
 
      
-
      Path parameters
      
+
      Path parameters
      
 
        
-
      • <DOC_ID>
        
-A unique value that identifies the document in the index. If specify an existing ID, the existing document in the index is overwritten.
        • 
+
      • <ID>
        
+     A unique value that identifies the document in the index. If specify an existing ID, the existing document in the index is overwritten.
        • 
 
      
 
      Request body
      
 
        
 
      • <DOCUMENT>
        
-Document(s) expressed in JSON or JSONL format
        • 
+     Document expressed in JSON format
 
      
 
      Example
      
 
      To put a document:
      
 
      $ curl -X PUT \
-    --header 'Content-Type: application/json' \
-    --data-binary @./examples/doc_1.json \
-    'http://localhost:8000/index/docs/1'
+       --header 'Content-Type: application/json' \
+       --data-binary @./examples/doc_1.json \
+       'http://localhost:8000/v1/documents/1'
 
      
-
      To put documents in bulk:
      
-
      $ curl -X PUT \
-    --header 'Content-Type: application/json' \
-    --data-binary @./examples/bulk_put.jsonl \
-    'http://localhost:8000/index/docs'
-
      
-
      Get API
      
-
      Get API gets a document with the specified ID.
      
-
      Request
      
-
      GET /index/docs/<DOC_ID>
+
      Delete document API
      
+
      Delete document API deletes a document with the specified ID.
      
+
      Request
      
+
      DELETE /v1/documents/<ID>
 
      
-
      Path parameters
      
+
      Path parameters
      
 
        
-
      • <DOC_ID>
        
-A unique value that identifies the document in the index.
        • 
+
      • <ID>
        
+     A unique value that identifies the document in the index.
        • 
 
      
-
      Examples
      
-
      To get a document:
      
-
      $ curl -X GET 'http://localhost:8000/index/docs/1'
+
      Examples
      
+
      To delete a document:
      
+
      $ curl -X DELETE 'http://localhost:8000/v1/documents/1'
 
      
-
      Delete API
      
-
      Delete API deletes a document with the specified ID.
      
-
      Request
      
-
      DELETE /index/docs/<DOC_ID>
-DELETE /index/docs
+
      Bulk set documents API
      
+
      Bulk set documents API sets documents in bulk with the specified ID and field. If specify an existing ID, it will be overwritten with the new document.
      
+
      Request
      
+
      PUT /v1/documents
 
      
-
      Path parameters
      
+
      Request body
      
 
        
-
      • <DOC_ID>
        
-A unique value that identifies the document in the index.
        • 
+
      • <DOCUMENTS>
        
+     Documents expressed in JSONL format
        • 
 
      
-
      Request body
      
+
      Example
      
+
      To put documents in bulk:
      
+
      $ curl -X PUT \
+       --header 'Content-Type: application/json' \
+       --data-binary @./examples/bulk_put.jsonl \
+       'http://localhost:8000/v1/documents'
+
      
+
      Bulk delete documents API
      
+
      Bulk delete documents API deletes documents in bulk with the specified ID.
      
+
      Request
      
+
      DELETE /v1/documents
+
      
+
      Request body
      
 
        
 
      • <DOCUMENT>
        
-Document(s) expressed in JSONL format
        • 
+     Document(s) expressed in JSONL format
 
      
-
      Examples
      
-
      To delete a document:
      
-
      $ curl -X DELETE 'http://localhost:8000/index/docs/1'
-
      
+
      Examples
      
 
      To delete documents in bulk:
      
-
      $ curl -X DELETE \
+
      $ curl -X DELETE \
     --header 'Content-Type: application/json' \
     --data-binary @./examples/bulk_delete.jsonl \
-    'http://localhost:8000/index/docs'
+    'http://localhost:8000/v1/documents'
+
      
+
      Commit API
      
+
      Commit API commits updates made to the index.
      
+
      Request
      
+
      GET /v1/commit
+
      
+
      Example
      
+
      To commit an index:
      
+
      $ curl -X GET 'http://localhost:8000/v1/commit'
+
      
+
      Rollback API
      
+
      Rollback API rolls back any updates made to the index to the last committed state.
      
+
      Request
      
+
      GET /v1/rollback
+
      
+
      Examples
      
+
      To rollback an index:
      
+
      $ curl -X GET 'http://localhost:8000/v1/rollback'
+
      
+
      Merge API
      
+
      Merge API merges fragmented segments in the index.
      
+
      Request
      
+
      GET /v1/merge
+
      
+
      Examples
      
+
      To merge segments in the index:
      
+
      $ curl -X GET 'http://localhost:8000/v1/merge'
+
      
+
      Schema API
      
+
      Schema API shows the index schema that the server applied.
      
+
      Request
      
+
      GET /v1/schema
+
      
+
      Examples
      
+
      To show the index schema:
      
+
      $ curl -X POST 'http://localhost:8000/v1/schema' | jq .
+
      
+
      You'll see the result in JSON format. The result of the above command is:
      
+
      [
+  {
+    "name": "_id",
+    "type": "text",
+    "options": {
+      "indexing": {
+        "record": "basic",
+        "tokenizer": "raw"
+      },
+      "stored": true
+    }
+  },
+  {
+    "name": "url",
+    "type": "text",
+    "options": {
+      "indexing": {
+        "record": "freq",
+        "tokenizer": "default"
+      },
+      "stored": true
+    }
+  },
+  {
+    "name": "name",
+    "type": "text",
+    "options": {
+      "indexing": {
+        "record": "position",
+        "tokenizer": "en_stem"
+      },
+      "stored": true
+    }
+  },
+  {
+    "name": "description",
+    "type": "text",
+    "options": {
+      "indexing": {
+        "record": "position",
+        "tokenizer": "en_stem"
+      },
+      "stored": true
+    }
+  },
+  {
+    "name": "popularity",
+    "type": "u64",
+    "options": {
+      "indexed": true,
+      "fast": "single",
+      "stored": true
+    }
+  },
+  {
+    "name": "category",
+    "type": "hierarchical_facet"
+  },
+  {
+    "name": "timestamp",
+    "type": "date",
+    "options": {
+      "indexed": true,
+      "fast": "single",
+      "stored": true
+    }
+  }
+]
 
      
 
      Search API
      
 
      Search API searches documents from the index.
      
-
      Request
      
-
      GET /index/search
+
      Request
      
+
      GET /v1/search
 
      
 
      Query parameters
      
 
-
      Example
      
+
      Example
      
 
      To search documents from the index:
      
-
      $ curl -X GET 'http://localhost:8000/index/search?query=search&from=0&limit=10'
-
      
-
      $ curl -X GET 'http://localhost:8000/index/search?query=search&from=0&limit=10&exclude_count'
-
      
-
      $ curl -X GET 'http://localhost:8000/index/search?query=search&from=0&limit=10&exclude_docs'
-
      
-
      $ curl -X GET 'http://localhost:8000/index/search?query=search&from=0&limit=10&facet_field=category&facet_prefix=/language&facet_prefix=/category/search'
-
      
-
      Commit API
      
-
      Commit API commits updates made to the index.
      
-
      Request
      
-
      GET /index/commit
+
      $ curl -X POST 'http://localhost:8000/v1/search?query=search&from=0&limit=10&facet_field=category&facet_prefix=/language&facet_prefix=/category/search'
 
      
-
      Example
      
-
      To commit an index:
      
-
      $ curl -X GET 'http://localhost:8000/index/commit'
-
      
-
      Rollback API
      
-
      Rollback API rolls back any updates made to the index to the last committed state.
      
-
      Request
      
-
      GET /index/rollback
-
      
-
      Examples
      
-
      To rollback an index:
      
-
      $ curl -X GET 'http://localhost:8000/index/rollback'
+
      You'll see the result in JSON format. The result of the above command is:
      
+
      {
+  "count": 9,
+  "docs": [
+    {
+      "fields": {
+        "_id": [
+          "3"
+        ],
+        "category": [
+          "/category/search/server",
+          "/language/java"
+        ],
+        "description": [
+          "Elasticsearch is a distributed, open source search and analytics engine for all types of data, including textual, numerical, geospatial, structured, and unstructured."
+        ],
+        "name": [
+          "Elasticsearch"
+        ],
+        "popularity": [
+          46054
+        ],
+        "timestamp": [
+          "2019-12-18T23:19:00+00:00"
+        ],
+        "url": [
+          "https://www.elastic.co/products/elasticsearch"
+        ]
+      },
+      "score": 10.516742
+    },
+    {
+      "fields": {
+        "_id": [
+          "1"
+        ],
+        "category": [
+          "/category/search/server",
+          "/language/rust"
+        ],
+        "description": [
+          "Bayard is a full text search and indexing server, written in Rust, built on top of Tantivy."
+        ],
+        "name": [
+          "Bayard"
+        ],
+        "popularity": [
+          1152
+        ],
+        "timestamp": [
+          "2019-12-19T01:41:00+00:00"
+        ],
+        "url": [
+          "https://github.com/bayard-search/bayard"
+        ]
+      },
+      "score": 1.4125781
+    },
+    {
+      "fields": {
+        "_id": [
+          "8"
+        ],
+        "category": [
+          "/category/search/library",
+          "/language/rust"
+        ],
+        "description": [
+          "Tantivy is a full-text search engine library inspired by Apache Lucene and written in Rust."
+        ],
+        "name": [
+          "Tantivy"
+        ],
+        "popularity": [
+          3142
+        ],
+        "timestamp": [
+          "2019-12-19T01:07:00+00:00"
+        ],
+        "url": [
+          "https://github.com/tantivy-search/tantivy"
+        ]
+      },
+      "score": 1.4125781
+    },
+    {
+      "fields": {
+        "_id": [
+          "6"
+        ],
+        "category": [
+          "/category/search/server",
+          "/language/rust"
+        ],
+        "description": [
+          "Toshi is meant to be a full-text search engine similar to Elasticsearch. Toshi strives to be to Elasticsearch what Tantivy is to Lucene."
+        ],
+        "name": [
+          "Toshi"
+        ],
+        "popularity": [
+          2448
+        ],
+        "timestamp": [
+          "2019-12-01T19:00:00+00:00"
+        ],
+        "url": [
+          "https://github.com/toshi-search/Toshi"
+        ]
+      },
+      "score": 1.389255
+    },
+    {
+      "fields": {
+        "_id": [
+          "11"
+        ],
+        "category": [
+          "/category/search/library",
+          "/language/python"
+        ],
+        "description": [
+          "Whoosh is a fast, pure Python search engine library."
+        ],
+        "name": [
+          "Whoosh"
+        ],
+        "popularity": [
+          0
+        ],
+        "timestamp": [
+          "2019-10-07T20:30:26+00:00"
+        ],
+        "url": [
+          "https://bitbucket.org/mchaput/whoosh/wiki/Home"
+        ]
+      },
+      "score": 0.2778122
+    },
+    {
+      "fields": {
+        "_id": [
+          "7"
+        ],
+        "category": [
+          "/category/search/server",
+          "/language/rust"
+        ],
+        "description": [
+          "Sonic is a fast, lightweight and schema-less search backend."
+        ],
+        "name": [
+          "Sonic"
+        ],
+        "popularity": [
+          7895
+        ],
+        "timestamp": [
+          "2019-12-10T14:13:00+00:00"
+        ],
+        "url": [
+          "https://github.com/valeriansaliou/sonic"
+        ]
+      },
+      "score": 0.2778122
+    },
+    {
+      "fields": {
+        "_id": [
+          "4"
+        ],
+        "category": [
+          "/category/search/server",
+          "/language/go"
+        ],
+        "description": [
+          "Blast is a full text search and indexing server, written in Go, built on top of Bleve."
+        ],
+        "name": [
+          "Blast"
+        ],
+        "popularity": [
+          654
+        ],
+        "timestamp": [
+          "2019-10-18T10:50:00+00:00"
+        ],
+        "url": [
+          "https://github.com/mosuka/blast"
+        ]
+      },
+      "score": 0.23746987
+    },
+    {
+      "fields": {
+        "_id": [
+          "5"
+        ],
+        "category": [
+          "/category/search/server",
+          "/language/go"
+        ],
+        "description": [
+          "Riot is Go Open Source, Distributed, Simple and efficient full text search engine."
+        ],
+        "name": [
+          "Riot"
+        ],
+        "popularity": [
+          4948
+        ],
+        "timestamp": [
+          "2019-12-15T22:12:00+00:00"
+        ],
+        "url": [
+          "https://github.com/go-ego/riot"
+        ]
+      },
+      "score": 0.23746987
+    },
+    {
+      "fields": {
+        "_id": [
+          "9"
+        ],
+        "category": [
+          "/category/search/library",
+          "/language/java"
+        ],
+        "description": [
+          "Apache Lucene is a high-performance, full-featured text search engine library written entirely in Java."
+        ],
+        "name": [
+          "Lucene"
+        ],
+        "popularity": [
+          3135
+        ],
+        "timestamp": [
+          "2019-12-19T05:08:00+00:00"
+        ],
+        "url": [
+          "https://lucene.apache.org/"
+        ]
+      },
+      "score": 0.22139496
+    }
+  ],
+  "facet": {
+    "category": {
+      "/language/java": 2,
+      "/language/python": 1,
+      "/category/search/library": 3,
+      "/language/rust": 4,
+      "/category/search/server": 6,
+      "/language/go": 2
+    }
+  }
+}
 
      
-
      Merge API
      
-
      Merge API merges fragmented segments in the index.
      
-
      Request
      
-
      GET /index/merge
+
      Status API
      
+
      Status API shows the cluster that the specified server is joining.
      
+
      Request
      
+
      GET /v1/status
 
      
 
      Examples
      
-
      To merge segments in the index:
      
-
      $ curl -X GET 'http://localhost:8000/index/merge'
+
      To show peers of the cluster:
      
+
      $ curl -X GET 'http://localhost:8000/v1/status'
 
      
-
      Schema API
      
-
      Schema API shows the index schema that the server applied.
      
-
      Request
      
-
      GET /index/schema
+
      You'll see the result in JSON format. The result of the above command is:
      
+
      {
+  "leader": 1,
+  "nodes": [
+    {
+      "address": {
+        "index_address": "0.0.0.0:5001",
+        "raft_address": "0.0.0.0:7001"
+      },
+      "id": 1
+    },
+    {
+      "address": {
+        "index_address": "0.0.0.0:5002",
+        "raft_address": "0.0.0.0:7002"
+      },
+      "id": 2
+    },
+    {
+      "address": {
+        "index_address": "0.0.0.0:5003",
+        "raft_address": "0.0.0.0:7003"
+      },
+      "id": 3
+    }
+  ],
+  "status": "OK"
+}
+
      
+
      Metrics API
      
+
      Metrics API shows the server metrics of the specified server. The metrics are output in Prometheus exposition format.
      
+
      Request
      
+
      GET /v1/metrics
 
      
 
      Examples
      
-
      To show the index schema:
      
-
      $ curl -X GET 'http://localhost:8000/index/schema'
+
      To show metrics:
      
+
      $ curl -X GET 'http://localhost:8000/v1/metrics'
+
      
+
      You'll see the result in Prometheus exposition format. The result of the above command is:
      
+
      # HELP bayard_applies_total Total number of applies.
+# TYPE bayard_applies_total counter
+bayard_applies_total{func="bulk_set"} 1
+bayard_applies_total{func="commit"} 1
+# HELP bayard_apply_duration_seconds The apply latencies in seconds.
+# TYPE bayard_apply_duration_seconds histogram
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.005"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.01"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.025"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.05"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.1"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.25"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.5"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="1"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="2.5"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="5"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="10"} 1
+bayard_apply_duration_seconds_bucket{func="bulk_set",le="+Inf"} 1
+bayard_apply_duration_seconds_sum{func="bulk_set"} 0.001098082
+bayard_apply_duration_seconds_count{func="bulk_set"} 1
+bayard_apply_duration_seconds_bucket{func="commit",le="0.005"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="0.01"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="0.025"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="0.05"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="0.1"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="0.25"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="0.5"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="1"} 0
+bayard_apply_duration_seconds_bucket{func="commit",le="2.5"} 1
+bayard_apply_duration_seconds_bucket{func="commit",le="5"} 1
+bayard_apply_duration_seconds_bucket{func="commit",le="10"} 1
+bayard_apply_duration_seconds_bucket{func="commit",le="+Inf"} 1
+bayard_apply_duration_seconds_sum{func="commit"} 1.727736793
+bayard_apply_duration_seconds_count{func="commit"} 1
 
      
 
                     
diff --git a/docs/reference.html b/docs/reference.html
index 3cf73af..216804a 100644
--- a/docs/reference.html
+++ b/docs/reference.html
@@ -81,7 +81,7 @@
 
         
@@ -150,8 +150,8 @@ 
      Bayard
      
                     
      
                         
      Reference
      
 
 
                     
      
@@ -165,7 +165,7 @@ 
      Reference
      
                         
 
                         
-                            
                         
@@ -183,7 +183,7 @@ 
      Reference
      
                 
 
                 
-                    
                 
diff --git a/docs/reference/bayard.html b/docs/reference/api.html
similarity index 59%
rename from docs/reference/bayard.html
rename to docs/reference/api.html
index af9f6a5..6c393f2 100644
--- a/docs/reference/bayard.html
+++ b/docs/reference/api.html
@@ -3,7 +3,7 @@
     
         
         
-        bayard - Bayard
+        APIs - Bayard
         
 
         
@@ -81,7 +81,7 @@
 
         
@@ -148,53 +148,73 @@ 
      Bayard
      
 
                 
      
                     
      
-                        
      bayard
      
-
      The bayard CLI manages server, cluster and index.
      
-
      USAGE
      
-
      bayard <SUBCOMMAND>
-
      
-
      FLAGS
      
-
      -h, --help       Prints help information.
--v, --version    Prints version information.
-
      
-
      SUBCOMMANDS
      
-
      serve       The `bayard serve` CLI starts the server.
-probe       The `bayard probe` CLI probes the server.
-peers       The `bayard peers` CLI shows the peer addresses of the cluster that the specified server is joining.
-metrics     The `bayard metrics` CLI shows the server metrics of the specified server. The metrics are output in
-            Prometheus exposition format.
-leave       The `bayard leave` CLI removes the server with the specified ID from the cluster that the specified
-            server is joining.
-put         The `bayard put` CLI puts a document with the specified ID and field. If specify an existing ID, it
-            will be overwritten with the new document.
-get         The `bayard get` CLI gets a document with the specified ID.
-delete      The `bayard delete` CLI deletes a document with the specified ID.
-commit      The `bayard commit` CLI commits updates made to the index.
-rollback    The `bayard rollback` CLI rolls back any updates made to the index to the last committed state.
-merge       The `bayard merge` CLI merges fragmented segments in the index.
-search      The `bayard search` CLI searches documents from the index.
-schema      The `bayard schema` CLI shows the index schema that the server applied.
-schedule    The `bayard schedule` CLI starts the job scheduler.
-gateway     The `bayard gateway` CLI starts a gateway for access the server over HTTP.
-help        Prints this message or the help of the given subcommand(s)
-
      
-
      EXAMPLES
      
-
      To print version information:
      
-
      $ ./bin/bayard -v
-
      
+                        
      REST API
      
+
      The REST API can be used by starting the gateway with the bayard-rest CLI.
      
+Several APIs are available to manage Bayard over the HTTP.
+See the following list:
      
+
        
+
      • 
+
        Get document API
        
+     Get API gets a document with the specified ID.
        
+
        • 
+
      • 
+
        Set document API
        
+     Set document API puts a document with the specified ID and field. If specify an existing ID, it will be overwritten with the new document.
        
+
        • 
+
      • 
+
        Delete document API
        
+     Delete document API deletes a document with the specified ID.
        
+
        • 
+
      • 
+
        Bulk set documents API
        
+     Bulk set API sets documents in bulk with the specified ID and field. If specify an existing ID, it will be overwritten with the new document.
        
+
        • 
+
      • 
+
        Bulk delete documents API
        
+     Bulk delete documents API deletes documents in bulk with the specified ID.
        
+
        • 
+
      • 
+
        Commit API
        
+     Commit API commits updates made to the index.
        
+
        • 
+
      • 
+
        rollback
        
+     TODO
        
+
        • 
+
      • 
+
        merge
        
+     TODO
        
+
        • 
+
      • 
+
        schema
        
+     TODO
        
+
        • 
+
      • 
+
        search
        
+     TODO
        
+
        • 
+
      • 
+
        status
        
+     TODO
        
+
        • 
+
      • 
+
        metrics
        
+     TODO
        
+
        • 
+