diff --git a/Cargo.lock b/Cargo.lock index ad575fa..ed699e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -123,29 +123,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "bayard" -version = "0.7.3" +version = "0.8.0" dependencies = [ - "bayard-client 0.7.3", - "bayard-proto 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", - "bayard-server 0.7.3", + "bayard-client 0.8.0", + "bayard-common 0.8.0", + "bayard-proto 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bayard-server 0.8.0", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "ctrlc 3.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "grpcio 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "iron 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "raft 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "router 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.52 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bayard-cli" +version = "0.8.0" +dependencies = [ + "bayard-client 0.8.0", + "bayard-common 0.8.0", + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.52 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "bayard-client" -version = "0.7.3" +version = "0.8.0" dependencies = [ - "bayard-proto 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", - "bayard-server 0.7.3", + "bayard-proto 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bayard-server 0.8.0", "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "grpcio 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -155,9 +167,18 @@ dependencies = [ "serde_json 1.0.52 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "bayard-common" +version = "0.8.0" +dependencies = [ + "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "ctrlc 3.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bayard-proto" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", @@ -168,11 +189,13 @@ dependencies = [ [[package]] name = "bayard-rest" -version = "0.7.3" +version = "0.8.0" dependencies = [ - "bayard-client 0.7.3", + "bayard-client 0.8.0", + "bayard-common 0.8.0", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "ctrlc 3.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "iron 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -184,14 +207,15 @@ dependencies = [ [[package]] name = "bayard-server" -version = "0.7.3" +version = "0.8.0" dependencies = [ "async-std 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "bayard-proto 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bayard-proto 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "cang-jie 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "grpcio 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "iron 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "jieba-rs 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "lindera-tantivy 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2052,7 +2076,7 @@ dependencies = [ "checksum backtrace-sys 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118" "checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" "checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" -"checksum bayard-proto 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)" = "fd1711d6bfd6ce49484d0e6d39cdefbc391240521cf645472414728e79f23dfc" +"checksum bayard-proto 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "16776e07b65db0f8eeb773ede6def4b60328ec4195f5add7d2b330d6b10b0c52" "checksum bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" "checksum bitpacking 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3744aff20a3437a99ebc0bb7733e9e60c7bf590478c9b897e95b38d57e5acb68" diff --git a/Cargo.toml b/Cargo.toml index 3523b68..89bf43f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,9 @@ [workspace] members = [ + "bayard-common", "bayard-server", "bayard-client", + "bayard-cli", "bayard-rest", "bayard", ] diff --git a/Makefile b/Makefile index 617ecd9..41797a0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ BIN_DIR ?= $(CURDIR)/bin DOCS_DIR ?= $(CURDIR)/docs +COMMON_VERSION ?= $(shell cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name=="bayard-common") | .version') SERVER_VERSION ?= $(shell cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name=="bayard-server") | .version') CLIENT_VERSION ?= $(shell cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name=="bayard-client") | .version') +CLI_VERSION ?= $(shell cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name=="bayard-cli") | .version') REST_VERSION ?= $(shell cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name=="bayard-rest") | .version') VERSION ?= $(shell cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name=="bayard") | .version') @@ -18,6 +20,7 @@ build: mkdir -p $(BIN_DIR) cargo build --release cp -p ./target/release/bayard $(BIN_DIR) + cp -p ./target/release/bayard-cli $(BIN_DIR) cp -p ./target/release/bayard-rest $(BIN_DIR) test: @@ -28,6 +31,10 @@ tag: git push origin v$(VERSION) publish: +ifeq ($(shell cargo show --json bayard-common | jq -r '.versions[].num' | grep $(COMMON_VERSION)),) + (cd bayard-common && cargo package && cargo publish) + sleep 10 +endif ifeq ($(shell cargo show --json bayard-server | jq -r '.versions[].num' | grep $(SERVER_VERSION)),) (cd bayard-server && cargo package && cargo publish) sleep 10 @@ -36,6 +43,10 @@ ifeq ($(shell cargo show --json bayard-client | jq -r '.versions[].num' | grep $ (cd bayard-client && cargo package && cargo publish) sleep 10 endif +ifeq ($(shell cargo show --json bayard-cli | jq -r '.versions[].num' | grep $(CLI_VERSION)),) + (cd bayard-cli && cargo package && cargo publish) + sleep 10 +endif ifeq ($(shell cargo show --json bayard-rest | jq -r '.versions[].num' | grep $(REST_VERSION)),) (cd bayard-rest && cargo package && cargo publish) sleep 10 diff --git a/bayard-cli/Cargo.toml b/bayard-cli/Cargo.toml new file mode 100644 index 0000000..2b901be --- /dev/null +++ b/bayard-cli/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "bayard-cli" +version = "0.8.0" +authors = ["Minoru Osuka "] +edition = "2018" +description = "Command line interface for Bayard." +documentation = "https://bayard-search.github.io/bayard/" +homepage = "https://github.com/bayard-search" +repository = "https://github.com/bayard-search/bayard/bayard-cli" +readme = "README.md" +keywords = ["full-text", "search", "index", "cli"] +categories = ["database"] +license = "MIT" + +[[bin]] +name = "bayard-cli" +path = "src/main.rs" + +[dependencies] +clap = "2.33.0" +serde_json = "1.0.51" + +bayard-client = { version = "0.8.0", path = "../bayard-client" } +bayard-common = { version = "0.8.0", path = "../bayard-common" } diff --git a/bayard-cli/README.md b/bayard-cli/README.md new file mode 100644 index 0000000..5587860 --- /dev/null +++ b/bayard-cli/README.md @@ -0,0 +1,8 @@ +# Bayard + +[![Join the chat at https://gitter.im/bayard-search/bayard](https://badges.gitter.im/bayard-search/bayard.svg)](https://gitter.im/bayard-search/bayard?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +Bayard is a full-text search and indexing server written in [Rust](https://www.rust-lang.org/) built on top of [Tantivy](https://github.com/tantivy-search/tantivy) that implements [Raft Consensus Algorithm](https://raft.github.io/) and [gRPC](https://grpc.io/). +Achieves consensus across all the nodes, ensures every change made to the system is made to a quorum of nodes. +Bayard makes easy for programmers to develop search applications with advanced features and high availability. diff --git a/bayard/src/cli/bulk_delete.rs b/bayard-cli/src/cli/bulk_delete.rs similarity index 88% rename from bayard/src/cli/bulk_delete.rs rename to bayard-cli/src/cli/bulk_delete.rs index a28e2c9..65ff7e5 100644 --- a/bayard/src/cli/bulk_delete.rs +++ b/bayard-cli/src/cli/bulk_delete.rs @@ -2,11 +2,7 @@ use clap::ArgMatches; use bayard_client::index::client::IndexClient; -use crate::log::set_logger; - pub fn run_bulk_delete_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - let server = matches.value_of("SERVER").unwrap(); let docs = matches.value_of("DOCS").unwrap(); diff --git a/bayard/src/cli/bulk_set.rs b/bayard-cli/src/cli/bulk_set.rs similarity index 88% rename from bayard/src/cli/bulk_set.rs rename to bayard-cli/src/cli/bulk_set.rs index 83a98cb..fd046fd 100644 --- a/bayard/src/cli/bulk_set.rs +++ b/bayard-cli/src/cli/bulk_set.rs @@ -2,11 +2,7 @@ use clap::ArgMatches; use bayard_client::index::client::IndexClient; -use crate::log::set_logger; - pub fn run_bulk_set_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - let server = matches.value_of("SERVER").unwrap(); let docs = matches.value_of("DOCS").unwrap(); diff --git a/bayard/src/cli/commit.rs b/bayard-cli/src/cli/commit.rs similarity index 85% rename from bayard/src/cli/commit.rs rename to bayard-cli/src/cli/commit.rs index ce76c57..2d04964 100644 --- a/bayard/src/cli/commit.rs +++ b/bayard-cli/src/cli/commit.rs @@ -2,11 +2,7 @@ use clap::ArgMatches; use bayard_client::index::client::IndexClient; -use crate::log::set_logger; - pub fn run_commit_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - let server = matches.value_of("SERVER").unwrap(); let mut index_client = IndexClient::new(server); diff --git a/bayard/src/cli/delete.rs b/bayard-cli/src/cli/delete.rs similarity index 87% rename from bayard/src/cli/delete.rs rename to bayard-cli/src/cli/delete.rs index 2d12c24..7a486b9 100644 --- a/bayard/src/cli/delete.rs +++ b/bayard-cli/src/cli/delete.rs @@ -2,11 +2,7 @@ use clap::ArgMatches; use bayard_client::index::client::IndexClient; -use crate::log::set_logger; - pub fn run_delete_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - let server = matches.value_of("SERVER").unwrap(); let id = matches.value_of("ID").unwrap(); diff --git a/bayard/src/cli/get.rs b/bayard-cli/src/cli/get.rs similarity index 91% rename from bayard/src/cli/get.rs rename to bayard-cli/src/cli/get.rs index 721329c..aca55d5 100644 --- a/bayard/src/cli/get.rs +++ b/bayard-cli/src/cli/get.rs @@ -2,11 +2,7 @@ use clap::ArgMatches; use bayard_client::index::client::IndexClient; -use crate::log::set_logger; - pub fn run_get_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - let server = matches.value_of("SERVER").unwrap(); let id = matches.value_of("ID").unwrap(); diff --git a/bayard/src/cli/leave.rs b/bayard-cli/src/cli/leave.rs similarity index 92% rename from bayard/src/cli/leave.rs rename to bayard-cli/src/cli/leave.rs index b50140c..a2e1de1 100644 --- a/bayard/src/cli/leave.rs +++ b/bayard-cli/src/cli/leave.rs @@ -2,11 +2,7 @@ use clap::ArgMatches; use bayard_client::raft::client::RaftClient; -use crate::log::set_logger; - pub fn run_leave_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - let server = matches.value_of("SERVER").unwrap(); let id = matches.value_of("ID").unwrap().parse::().unwrap(); diff --git a/bayard/src/cli/merge.rs b/bayard-cli/src/cli/merge.rs similarity index 85% rename from bayard/src/cli/merge.rs rename to bayard-cli/src/cli/merge.rs index 0a6556d..d5f62cf 100644 --- a/bayard/src/cli/merge.rs +++ b/bayard-cli/src/cli/merge.rs @@ -2,11 +2,7 @@ use clap::ArgMatches; use bayard_client::index::client::IndexClient; -use crate::log::set_logger; - pub fn run_merge_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - let server = matches.value_of("SERVER").unwrap(); let mut index_client = IndexClient::new(server); diff --git a/bayard/src/cli/mod.rs b/bayard-cli/src/cli/mod.rs similarity index 86% rename from bayard/src/cli/mod.rs rename to bayard-cli/src/cli/mod.rs index 9c14610..dfdf9a9 100644 --- a/bayard/src/cli/mod.rs +++ b/bayard-cli/src/cli/mod.rs @@ -5,11 +5,9 @@ pub mod delete; pub mod get; pub mod leave; pub mod merge; -pub mod metrics; pub mod rollback; pub mod root; pub mod schema; pub mod search; pub mod set; -pub mod start; pub mod status; diff --git a/bayard/src/cli/rollback.rs b/bayard-cli/src/cli/rollback.rs similarity index 85% rename from bayard/src/cli/rollback.rs rename to bayard-cli/src/cli/rollback.rs index 44aeb66..c4f5d4d 100644 --- a/bayard/src/cli/rollback.rs +++ b/bayard-cli/src/cli/rollback.rs @@ -2,11 +2,7 @@ use clap::ArgMatches; use bayard_client::index::client::IndexClient; -use crate::log::set_logger; - pub fn run_rollback_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - let server = matches.value_of("SERVER").unwrap(); let mut index_client = IndexClient::new(server); diff --git a/bayard/src/cli/root.rs b/bayard-cli/src/cli/root.rs similarity index 74% rename from bayard/src/cli/root.rs rename to bayard-cli/src/cli/root.rs index 4cbcb72..70a5f49 100644 --- a/bayard/src/cli/root.rs +++ b/bayard-cli/src/cli/root.rs @@ -1,5 +1,4 @@ use clap::{App, AppSettings, Arg, SubCommand}; -use num_cpus; use crate::cli::bulk_delete::run_bulk_delete_cli; use crate::cli::bulk_set::run_bulk_set_cli; @@ -8,129 +7,22 @@ use crate::cli::delete::run_delete_cli; use crate::cli::get::run_get_cli; use crate::cli::leave::run_leave_cli; use crate::cli::merge::run_merge_cli; -use crate::cli::metrics::run_metrics_cli; use crate::cli::rollback::run_rollback_cli; use crate::cli::schema::run_schema_cli; use crate::cli::search::run_search_cli; use crate::cli::set::run_set_cli; -use crate::cli::start::run_start_cli; use crate::cli::status::run_status_cli; pub fn run_root_cli() -> Result<(), std::io::Error> { - let cpus = num_cpus::get().to_owned(); - let threads; - if cpus > 1 { - threads = format!("{}", cpus - 1); - } else { - threads = format!("{}", cpus); - } - let app = App::new(crate_name!()) .setting(AppSettings::DeriveDisplayOrder) .setting(AppSettings::SubcommandRequiredElseHelp) .version(crate_version!()) .author(crate_authors!()) - .about("Manage Bayard index server") + .about("Bayard command-line interface") .help_message("Prints help information.") .version_message("Prints version information.") .version_short("v") - .subcommand( - SubCommand::with_name("start") - .name("start") - .setting(AppSettings::DeriveDisplayOrder) - .version(crate_version!()) - .author(crate_authors!()) - .about("Start index server") - .help_message("Prints help information.") - .version_message("Prints version information.") - .version_short("v") - .arg( - Arg::with_name("ID") - .help("Node ID.") - .value_name("ID") - .takes_value(true), - ) - .arg( - Arg::with_name("HOST") - .help("Node address.") - .short("H") - .long("host") - .value_name("HOST") - .default_value("0.0.0.0") - .takes_value(true), - ) - .arg( - Arg::with_name("RAFT_PORT") - .help("Raft service port number.") - .short("r") - .long("raft-port") - .value_name("RAFT_PORT") - .default_value("7000") - .takes_value(true), - ) - .arg( - Arg::with_name("INDEX_PORT") - .help("Index service port number") - .short("i") - .long("index-port") - .value_name("INDEX_PORT") - .default_value("5000") - .takes_value(true), - ) - .arg( - Arg::with_name("PEER_RAFT_ADDRESS") - .help("Raft address of a peer node running in an existing cluster.") - .short("p") - .long("peer-raft-address") - .value_name("IP:PORT") - .takes_value(true), - ) - .arg( - Arg::with_name("DATA_DIRECTORY") - .help("Data directory. Stores index, snapshots, and raft logs.") - .short("d") - .long("data-directory") - .value_name("DATA_DIRECTORY") - .default_value("./data") - .takes_value(true), - ) - .arg( - Arg::with_name("SCHEMA_FILE") - .help("Schema file. Must specify An existing file name.") - .short("s") - .long("schema-file") - .value_name("SCHEMA_FILE") - .default_value("./etc/schema.json") - .takes_value(true), - ) - .arg( - Arg::with_name("TOKENIZER_FILE") - .help("Tokenizer file. Must specify An existing file name.") - .short("T") - .long("tokenizer-file") - .value_name("TOKENIZER_FILE") - .default_value("./etc/tokenizer.json") - .takes_value(true), - ) - .arg( - Arg::with_name("INDEXER_THREADS") - .help("Number of indexer threads.") - .short("t") - .long("indexer-threads") - .value_name("INDEXER_THREADS") - .default_value(&threads) - .takes_value(true), - ) - .arg( - Arg::with_name("INDEXER_MEMORY_SIZE") - .help("Total memory size (in bytes) used by the indexer.") - .short("m") - .long("indexer-memory-size") - .value_name("INDEXER_MEMORY_SIZE") - .default_value("1000000000") - .takes_value(true), - ), - ) .subcommand( SubCommand::with_name("leave") .name("leave") @@ -469,32 +361,11 @@ pub fn run_root_cli() -> Result<(), std::io::Error> { .takes_value(true), ), ) - .subcommand( - SubCommand::with_name("metrics") - .name("metrics") - .setting(AppSettings::DeriveDisplayOrder) - .version(crate_version!()) - .author(crate_authors!()) - .about("Shows system metrics") - .help_message("Prints help information.") - .version_message("Prints version information.") - .version_short("v") - .arg( - Arg::with_name("SERVER") - .help("Index service address.") - .short("s") - .long("server") - .value_name("IP:PORT") - .default_value("0.0.0.0:5000") - .takes_value(true), - ), - ) .get_matches(); let (subcommand, some_options) = app.subcommand(); let options = some_options.unwrap(); let run_cli = match subcommand { - "start" => run_start_cli, "leave" => run_leave_cli, "search" => run_search_cli, "get" => run_get_cli, @@ -506,7 +377,6 @@ pub fn run_root_cli() -> Result<(), std::io::Error> { "rollback" => run_rollback_cli, "merge" => run_merge_cli, "schema" => run_schema_cli, - "metrics" => run_metrics_cli, "status" => run_status_cli, _ => panic!("Subcommand {} is unknown", subcommand), }; diff --git a/bayard/src/cli/schema.rs b/bayard-cli/src/cli/schema.rs similarity index 90% rename from bayard/src/cli/schema.rs rename to bayard-cli/src/cli/schema.rs index 2dbd6a0..b9be895 100644 --- a/bayard/src/cli/schema.rs +++ b/bayard-cli/src/cli/schema.rs @@ -2,11 +2,7 @@ use clap::ArgMatches; use bayard_client::index::client::IndexClient; -use crate::log::set_logger; - pub fn run_schema_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - let server = matches.value_of("SERVER").unwrap(); let mut index_client = IndexClient::new(server); diff --git a/bayard/src/cli/search.rs b/bayard-cli/src/cli/search.rs similarity index 96% rename from bayard/src/cli/search.rs rename to bayard-cli/src/cli/search.rs index c56707e..b1ee2d0 100644 --- a/bayard/src/cli/search.rs +++ b/bayard-cli/src/cli/search.rs @@ -2,11 +2,7 @@ use clap::ArgMatches; use bayard_client::index::client::IndexClient; -use crate::log::set_logger; - pub fn run_search_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - let server = matches.value_of("SERVER").unwrap(); let from = matches.value_of("FROM").unwrap().parse::().unwrap(); let limit = matches.value_of("LIMIT").unwrap().parse::().unwrap(); diff --git a/bayard/src/cli/set.rs b/bayard-cli/src/cli/set.rs similarity index 92% rename from bayard/src/cli/set.rs rename to bayard-cli/src/cli/set.rs index c288742..5d06583 100644 --- a/bayard/src/cli/set.rs +++ b/bayard-cli/src/cli/set.rs @@ -3,11 +3,7 @@ use serde_json::Value; use bayard_client::index::client::IndexClient; -use crate::log::set_logger; - pub fn run_set_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - let server = matches.value_of("SERVER").unwrap(); let id = matches.value_of("ID").unwrap(); let fields = matches.value_of("FIELDS").unwrap(); diff --git a/bayard/src/cli/status.rs b/bayard-cli/src/cli/status.rs similarity index 90% rename from bayard/src/cli/status.rs rename to bayard-cli/src/cli/status.rs index c744be9..550d803 100644 --- a/bayard/src/cli/status.rs +++ b/bayard-cli/src/cli/status.rs @@ -2,11 +2,7 @@ use clap::ArgMatches; use bayard_client::index::client::IndexClient; -use crate::log::set_logger; - pub fn run_status_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - let server = matches.value_of("SERVER").unwrap(); let mut index_client = IndexClient::new(server); diff --git a/bayard/src/lib.rs b/bayard-cli/src/lib.rs similarity index 61% rename from bayard/src/lib.rs rename to bayard-cli/src/lib.rs index 0946d2f..4a7e8b1 100644 --- a/bayard/src/lib.rs +++ b/bayard-cli/src/lib.rs @@ -2,5 +2,3 @@ extern crate clap; pub mod cli; -pub mod log; -pub mod signal; diff --git a/bayard-cli/src/main.rs b/bayard-cli/src/main.rs new file mode 100644 index 0000000..9547191 --- /dev/null +++ b/bayard-cli/src/main.rs @@ -0,0 +1,8 @@ +use bayard_cli::cli::root::run_root_cli; +use bayard_common::log::set_logger; + +fn main() -> Result<(), std::io::Error> { + set_logger(); + + run_root_cli() +} diff --git a/bayard-client/Cargo.toml b/bayard-client/Cargo.toml index d9ca201..275b424 100644 --- a/bayard-client/Cargo.toml +++ b/bayard-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bayard-client" -version = "0.7.3" +version = "0.8.0" authors = ["Minoru Osuka "] edition = "2018" description = "Client library for Bayard." @@ -21,5 +21,6 @@ raft = "0.4.3" rand = "0.7.3" serde_json = "1.0.51" -bayard-proto = "0.7.4" -bayard-server = { version = "0.7.3", path = "../bayard-server" } +bayard-proto = "0.8.0" + +bayard-server = { version = "0.8.0", path = "../bayard-server" } diff --git a/bayard-client/src/index/client.rs b/bayard-client/src/index/client.rs index 182f328..2a2b544 100644 --- a/bayard-client/src/index/client.rs +++ b/bayard-client/src/index/client.rs @@ -2,21 +2,21 @@ use std::collections::HashMap; use std::io::{Error, ErrorKind}; use std::sync::Arc; -use bincode::deserialize; -use grpcio::{ChannelBuilder, EnvBuilder}; -use log::*; -use protobuf::RepeatedField; - use bayard_proto::proto::commonpb::State; use bayard_proto::proto::indexpb::{ - BulkDeleteReq, BulkSetReq, CommitReq, DeleteReq, GetReq, MergeReq, MetricsReq, RollbackReq, - SchemaReq, SearchReq, SetReq, StatusReq, + BulkDeleteReq, BulkSetReq, CommitReq, DeleteReq, GetReq, MergeReq, RollbackReq, SchemaReq, + SearchReq, SetReq, StatusReq, }; use bayard_proto::proto::indexpb_grpc::IndexServiceClient; -use bayard_server::raft::config::NodeAddress; +use bincode::deserialize; +use grpcio::{ChannelBuilder, EnvBuilder}; +use log::*; +use protobuf::RepeatedField; use serde_json::json; use serde_json::Value; +use bayard_server::raft::config::NodeAddress; + fn create_client(address: String) -> IndexServiceClient { let env = Arc::new(EnvBuilder::new().build()); let ch = ChannelBuilder::new(env).connect(&address); @@ -24,13 +24,14 @@ fn create_client(address: String) -> IndexServiceClient { client } +#[derive(Clone)] pub struct IndexClient { - address: String, // specified server address - leader_id: u64, // leader's node id + address: String, + leader_id: u64, clients: HashMap>, addresses: HashMap, next_index: usize, - node_id: u64, // node id + node_id: u64, client_id: u64, } @@ -1088,108 +1089,6 @@ impl IndexClient { } } - pub fn metrics(&mut self) -> Result { - let mut req = MetricsReq::new(); - req.set_client_id(self.client_id); - - let max_retry = 10; - let mut cnt_retry = 0; - - loop { - if max_retry < cnt_retry { - return Err(Error::new( - ErrorKind::Other, - format!("max retry count has been exceeded: max_retry={}", max_retry), - )); - } - - let client = match self.clients.get(&self.node_id) { - Some(c) => c, - _ => { - return Err(Error::new( - ErrorKind::Other, - format!("failed to get client for node: id={}", self.node_id), - )); - } - }; - - let reply = match client.metrics(&req) { - Ok(r) => r, - _ => { - return Err(Error::new( - ErrorKind::Other, - "failed to request getting metrics", - )); - } - }; - - // update address list and clients - if reply.get_address_map().len() > 0 { - let address_map: HashMap = - deserialize(&reply.get_address_map()).unwrap(); - // add new ids - for (id, address) in &address_map { - if let Some(grpc_address) = self.addresses.get(&id) { - if grpc_address == address.index_address.as_str() { - debug!( - "node has not been changed: id={}, address={}", - id, grpc_address - ); - } else { - debug!("update node: id={}, address={}", id, address.index_address); - self.addresses - .insert(id.clone(), address.index_address.clone()); - self.clients.insert( - id.clone(), - Arc::new(create_client(address.index_address.clone())), - ); - } - } else { - debug!("add node: id={}, address={}", id, address.index_address); - self.addresses - .insert(id.clone(), address.index_address.clone()); - self.clients.insert( - id.clone(), - Arc::new(create_client(address.index_address.clone())), - ); - } - } - - // remove unused ids - for (id, address) in &self.addresses.clone() { - if let Some(_) = address_map.get(&id) { - debug!("node is in use: id={}, address={}", id, address); - } else { - debug!("node is not in use: id={}, address={}", id, address); - self.addresses.remove(id); - self.clients.remove(id); - } - } - - debug!("addresses={:?}", self.addresses); - } - - // continue to use the same node. - for (id, address) in &self.addresses.clone() { - if address == self.address.as_str() { - self.node_id = *id; - break; - } - } - - match reply.get_state() { - State::OK => { - self.leader_id = reply.get_leader_id(); - return Ok(String::from(reply.get_metrics())); - } - _ => { - cnt_retry += 1; - warn!("failed to get metrics"); - } - } - } - } - pub fn status(&mut self) -> Result { let mut req = StatusReq::new(); req.set_client_id(self.client_id); diff --git a/bayard-common/Cargo.toml b/bayard-common/Cargo.toml new file mode 100644 index 0000000..a7cb3ca --- /dev/null +++ b/bayard-common/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "bayard-common" +version = "0.8.0" +authors = ["Minoru Osuka "] +edition = "2018" +description = "Common library for Bayard." +documentation = "https://bayard-search.github.io/bayard/" +homepage = "https://github.com/bayard-search" +repository = "https://github.com/bayard-search/bayard/bayard-common" +readme = "README.md" +keywords = ["full-text", "search", "index", "common"] +categories = ["database"] +license = "MIT" + +[dependencies] +crossbeam-channel = "0.4.2" +ctrlc = { version = "3.1.4", features = ["termination"] } +env_logger = "0.7.1" diff --git a/bayard-common/README.md b/bayard-common/README.md new file mode 100644 index 0000000..5587860 --- /dev/null +++ b/bayard-common/README.md @@ -0,0 +1,8 @@ +# Bayard + +[![Join the chat at https://gitter.im/bayard-search/bayard](https://badges.gitter.im/bayard-search/bayard.svg)](https://gitter.im/bayard-search/bayard?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +Bayard is a full-text search and indexing server written in [Rust](https://www.rust-lang.org/) built on top of [Tantivy](https://github.com/tantivy-search/tantivy) that implements [Raft Consensus Algorithm](https://raft.github.io/) and [gRPC](https://grpc.io/). +Achieves consensus across all the nodes, ensures every change made to the system is made to a quorum of nodes. +Bayard makes easy for programmers to develop search applications with advanced features and high availability. diff --git a/bayard-common/src/lib.rs b/bayard-common/src/lib.rs new file mode 100644 index 0000000..34b9ac3 --- /dev/null +++ b/bayard-common/src/lib.rs @@ -0,0 +1,2 @@ +pub mod log; +pub mod signal; diff --git a/bayard/src/log/mod.rs b/bayard-common/src/log/mod.rs similarity index 100% rename from bayard/src/log/mod.rs rename to bayard-common/src/log/mod.rs diff --git a/bayard/src/signal/mod.rs b/bayard-common/src/signal/mod.rs similarity index 100% rename from bayard/src/signal/mod.rs rename to bayard-common/src/signal/mod.rs diff --git a/bayard-rest/Cargo.toml b/bayard-rest/Cargo.toml index b1cc364..9421831 100644 --- a/bayard-rest/Cargo.toml +++ b/bayard-rest/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bayard-rest" -version = "0.7.3" +version = "0.8.0" authors = ["Minoru Osuka "] edition = "2018" description = "REST API server for Bayard." @@ -18,7 +18,8 @@ path = "src/main.rs" [dependencies] clap = "2.33.0" -env_logger = "0.7.1" +crossbeam-channel = "0.4.2" +ctrlc = { version = "3.1.4", features = ["termination"] } iron = "0.6.1" log = "0.4.8" logger = "0.4.0" @@ -27,4 +28,5 @@ router = "0.6.0" serde_json = "1.0.51" urlencoded = "0.6.0" -bayard-client = { version = "0.7.3", path = "../bayard-client" } +bayard-client = { version = "0.8.0", path = "../bayard-client" } +bayard-common = { version = "0.8.0", path = "../bayard-common" } diff --git a/bayard-rest/src/cli/mod.rs b/bayard-rest/src/cli/mod.rs deleted file mode 100644 index 0d2475f..0000000 --- a/bayard-rest/src/cli/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod root; -pub mod start; diff --git a/bayard-rest/src/cli/root.rs b/bayard-rest/src/cli/root.rs deleted file mode 100644 index 01f004e..0000000 --- a/bayard-rest/src/cli/root.rs +++ /dev/null @@ -1,63 +0,0 @@ -use clap::{App, AppSettings, Arg, SubCommand}; - -use crate::cli::start::run_start_cli; - -pub fn run_root_cli() -> Result<(), std::io::Error> { - let app = App::new(crate_name!()) - .setting(AppSettings::DeriveDisplayOrder) - .setting(AppSettings::SubcommandRequiredElseHelp) - .version(crate_version!()) - .author(crate_authors!()) - .about("Manage Bayard REST server") - .help_message("Prints help information.") - .version_message("Prints version information.") - .version_short("v") - .subcommand( - SubCommand::with_name("start") - .name("start") - .setting(AppSettings::DeriveDisplayOrder) - .version(crate_version!()) - .author(crate_authors!()) - .about("Start REST server") - .help_message("Prints help information.") - .version_message("Prints version information.") - .version_short("v") - .arg( - Arg::with_name("HOST") - .help("Node address.") - .short("H") - .long("host") - .value_name("HOST") - .default_value("0.0.0.0") - .takes_value(true), - ) - .arg( - Arg::with_name("PORT") - .help("HTTP service port number.") - .short("p") - .long("port") - .value_name("PORT") - .default_value("8000") - .takes_value(true), - ) - .arg( - Arg::with_name("SERVER") - .help("Index service address.") - .short("s") - .long("server") - .value_name("IP:PORT") - .default_value("0.0.0.0:5000") - .takes_value(true), - ), - ) - .get_matches(); - - let (subcommand, some_options) = app.subcommand(); - let options = some_options.unwrap(); - let run_cli = match subcommand { - "start" => run_start_cli, - _ => panic!("Subcommand {} is unknown", subcommand), - }; - - run_cli(options) -} diff --git a/bayard-rest/src/cli/start.rs b/bayard-rest/src/cli/start.rs deleted file mode 100644 index 8a66176..0000000 --- a/bayard-rest/src/cli/start.rs +++ /dev/null @@ -1,57 +0,0 @@ -use clap::ArgMatches; -use iron::{Chain, Iron}; -use logger::Logger; -use persistent::Write; -use router::Router; - -use bayard_client::index::client::IndexClient; - -use crate::handler::bulk_delete::bulk_delete; -use crate::handler::bulk_set::bulk_set; -use crate::handler::commit::commit; -use crate::handler::delete::delete; -use crate::handler::get::get; -use crate::handler::merge::merge; -use crate::handler::metrics::metrics; -use crate::handler::rollback::rollback; -use crate::handler::schema::schema; -use crate::handler::search::search; -use crate::handler::set::set; -use crate::handler::status::status; -use crate::handler::Client; -use crate::log::set_http_logger; - -pub fn run_start_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_http_logger(); - - let host = matches.value_of("HOST").unwrap(); - let port = matches.value_of("PORT").unwrap().parse::().unwrap(); - let server = matches.value_of("SERVER").unwrap(); - - let (logger_before, logger_after) = Logger::new(None); - - let index_client = IndexClient::new(server); - - let mut router = Router::new(); - router.get("/v1/documents/:id", get, "get"); - router.put("/v1/documents/:id", set, "set"); - router.delete("/v1/documents/:id", delete, "delete"); - router.put("/v1/documents", bulk_set, "bulk_set"); - router.delete("/v1/documents", bulk_delete, "bulk_delete"); - router.get("/v1/commit", commit, "commit"); - router.get("/v1/rollback", rollback, "rollback"); - router.get("/v1/merge", merge, "merge"); - router.get("/v1/schema", schema, "schema"); - router.post("/v1/search", search, "search"); - router.get("/v1/status", status, "status"); - router.get("/v1/metrics", metrics, "metrics"); - - let mut chain = Chain::new(router); - chain.link_before(logger_before); - chain.link(Write::::both(index_client)); - chain.link_after(logger_after); - - Iron::new(chain).http(format!("{}:{}", host, port)).unwrap(); - - Ok(()) -} diff --git a/bayard-rest/src/handler/metrics.rs b/bayard-rest/src/handler/metrics.rs deleted file mode 100644 index 84972c8..0000000 --- a/bayard-rest/src/handler/metrics.rs +++ /dev/null @@ -1,18 +0,0 @@ -use iron::headers::ContentType; -use iron::prelude::*; -use iron::{status, IronError, IronResult, Request, Response}; -use persistent::Write; - -use crate::handler::Client; - -pub fn metrics(req: &mut Request) -> IronResult { - let client = req.get::>().unwrap(); - let mut index_client = client.lock().unwrap(); - match index_client.metrics() { - Ok(s) => Ok(Response::with((ContentType::plaintext().0, status::Ok, s))), - Err(e) => Err(IronError::new( - e, - (status::InternalServerError, "failed to get metrics"), - )), - } -} diff --git a/bayard-rest/src/handler/mod.rs b/bayard-rest/src/handler/mod.rs index 3a7e918..52d5b3d 100644 --- a/bayard-rest/src/handler/mod.rs +++ b/bayard-rest/src/handler/mod.rs @@ -1,13 +1,13 @@ -use bayard_client::index::client::IndexClient; use iron::typemap::Key; +use bayard_client::index::client::IndexClient; + pub mod bulk_delete; pub mod bulk_set; pub mod commit; pub mod delete; pub mod get; pub mod merge; -pub mod metrics; pub mod rollback; pub mod schema; pub mod search; diff --git a/bayard-rest/src/lib.rs b/bayard-rest/src/lib.rs index 9b3270c..062ae9d 100644 --- a/bayard-rest/src/lib.rs +++ b/bayard-rest/src/lib.rs @@ -1,6 +1 @@ -#[macro_use] -extern crate clap; - -pub mod cli; pub mod handler; -pub mod log; diff --git a/bayard-rest/src/log/mod.rs b/bayard-rest/src/log/mod.rs deleted file mode 100644 index 395f6ea..0000000 --- a/bayard-rest/src/log/mod.rs +++ /dev/null @@ -1,60 +0,0 @@ -use std::env; -use std::io::Write; - -use env_logger; -use env_logger::Target; - -pub fn set_logger() { - match env::var("RUST_LOG") { - Ok(val) => { - let log_level: &str = &val; - match log_level { - "error" => { /* noop */ } - "warn" => { /* noop */ } - "info" => { /* noop */ } - "debug" => { /* noop */ } - _ => env::set_var("RUST_LOG", "info"), - } - } - Err(_e) => env::set_var("RUST_LOG", "info"), - }; - env_logger::Builder::from_default_env() - .format(|buf, record| { - let ts = buf.timestamp(); - writeln!( - buf, - "[{} {} {} {}:{}] {}", - ts, - record.level(), - record.target(), - record.file().unwrap_or("unknown"), - record.line().unwrap_or(0), - record.args(), - ) - }) - .target(Target::Stderr) - .init(); -} - -pub fn set_http_logger() { - match env::var("RUST_LOG") { - Ok(val) => { - let log_level: &str = &val; - match log_level { - "error" => { /* noop */ } - "warn" => { /* noop */ } - "info" => { /* noop */ } - "debug" => { /* noop */ } - _ => env::set_var("RUST_LOG", "info"), - } - } - Err(_e) => env::set_var("RUST_LOG", "info"), - }; - env_logger::Builder::from_default_env() - .format(|buf, record| { - let ts = buf.timestamp(); - writeln!(buf, "[{} {}] {}", ts, record.level(), record.args(),) - }) - .target(Target::Stderr) - .init(); -} diff --git a/bayard-rest/src/main.rs b/bayard-rest/src/main.rs index ab37d82..e31ad49 100644 --- a/bayard-rest/src/main.rs +++ b/bayard-rest/src/main.rs @@ -1,5 +1,118 @@ -use bayard_rest::cli::root::run_root_cli; +#[macro_use] +extern crate clap; + +use clap::{App, AppSettings, Arg}; +use crossbeam_channel::select; +use iron::{Chain, Iron}; +use log::*; +use logger::Logger; +use persistent::Write; +use router::Router; + +use bayard_client::index::client::IndexClient; +use bayard_common::log::set_logger; +use bayard_common::signal::sigterm_channel; +use bayard_rest::handler::bulk_delete::bulk_delete; +use bayard_rest::handler::bulk_set::bulk_set; +use bayard_rest::handler::commit::commit; +use bayard_rest::handler::delete::delete; +use bayard_rest::handler::get::get; +use bayard_rest::handler::merge::merge; +use bayard_rest::handler::rollback::rollback; +use bayard_rest::handler::schema::schema; +use bayard_rest::handler::search::search; +use bayard_rest::handler::set::set; +use bayard_rest::handler::status::status; +use bayard_rest::handler::Client; fn main() -> Result<(), std::io::Error> { - run_root_cli() + set_logger(); + + let app = App::new(crate_name!()) + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Bayard REST server") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("HOST") + .help("Node address.") + .short("H") + .long("host") + .value_name("HOST") + .default_value("0.0.0.0") + .takes_value(true), + ) + .arg( + Arg::with_name("PORT") + .help("HTTP service port number.") + .short("p") + .long("port") + .value_name("PORT") + .default_value("8000") + .takes_value(true), + ) + .arg( + Arg::with_name("SERVER") + .help("Index service address.") + .short("s") + .long("server") + .value_name("IP:PORT") + .default_value("0.0.0.0:5000") + .takes_value(true), + ); + + let matches = app.get_matches(); + + let host = matches.value_of("HOST").unwrap(); + let port = matches.value_of("PORT").unwrap().parse::().unwrap(); + let server = matches.value_of("SERVER").unwrap(); + + let index_client = IndexClient::new(server); + + let (logger_before, logger_after) = Logger::new(None); + let mut router = Router::new(); + router.get("/v1/documents/:id", get, "get"); + router.put("/v1/documents/:id", set, "set"); + router.delete("/v1/documents/:id", delete, "delete"); + router.put("/v1/documents", bulk_set, "bulk_set"); + router.delete("/v1/documents", bulk_delete, "bulk_delete"); + router.get("/v1/commit", commit, "commit"); + router.get("/v1/rollback", rollback, "rollback"); + router.get("/v1/merge", merge, "merge"); + router.get("/v1/schema", schema, "schema"); + router.post("/v1/search", search, "search"); + router.get("/v1/status", status, "status"); + + let mut chain = Chain::new(router); + chain.link_before(logger_before); + chain.link(Write::::both(index_client)); + chain.link_after(logger_after); + + let mut rest_server = Iron::new(chain).http(format!("{}:{}", host, port)).unwrap(); + info!("start rest service on {}:{}", host, port); + + // Wait for signals for termination (SIGINT, SIGTERM). + let sigterm_receiver = sigterm_channel().unwrap(); + loop { + select! { + recv(sigterm_receiver) -> _ => { + info!("receive signal"); + break; + } + } + } + + match rest_server.close() { + Ok(_) => { + info!("stop metrics service on {}:{}", host, port); + } + Err(e) => error!("{}", e), + } + + info!("exit"); + + Ok(()) } diff --git a/bayard-server/Cargo.toml b/bayard-server/Cargo.toml index b6304c5..9d84265 100644 --- a/bayard-server/Cargo.toml +++ b/bayard-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bayard-server" -version = "0.7.3" +version = "0.8.0" authors = ["MinoruOsuka "] edition = "2018" description = "Bayard is a distributed search server." @@ -18,6 +18,7 @@ bincode = "1.2.1" cang-jie = "0.7.0" futures = "0.1.29" grpcio = { version = "0.4.7", features = [ "secure" ] } +iron = "0.6.1" jieba-rs = "0.4.10" lazy_static = "1.4.0" lindera-tantivy = "0.1.1" @@ -30,4 +31,4 @@ serde_json = "1.0.51" stringreader = "0.1.1" tantivy = "0.12.0" -bayard-proto = "0.7.4" +bayard-proto = "0.8.0" diff --git a/bayard-server/src/index/server.rs b/bayard-server/src/index/server.rs index de65d39..666302b 100644 --- a/bayard-server/src/index/server.rs +++ b/bayard-server/src/index/server.rs @@ -7,10 +7,17 @@ use std::time::Duration; use std::{fs, thread}; use async_std::task::block_on; +use bayard_proto::proto::commonpb::State; +use bayard_proto::proto::indexpb::{ + BulkDeleteReply, BulkDeleteReq, BulkSetReply, BulkSetReq, CommitReply, CommitReq, DeleteReply, + DeleteReq, GetReply, GetReq, MergeReply, MergeReq, RollbackReply, RollbackReq, SchemaReply, + SchemaReq, SearchReply, SearchReq, SetReply, SetReq, StatusReply, StatusReq, +}; +use bayard_proto::proto::indexpb_grpc::IndexService; use futures::Future; use grpcio::{RpcContext, UnarySink}; use log::*; -use prometheus::{CounterVec, Encoder, HistogramVec, TextEncoder}; +use prometheus::{CounterVec, HistogramVec}; use raft::storage::MemStorage; use serde::{Deserialize, Serialize}; use stringreader::StringReader; @@ -20,15 +27,6 @@ use tantivy::query::{QueryParser, TermQuery}; use tantivy::schema::{Field, FieldType, IndexRecordOption, Schema}; use tantivy::{Document, Index, IndexWriter, Term}; -use bayard_proto::proto::commonpb::State; -use bayard_proto::proto::indexpb::{ - BulkDeleteReply, BulkDeleteReq, BulkSetReply, BulkSetReq, CommitReply, CommitReq, DeleteReply, - DeleteReq, GetReply, GetReq, MergeReply, MergeReq, MetricsReply, MetricsReq, RollbackReply, - RollbackReq, SchemaReply, SchemaReq, SearchReply, SearchReq, SetReply, SetReq, StatusReply, - StatusReq, -}; -use bayard_proto::proto::indexpb_grpc::IndexService; - use crate::index::search_result::{ScoredNamedFieldDocument, SearchResult}; use crate::raft::config; use crate::raft::config::NodeAddress; @@ -83,7 +81,6 @@ pub enum Op { Rollback {}, Merge {}, Schema {}, - Metrics {}, Status {}, } @@ -811,66 +808,6 @@ impl IndexService for IndexServer { timer.observe_duration(); } - fn metrics(&mut self, ctx: RpcContext, req: MetricsReq, sink: UnarySink) { - let (s1, r1) = mpsc::channel(); - let id = self.id.clone(); - let sender = self.sender.clone(); - let op = Op::Metrics {}; - let seq = self.seq; - self.seq += 1; - - debug!("metrics: req={:?}", req); - REQUEST_COUNTER.with_label_values(&["metrics"]).inc(); - let timer = REQUEST_HISTOGRAM - .with_label_values(&["metrics"]) - .start_timer(); - - sender - .send(config::Msg::Propose { - seq, - op, - cb: Box::new(move |leader_id: i32, addresses: Vec| { - let metric_families = prometheus::gather(); - let mut buffer = Vec::::new(); - let encoder = TextEncoder::new(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - let metrics_text = String::from_utf8(buffer.clone()).unwrap(); - - let (state, metrics_text) = (State::OK, metrics_text); - - let mut reply = MetricsReply::new(); - reply.set_state(state); - if leader_id >= 0 { - // not a leader - reply.set_leader_id(leader_id as u64); - } else { - // a leader - reply.set_leader_id(id); - } - reply.set_metrics(metrics_text); - reply.set_address_map(addresses); - s1.send(reply).expect("cb channel closed"); - }), - }) - .unwrap(); - - let reply = match r1.recv_timeout(Duration::from_secs(2)) { - Ok(r) => r, - Err(_e) => { - let mut r = MetricsReply::new(); - r.set_state(State::IO_ERROR); - r - } - }; - - let f = sink - .success(reply.clone()) - .map_err(move |err| error!("Failed to reply metrics: {:?}", err)); - ctx.spawn(f); - - timer.observe_duration(); - } - fn status(&mut self, ctx: RpcContext, req: StatusReq, sink: UnarySink) { let (s1, r1) = mpsc::channel(); let id = self.id.clone(); @@ -1104,9 +1041,6 @@ fn apply_daemon(receiver: Receiver, index: Arc, index_writer: Arc { // noop } - Op::Metrics {} => { - // noop - } Op::Status {} => { // noop } diff --git a/bayard-server/src/lib.rs b/bayard-server/src/lib.rs index 1922d3f..5da2d7e 100644 --- a/bayard-server/src/lib.rs +++ b/bayard-server/src/lib.rs @@ -4,5 +4,6 @@ extern crate lazy_static; extern crate prometheus; pub mod index; +pub mod metric; pub mod raft; pub mod tokenizer; diff --git a/bayard-server/src/metric/handler.rs b/bayard-server/src/metric/handler.rs new file mode 100644 index 0000000..521506a --- /dev/null +++ b/bayard-server/src/metric/handler.rs @@ -0,0 +1,16 @@ +use iron::{status, IronResult, Request, Response}; +use prometheus::{Encoder, TextEncoder}; + +pub fn metrics(_req: &mut Request) -> IronResult { + let metric_families = prometheus::gather(); + let mut buffer = Vec::::new(); + let encoder = TextEncoder::new(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + let metrics_text = String::from_utf8(buffer.clone()).unwrap(); + + Ok(Response::with(( + encoder.format_type(), + status::Ok, + metrics_text, + ))) +} diff --git a/bayard-server/src/metric/mod.rs b/bayard-server/src/metric/mod.rs new file mode 100644 index 0000000..062ae9d --- /dev/null +++ b/bayard-server/src/metric/mod.rs @@ -0,0 +1 @@ +pub mod handler; diff --git a/bayard/Cargo.toml b/bayard/Cargo.toml index c827b34..5703484 100644 --- a/bayard/Cargo.toml +++ b/bayard/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bayard" -version = "0.7.3" +version = "0.8.0" authors = ["Minoru Osuka "] edition = "2018" description = "Bayard is a distributed search server." @@ -19,15 +19,18 @@ path = "src/main.rs" [dependencies] clap = "2.33.0" crossbeam-channel = "0.4.2" -ctrlc = { version = "3.1.4", features = ["termination"] } -env_logger = "0.7.1" futures = "0.1.29" -grpcio = { version = "0.4.7", features = [ "secure" ] } +grpcio = { version = "0.4.7", features = ["secure"] } +iron = "0.6.1" log = "0.4.8" +logger = "0.4.0" num_cpus = "1.13.0" raft = "0.4.3" +router = "0.6.0" serde_json = "1.0.51" -bayard-client = { version = "0.7.3", path = "../bayard-client" } -bayard-proto = "0.7.4" -bayard-server = { version = "0.7.3", path = "../bayard-server" } +bayard-proto = "0.8.0" + +bayard-client = { version = "0.8.0", path = "../bayard-client" } +bayard-common = { version = "0.8.0", path = "../bayard-common" } +bayard-server = { version = "0.8.0", path = "../bayard-server" } diff --git a/bayard/src/cli/metrics.rs b/bayard/src/cli/metrics.rs deleted file mode 100644 index 2e32ccc..0000000 --- a/bayard/src/cli/metrics.rs +++ /dev/null @@ -1,24 +0,0 @@ -use clap::ArgMatches; - -use bayard_client::index::client::IndexClient; - -use crate::log::set_logger; - -pub fn run_metrics_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - - let server = matches.value_of("SERVER").unwrap(); - - let mut index_client = IndexClient::new(server); - - match index_client.metrics() { - Ok(v) => { - println!("{}", v); - Ok(()) - } - Err(e) => { - println!("{}", e); - Err(e) - } - } -} diff --git a/bayard/src/cli/start.rs b/bayard/src/cli/start.rs deleted file mode 100644 index efd2905..0000000 --- a/bayard/src/cli/start.rs +++ /dev/null @@ -1,144 +0,0 @@ -use std::collections::HashMap; -use std::path::Path; -use std::sync::Arc; - -use bayard_proto::proto::{indexpb_grpc, raftpb_grpc}; -use clap::ArgMatches; -use crossbeam_channel::select; -use futures::Future; -use grpcio::{Environment, ServerBuilder}; -use log::*; -use raft::storage::MemStorage; - -use bayard_client::raft::client::RaftClient; -use bayard_server::index::server::IndexServer; -use bayard_server::raft::config::NodeAddress; - -use crate::log::set_logger; -use crate::signal::sigterm_channel; - -pub fn run_start_cli(matches: &ArgMatches) -> Result<(), std::io::Error> { - set_logger(); - - let id = matches.value_of("ID").unwrap().parse::().unwrap(); - let host = matches.value_of("HOST").unwrap(); - let raft_port = matches - .value_of("RAFT_PORT") - .unwrap() - .parse::() - .unwrap(); - let index_port = matches - .value_of("INDEX_PORT") - .unwrap() - .parse::() - .unwrap(); - let mut peer_address = ""; - if let Some(_peer_address) = matches.value_of("PEER_RAFT_ADDRESS") { - peer_address = _peer_address; - } - let data_directory = matches.value_of("DATA_DIRECTORY").unwrap(); - let schema_file = matches.value_of("SCHEMA_FILE").unwrap(); - let tokenizer_file = matches.value_of("TOKENIZER_FILE").unwrap(); - let indexer_threads = matches - .value_of("INDEXER_THREADS") - .unwrap() - .parse::() - .unwrap(); - let indexer_memory_size = matches - .value_of("INDEXER_MEMORY_SIZE") - .unwrap() - .parse::() - .unwrap(); - - let raft_address = format!("{}:{}", host, raft_port); - let index_address = format!("{}:{}", host, index_port); - - let node_address = NodeAddress { - index_address, - raft_address, - }; - - let mut addresses = HashMap::new(); - - // change config - if peer_address != "" { - let mut client = RaftClient::new(peer_address); - match client.join(id, node_address.clone()) { - Ok(_addresses) => addresses = _addresses, - Err(e) => return Err(e), - }; - } - - let env_index = Arc::new(Environment::new(10)); - let env_raft = Arc::new(Environment::new(10)); - - let index_path = Path::new(data_directory) - .join("index") - .to_str() - .unwrap() - .to_string(); - let raft_storage = MemStorage::new(); - - let (index, raft) = IndexServer::new( - index_path, - schema_file, - tokenizer_file, - indexer_threads, - indexer_memory_size, - raft_storage, - id, - node_address, - addresses, - ); - - let index_service = indexpb_grpc::create_index_service(index); - let raft_service = raftpb_grpc::create_raft_service(raft); - - let mut index_server = ServerBuilder::new(env_index) - .register_service(index_service) - .bind(host, index_port) - .build() - .unwrap(); - let mut raft_server = ServerBuilder::new(env_raft) - .register_service(raft_service) - .bind(host, raft_port) - .build() - .unwrap(); - - index_server.start(); - raft_server.start(); - - for &(ref h, p) in index_server.bind_addrs() { - info!("start index service on {}:{}", h, p); - } - - for &(ref h, p) in raft_server.bind_addrs() { - info!("start Raft service on {}:{}", h, p); - } - - // Wait for signals for termination (SIGINT, SIGTERM). - let sigterm_receiver = sigterm_channel().unwrap(); - loop { - select! { - recv(sigterm_receiver) -> _ => { - debug!("receive signal"); - break; - } - } - } - - match index_server.shutdown().wait() { - Ok(_) => { - info!("stop index service on {}:{}", host, index_port); - } - Err(e) => error!("{}", e), - } - match raft_server.shutdown().wait() { - Ok(_) => { - info!("stop Raft service on {}:{}", host, raft_port); - } - Err(e) => error!("{}", e), - } - - Ok(()) -} diff --git a/bayard/src/main.rs b/bayard/src/main.rs index b877ef1..ff762d7 100644 --- a/bayard/src/main.rs +++ b/bayard/src/main.rs @@ -1,5 +1,290 @@ -use bayard::cli::root::run_root_cli; +#[macro_use] +extern crate clap; + +use std::collections::HashMap; +use std::path::Path; +use std::sync::Arc; + +use bayard_proto::proto::{indexpb_grpc, raftpb_grpc}; +use clap::{App, AppSettings, Arg}; +use crossbeam_channel::select; +use futures::Future; +use grpcio::{Environment, ServerBuilder}; +use iron::{Chain, Iron}; +use log::*; +use logger::Logger; +use raft::storage::MemStorage; +use router::Router; + +use bayard_client::raft::client::RaftClient; +use bayard_common::log::set_logger; +use bayard_common::signal::sigterm_channel; +use bayard_server::index::server::IndexServer; +use bayard_server::metric::handler::metrics; +use bayard_server::raft::config::NodeAddress; fn main() -> Result<(), std::io::Error> { - run_root_cli() + set_logger(); + + let cpus = num_cpus::get().to_owned(); + let threads; + if cpus > 1 { + threads = format!("{}", cpus - 1); + } else { + threads = format!("{}", cpus); + } + + let app = App::new(crate_name!()) + .setting(AppSettings::DeriveDisplayOrder) + .version(crate_version!()) + .author(crate_authors!()) + .about("Bayard server") + .help_message("Prints help information.") + .version_message("Prints version information.") + .version_short("v") + .arg( + Arg::with_name("ID") + .help("Node ID.") + .value_name("ID") + .takes_value(true), + ) + .arg( + Arg::with_name("HOST") + .help("Node address.") + .short("H") + .long("host") + .value_name("HOST") + .default_value("0.0.0.0") + .takes_value(true), + ) + .arg( + Arg::with_name("RAFT_PORT") + .help("Raft service port number.") + .short("r") + .long("raft-port") + .value_name("RAFT_PORT") + .default_value("7000") + .takes_value(true), + ) + .arg( + Arg::with_name("INDEX_PORT") + .help("Index service port number") + .short("i") + .long("index-port") + .value_name("INDEX_PORT") + .default_value("5000") + .takes_value(true), + ) + .arg( + Arg::with_name("METRICS_PORT") + .help("Metrics service port number") + .short("M") + .long("metrics-port") + .value_name("METRICS_PORT") + .default_value("9000") + .takes_value(true), + ) + .arg( + Arg::with_name("PEER_RAFT_ADDRESS") + .help("Raft address of a peer node running in an existing cluster.") + .short("p") + .long("peer-raft-address") + .value_name("IP:PORT") + .takes_value(true), + ) + .arg( + Arg::with_name("DATA_DIRECTORY") + .help("Data directory. Stores index, snapshots, and raft logs.") + .short("d") + .long("data-directory") + .value_name("DATA_DIRECTORY") + .default_value("./data") + .takes_value(true), + ) + .arg( + Arg::with_name("SCHEMA_FILE") + .help("Schema file. Must specify An existing file name.") + .short("s") + .long("schema-file") + .value_name("SCHEMA_FILE") + .default_value("./etc/schema.json") + .takes_value(true), + ) + .arg( + Arg::with_name("TOKENIZER_FILE") + .help("Tokenizer file. Must specify An existing file name.") + .short("T") + .long("tokenizer-file") + .value_name("TOKENIZER_FILE") + .default_value("./etc/tokenizer.json") + .takes_value(true), + ) + .arg( + Arg::with_name("INDEXER_THREADS") + .help("Number of indexer threads.") + .short("t") + .long("indexer-threads") + .value_name("INDEXER_THREADS") + .default_value(&threads) + .takes_value(true), + ) + .arg( + Arg::with_name("INDEXER_MEMORY_SIZE") + .help("Total memory size (in bytes) used by the indexer.") + .short("m") + .long("indexer-memory-size") + .value_name("INDEXER_MEMORY_SIZE") + .default_value("1000000000") + .takes_value(true), + ); + + let matches = app.get_matches(); + + let id = matches.value_of("ID").unwrap().parse::().unwrap(); + let host = matches.value_of("HOST").unwrap(); + let raft_port = matches + .value_of("RAFT_PORT") + .unwrap() + .parse::() + .unwrap(); + let index_port = matches + .value_of("INDEX_PORT") + .unwrap() + .parse::() + .unwrap(); + let metrics_port = matches + .value_of("METRICS_PORT") + .unwrap() + .parse::() + .unwrap(); + let mut peer_address = ""; + if let Some(_peer_address) = matches.value_of("PEER_RAFT_ADDRESS") { + peer_address = _peer_address; + } + let data_directory = matches.value_of("DATA_DIRECTORY").unwrap(); + let schema_file = matches.value_of("SCHEMA_FILE").unwrap(); + let tokenizer_file = matches.value_of("TOKENIZER_FILE").unwrap(); + let indexer_threads = matches + .value_of("INDEXER_THREADS") + .unwrap() + .parse::() + .unwrap(); + let indexer_memory_size = matches + .value_of("INDEXER_MEMORY_SIZE") + .unwrap() + .parse::() + .unwrap(); + + let raft_address = format!("{}:{}", host, raft_port); + let index_address = format!("{}:{}", host, index_port); + let metrics_address = format!("{}:{}", host, metrics_port); + + let node_address = NodeAddress { + index_address, + raft_address, + }; + + let mut addresses = HashMap::new(); + + // change config + if peer_address != "" { + let mut client = RaftClient::new(peer_address); + match client.join(id, node_address.clone()) { + Ok(_addresses) => addresses = _addresses, + // Err(e) => return Err(Box::try_from(e).unwrap()), + Err(e) => return Err(e), + }; + } + + let env_index = Arc::new(Environment::new(10)); + let env_raft = Arc::new(Environment::new(10)); + + let index_path = Path::new(data_directory) + .join("index") + .to_str() + .unwrap() + .to_string(); + let raft_storage = MemStorage::new(); + + let (index, raft) = IndexServer::new( + index_path, + schema_file, + tokenizer_file, + indexer_threads, + indexer_memory_size, + raft_storage, + id, + node_address, + addresses, + ); + + let index_service = indexpb_grpc::create_index_service(index); + let raft_service = raftpb_grpc::create_raft_service(raft); + + let mut index_server = ServerBuilder::new(env_index) + .register_service(index_service) + .bind(host, index_port) + .build() + .unwrap(); + let mut raft_server = ServerBuilder::new(env_raft) + .register_service(raft_service) + .bind(host, raft_port) + .build() + .unwrap(); + + index_server.start(); + raft_server.start(); + + for &(ref h, p) in index_server.bind_addrs() { + info!("start index service on {}:{}", h, p); + } + + for &(ref h, p) in raft_server.bind_addrs() { + info!("start raft service on {}:{}", h, p); + } + + // metrics service + let (logger_before, logger_after) = Logger::new(None); + let mut router = Router::new(); + router.get("/metrics", metrics, "metrics"); + + let mut chain = Chain::new(router); + chain.link_before(logger_before); + chain.link_after(logger_after); + + let mut metrics_server = Iron::new(chain).http(metrics_address).unwrap(); + info!("start metrics service on {}:{}", host, metrics_port); + + // Wait for signals for termination (SIGINT, SIGTERM). + let sigterm_receiver = sigterm_channel().unwrap(); + loop { + select! { + recv(sigterm_receiver) -> _ => { + info!("receive signal"); + break; + } + } + } + + match metrics_server.close() { + Ok(_) => { + info!("stop metrics service on {}:{}", host, metrics_port); + } + Err(e) => error!("{}", e), + } + + match index_server.shutdown().wait() { + Ok(_) => { + info!("stop index service on {}:{}", host, index_port); + } + Err(e) => error!("{}", e), + } + match raft_server.shutdown().wait() { + Ok(_) => { + info!("stop raft service on {}:{}", host, raft_port); + } + Err(e) => error!("{}", e), + } + + return Ok(()); } diff --git a/docs/accessing_over_http.html b/docs/accessing_over_http.html index c3de89c..448fd6a 100644 --- a/docs/accessing_over_http.html +++ b/docs/accessing_over_http.html @@ -81,7 +81,7 @@ @@ -152,7 +152,7 @@

Using Gateway

Starting a REST server is easy.

-
$ ./bin/bayard-rest start --port=8000 --server=0.0.0.0:5001
+
$ ./bin/bayard-rest --port=8000 --server=0.0.0.0:5001
 

 
REST API

 
See following documents:

diff --git a/docs/building_bayard.html b/docs/building_bayard.html
index 0454798..f37d6f8 100644
--- a/docs/building_bayard.html
+++ b/docs/building_bayard.html
@@ -81,7 +81,7 @@
 
         
@@ -156,14 +156,6 @@ 
Requirementsmake >= 3.81
 
  • protoc >= 3.9.2
    • 
 
-
    Install protoc-gen-rust
    
-
    $ cargo install protobuf-codegen
-$ cargo install grpcio-compiler
-
    
-
    Install protoc-gen-grpc-web
    
-
    $ curl -o /usr/local/bin/protoc-gen-grpc-web -L https://github.com/grpc/grpc-web/releases/download/1.0.7/protoc-gen-grpc-web-1.0.7-darwin-x86_64
-$ chmod +x /usr/local/bin/protoc-gen-grpc-web
-
    
 
    Build
    
 
    Build Bayard with the following command:
    
 
    $ make build
diff --git a/docs/cluster_mode.html b/docs/cluster_mode.html
index b73715d..0a8eabd 100644
--- a/docs/cluster_mode.html
+++ b/docs/cluster_mode.html
@@ -81,7 +81,7 @@
 
         
@@ -153,42 +153,41 @@ 
    Cluster modeStarting in cluster mode (3-node cluster)
    
 
    Running in standalone is not fault tolerant. If you need to improve fault tolerance, start servers in cluster mode.
 You can start servers in cluster mode with the following command:
    
-
    $ ./bin/bayard start \
-               --host=0.0.0.0 \
-               --raft-port=7001 \
-               --index-port=5001 \
-               --data-directory=./data/node1 \
-               --schema-file=./etc/schema.json \
-               --tokenizer-file=./etc/tokenizer.json \
-               1
+
    $ bayard --host=0.0.0.0 \
+         --raft-port=7001 \
+         --index-port=5001 \
+         --metrics-port=9001 \
+         --data-directory=./data/node1 \
+         --schema-file=./etc/schema.json \
+         --tokenizer-file=./etc/tokenizer.json \
+         1
 
    
-
    $ ./bin/bayard start \
-               --host=0.0.0.0 \
-               --raft-port=7002 \
-               --index-port=5002 \
-               --peer-raft-address=0.0.0.0:7001 \
-               --data-directory=./data/node2 \
-               --schema-file=./etc/schema.json \
-               --tokenizer-file=./etc/tokenizer.json \
-               2
+
    $ bayard --host=0.0.0.0 \
+         --raft-port=7002 \
+         --index-port=5002 \
+         --metrics-port=9002 \
+         --peer-raft-address=0.0.0.0:7001 \
+         --data-directory=./data/node2 \
+         --schema-file=./etc/schema.json \
+         --tokenizer-file=./etc/tokenizer.json \
+         2
 
    
-
    $ ./bin/bayard start \
-               --host=0.0.0.0 \
-               --raft-port=7003 \
-               --index-port=5003 \
-               --peer-raft-address=0.0.0.0:7001 \
-               --data-directory=./data/node3 \
-               --schema-file=./etc/schema.json \
-               --tokenizer-file=./etc/tokenizer.json \
-               3
+
    $ bayard --host=0.0.0.0 \
+         --raft-port=7003 \
+         --index-port=5003 \
+         --metrics-port=9003 \
+         --peer-raft-address=0.0.0.0:7001 \
+         --data-directory=./data/node3 \
+         --schema-file=./etc/schema.json \
+         --tokenizer-file=./etc/tokenizer.json \
+         3
 
    
 
    The above commands run servers on the same host, so each server must listen on a different port. This would not be necessary if each server runs on a different host.
 Recommend 3 or more odd number of servers in the cluster to avoid split-brain.
    
 When deploying to a single host, if that host goes down due to hardware failure, all of the servers in the cluster will be stopped, so recommend deploying to a different host.
    
 
    Cluster peers
    
 
    You can check the peers in the cluster with the following command:
    
-
    $ ./bin/bayard status
-               --server=0.0.0.0:5001 | jq .
+
    $ bayard-cli status --server=0.0.0.0:5001 | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    {
@@ -223,9 +222,7 @@ 
    $ ./bin/bayard leave \
-               --servers=0.0.0.0:5001 \
-               3
+
    $ bayard-cli leave --server=0.0.0.0:5001 3
 
    
 
                     
diff --git a/docs/configure_text_analyzers.html b/docs/configure_text_analyzers.html
index 65cac71..57a5d06 100644
--- a/docs/configure_text_analyzers.html
+++ b/docs/configure_text_analyzers.html
@@ -81,7 +81,7 @@
 
             
diff --git a/docs/designing_schema.html b/docs/designing_schema.html
index 4ce07a7..13e3882 100644
--- a/docs/designing_schema.html
+++ b/docs/designing_schema.html
@@ -81,7 +81,7 @@
 
         
diff --git a/docs/getting_started.html b/docs/getting_started.html
index 5e60f02..34b3688 100644
--- a/docs/getting_started.html
+++ b/docs/getting_started.html
@@ -81,7 +81,7 @@
 
         
@@ -151,11 +151,11 @@ 
    Bayard
    
                         
    Getting started
    
 
    Starting in standalone mode (Single node cluster)
    
 
    Running node in standalone mode is easy. You can start server with the following command:
    
-
    $ ./bin/bayard start 1
+
    $ ./bin/bayard 1
 
    
 
    Getting schema
    
 
    You can confirm current schema with the following command:
    
-
    $ ./bin/bayard schema | jq .
+
    $ ./bin/bayard-cli schema | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    [
@@ -229,12 +229,12 @@ 
    Getting schema<
 
    
 
    Indexing document
    
 
    You can index document with the following command:
    
-
    $ cat ./examples/doc_1.json | xargs -0 ./bin/bayard set 1
-$ ./bin/bayard commit
+
    $ cat ./examples/doc_1.json | xargs -0 ./bin/bayard-cli set 1
+$ ./bin/bayard-cli commit
 
    
 
    Getting document
    
 
    You can get document with the following command:
    
-
    $ ./bin/bayard get 1 | jq .
+
    $ ./bin/bayard-cli get 1 | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    {
@@ -264,12 +264,12 @@ 
    Getting doc
 
    
 
    Indexing documents in bulk
    
 
    You can index documents in bulk with the following command:
    
-
    $ cat ./examples/bulk_put.jsonl | xargs -0 ./bin/bayard bulk-set
-$ ./bin/bayard commit
+
    $ cat ./examples/bulk_put.jsonl | xargs -0 ./bin/bayard-cli bulk-set
+$ ./bin/bayard-cli commit
 
    
 
    Searching documents
    
 
    You can search documents with the following command:
    
-
    $ ./bin/bayard search --facet-field=category --facet-prefix=/category/search --facet-prefix=/language description:rust | jq .
+
    $ ./bin/bayard-cli search --facet-field=category --facet-prefix=/category/search --facet-prefix=/language description:rust | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    {
@@ -341,13 +341,13 @@ 
    Searc
 
    
 
    Deleting document
    
 
    You can delete document with the following command:
    
-
    $ ./bin/bayard delete 1
-$ ./bin/bayard commit
+
    $ ./bin/bayard-cli delete 1
+$ ./bin/bayard-cli commit
 
    
 
    Deleting documents in bulk
    
 
    You can delete documents in bulk with the following command:
    
-
    $ cat ./examples/bulk_delete.jsonl | xargs -0 ./bin/bayard bulk-delete
-$ ./bin/bayard commit
+
    $ cat ./examples/bulk_delete.jsonl | xargs -0 ./bin/bayard-cli bulk-delete
+$ ./bin/bayard-cli commit
 
    
 
                     
diff --git a/docs/index.html b/docs/index.html
index d6efeb8..7713be0 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -81,7 +81,7 @@
 
         
diff --git a/docs/install.html b/docs/install.html
index ffb66cd..50753ec 100644
--- a/docs/install.html
+++ b/docs/install.html
@@ -81,7 +81,7 @@
 
         
@@ -157,6 +157,8 @@ 
    RequirementsInstall
    
 
    Install Bayard with the following command:
    
 
    $ cargo install bayard
+$ cargo install bayard-cli
+$ cargo install bayard-rest
 
    
 
                     
diff --git a/docs/overview.html b/docs/overview.html
index 2fe0666..77d20c7 100644
--- a/docs/overview.html
+++ b/docs/overview.html
@@ -81,7 +81,7 @@
 
         
diff --git a/docs/print.html b/docs/print.html
index 26f01b1..7b0025d 100644
--- a/docs/print.html
+++ b/docs/print.html
@@ -83,7 +83,7 @@
 
         
@@ -184,6 +184,8 @@ 
    RequirementsInstall
    
 
    Install Bayard with the following command:
    
 
    $ cargo install bayard
+$ cargo install bayard-cli
+$ cargo install bayard-rest
 
    
 
    Building Bayard
    
 
    Requirements
    
@@ -193,14 +195,6 @@ 
    Requirementsmake >= 3.81
 
  • protoc >= 3.9.2
    • 
 
-
    Install protoc-gen-rust
    
-
    $ cargo install protobuf-codegen
-$ cargo install grpcio-compiler
-
    
-
    Install protoc-gen-grpc-web
    
-
    $ curl -o /usr/local/bin/protoc-gen-grpc-web -L https://github.com/grpc/grpc-web/releases/download/1.0.7/protoc-gen-grpc-web-1.0.7-darwin-x86_64
-$ chmod +x /usr/local/bin/protoc-gen-grpc-web
-
    
 
    Build
    
 
    Build Bayard with the following command:
    
 
    $ make build
@@ -211,11 +205,11 @@ 
    Build
    
 
    Getting started
    
 
    Starting in standalone mode (Single node cluster)
    
 
    Running node in standalone mode is easy. You can start server with the following command:
    
-
    $ ./bin/bayard start 1
+
    $ ./bin/bayard 1
 
    
 
    Getting schema
    
 
    You can confirm current schema with the following command:
    
-
    $ ./bin/bayard schema | jq .
+
    $ ./bin/bayard-cli schema | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    [
@@ -289,12 +283,12 @@ 
    Getting schema<
 
    
 
    Indexing document
    
 
    You can index document with the following command:
    
-
    $ cat ./examples/doc_1.json | xargs -0 ./bin/bayard set 1
-$ ./bin/bayard commit
+
    $ cat ./examples/doc_1.json | xargs -0 ./bin/bayard-cli set 1
+$ ./bin/bayard-cli commit
 
    
 
    Getting document
    
 
    You can get document with the following command:
    
-
    $ ./bin/bayard get 1 | jq .
+
    $ ./bin/bayard-cli get 1 | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    {
@@ -324,12 +318,12 @@ 
    Getting doc
 
    
 
    Indexing documents in bulk
    
 
    You can index documents in bulk with the following command:
    
-
    $ cat ./examples/bulk_put.jsonl | xargs -0 ./bin/bayard bulk-set
-$ ./bin/bayard commit
+
    $ cat ./examples/bulk_put.jsonl | xargs -0 ./bin/bayard-cli bulk-set
+$ ./bin/bayard-cli commit
 
    
 
    Searching documents
    
 
    You can search documents with the following command:
    
-
    $ ./bin/bayard search --facet-field=category --facet-prefix=/category/search --facet-prefix=/language description:rust | jq .
+
    $ ./bin/bayard-cli search --facet-field=category --facet-prefix=/category/search --facet-prefix=/language description:rust | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    {
@@ -401,13 +395,13 @@ 
    Searc
 
    
 
    Deleting document
    
 
    You can delete document with the following command:
    
-
    $ ./bin/bayard delete 1
-$ ./bin/bayard commit
+
    $ ./bin/bayard-cli delete 1
+$ ./bin/bayard-cli commit
 
    
 
    Deleting documents in bulk
    
 
    You can delete documents in bulk with the following command:
    
-
    $ cat ./examples/bulk_delete.jsonl | xargs -0 ./bin/bayard bulk-delete
-$ ./bin/bayard commit
+
    $ cat ./examples/bulk_delete.jsonl | xargs -0 ./bin/bayard-cli bulk-delete
+$ ./bin/bayard-cli commit
 
    
 
    Designing schema
    
 
    Schema
    
@@ -884,42 +878,41 @@ 
    Cluster modeStarting in cluster mode (3-node cluster)
    
 
    Running in standalone is not fault tolerant. If you need to improve fault tolerance, start servers in cluster mode.
 You can start servers in cluster mode with the following command:
    
-
    $ ./bin/bayard start \
-               --host=0.0.0.0 \
-               --raft-port=7001 \
-               --index-port=5001 \
-               --data-directory=./data/node1 \
-               --schema-file=./etc/schema.json \
-               --tokenizer-file=./etc/tokenizer.json \
-               1
-
    
-
    $ ./bin/bayard start \
-               --host=0.0.0.0 \
-               --raft-port=7002 \
-               --index-port=5002 \
-               --peer-raft-address=0.0.0.0:7001 \
-               --data-directory=./data/node2 \
-               --schema-file=./etc/schema.json \
-               --tokenizer-file=./etc/tokenizer.json \
-               2
-
    
-
    $ ./bin/bayard start \
-               --host=0.0.0.0 \
-               --raft-port=7003 \
-               --index-port=5003 \
-               --peer-raft-address=0.0.0.0:7001 \
-               --data-directory=./data/node3 \
-               --schema-file=./etc/schema.json \
-               --tokenizer-file=./etc/tokenizer.json \
-               3
+
    $ bayard --host=0.0.0.0 \
+         --raft-port=7001 \
+         --index-port=5001 \
+         --metrics-port=9001 \
+         --data-directory=./data/node1 \
+         --schema-file=./etc/schema.json \
+         --tokenizer-file=./etc/tokenizer.json \
+         1
+
    
+
    $ bayard --host=0.0.0.0 \
+         --raft-port=7002 \
+         --index-port=5002 \
+         --metrics-port=9002 \
+         --peer-raft-address=0.0.0.0:7001 \
+         --data-directory=./data/node2 \
+         --schema-file=./etc/schema.json \
+         --tokenizer-file=./etc/tokenizer.json \
+         2
+
    
+
    $ bayard --host=0.0.0.0 \
+         --raft-port=7003 \
+         --index-port=5003 \
+         --metrics-port=9003 \
+         --peer-raft-address=0.0.0.0:7001 \
+         --data-directory=./data/node3 \
+         --schema-file=./etc/schema.json \
+         --tokenizer-file=./etc/tokenizer.json \
+         3
 
    
 
    The above commands run servers on the same host, so each server must listen on a different port. This would not be necessary if each server runs on a different host.
 Recommend 3 or more odd number of servers in the cluster to avoid split-brain.
    
 When deploying to a single host, if that host goes down due to hardware failure, all of the servers in the cluster will be stopped, so recommend deploying to a different host.
    
 
    Cluster peers
    
 
    You can check the peers in the cluster with the following command:
    
-
    $ ./bin/bayard status
-               --server=0.0.0.0:5001 | jq .
+
    $ bayard-cli status --server=0.0.0.0:5001 | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    {
@@ -954,15 +947,13 @@ 
    $ ./bin/bayard leave \
-               --servers=0.0.0.0:5001 \
-               3
+
    $ bayard-cli leave --server=0.0.0.0:5001 3
 
    
 
    Accessing over the HTTP
    
 
    Bayard supports gRPC connections, but some users may want to use the traditional RESTful API over HTTP. Bayard REST server is useful in such cases.
    
 
    Using Gateway
    
 
    Starting a REST server is easy.
    
-
    $ ./bin/bayard-rest start --port=8000 --server=0.0.0.0:5001
+
    $ ./bin/bayard-rest --port=8000 --server=0.0.0.0:5001
 
    
 
    REST API
    
 
    See following documents:
    
@@ -995,18 +986,22 @@ 
    
 
  • 
 
    bayard
    
-     Manage Bayard index server
    
+     Bayard server
    
+
    • 
+
  • 
+
    bayard-cli
    
+     Bayard command-line interface
    
 
    • 
 
  • 
 
    bayard-rest
    
-     Manage Bayard REST server
    
+     Bayard REST server
    
 
    • 
 
 
    bayard
    
 
    DESCRIPTION
    
-
    Manage Bayard index server
    
+
    Bayard server
    
 
    USAGE
    
-
    bayard <SUBCOMMAND>
    
+
    bayard [OPTIONS] [ID]
    
 
    FLAGS
    
 
      
 
    • 
@@ -1018,74 +1013,73 @@ 
      FLAGS
      
      Prints version information.
      
 
      • 
 
    
-
    SUBCOMMANDS
    
+
    OPTIONS
    
 
      
 
    • 
-
      start
      
-     Start index server
      
-
      • 
-
    • 
-
      leave
      
-     Delete document from index server
      
-
      • 
-
    • 
-
      get
      
-     Get document from index server
      
-
      • 
-
    • 
-
      set
      
-     Set document to index server
      
-
      • 
-
    • 
-
      delete
      
-     Delete document from index server
      
-
      • 
-
    • 
-
      bulk-set
      
-     Set documents to index server in bulk
      
+
      -H, --host <HOST>
      
+     Node address. [default: 0.0.0.0]
      
 
      • 
 
    • 
-
      bulk-delete
      
-     Delete documents from index server in bulk
      
+
      -r, --raft-port <RAFT_PORT>
      
+     Raft service port number. [default: 7000]
      
 
      • 
 
    • 
-
      commit
      
-     Commit index
      
+
      -i, --index-port <INDEX_PORT>
      
+     Index service port number [default: 5000]
      
 
      • 
 
    • 
-
      rollback
      
-     Rollback index
      
+
      -M, --metrics-port <METRICS_PORT>
      
+     Metrics service port number [default: 9000]
      
 
      • 
 
    • 
-
      merge
      
-     Merge index
      
+
      -p, --peer-raft-address <IP:PORT>
      
+     Raft address of a peer node running in an existing cluster.
      
 
      • 
 
    • 
-
      schema
      
-     Shows index schema that applied
      
+
      -d, --data-directory <DATA_DIRECTORY>
      
+     Data directory. Stores index, snapshots, and raft logs. [default: ./data]
      
 
      • 
 
    • 
-
      search
      
-     Get document from index server
      
+
      -s, --schema-file <SCHEMA_FILE>
      
+     Schema file. Must specify An existing file name. [default: ./etc/schema.json]
      
 
      • 
 
    • 
-
      status
      
-     Shows system status
      
+
      -T, --tokenizer-file <TOKENIZER_FILE>
      
+     Tokenizer file. Must specify An existing file name. [default: ./etc/tokenizer.json]
      
 
      • 
 
    • 
-
      metrics
      
-     Shows system metrics
      
+
      -t, --indexer-threads <INDEXER_THREADS>
+     Number of indexer threads. [default: 7]
      
 
      • 
 
    • 
-
      help
      
-     Prints this message or the help of the given subcommand(s)
      
+
      -m, --indexer-memory-size <INDEXER_MEMORY_SIZE>
      
+     Total memory size (in bytes) used by the indexer. [default: 1000000000]
      
 
      • 
 
    
-
    bayard start
    
+
    ARGS
    
+
      
+
    • <ID>
      
+     Node ID.
      • 
+
    
+
    EXAMPLES
    
+
    To start a server with default options:
    
+
    $ bayard 1
+
    
+
    To start a server with options:
    
+
    $ bayard --host=0.0.0.0 \
+         --raft-port=7001 \
+         --index-port=5001 \
+         --metrics-port=9001 \
+         --data-directory=./data/node1 \
+         --schema-file=./etc/schema.json \
+         --tokenizer-file=./etc/tokenizer.json \
+         1
+
    
+
    bayard-cli
    
 
    DESCRIPTION
    
-
    Start index server.
    
+
    Bayard command-line interface
    
 
    USAGE
    
-
    bayard start [OPTIONS] [ID]
    
+
    bayard-cli <SUBCOMMAND>
    
 
    FLAGS
    
 
      
 
    • 
@@ -1097,69 +1091,70 @@ 
      FLAGS
      
      Prints version information.
      
 
      • 
 
    
-
    OPTIONS
    
+
    SUBCOMMANDS
    
 
      
 
    • 
-
      -H, --host <HOST>
      
-     Node address. [default: 0.0.0.0]
      
+
      leave
      
+     Delete document from index server
      
 
      • 
 
    • 
-
      -r, --raft-port <RAFT_PORT>
      
-     Raft service port number. [default: 7000]
      
+
      get
      
+     Get document from index server
      
 
      • 
 
    • 
-
      -i, --index-port <INDEX_PORT>
      
-     Index service port number [default: 5000]
      
+
      set
      
+     Set document to index server
      
 
      • 
 
    • 
-
      -p, --peer-raft-address <IP:PORT>
      
-     Raft address of a peer node running in an existing cluster.
      
+
      delete
      
+     Delete document from index server
      
 
      • 
 
    • 
-
      -d, --data-directory <DATA_DIRECTORY>
      
-     Data directory. Stores index, snapshots, and raft logs. [default: ./data]
      
+
      bulk-set
      
+     Set documents to index server in bulk
      
 
      • 
 
    • 
-
      -s, --schema-file <SCHEMA_FILE>
      
-     Schema file. Must specify An existing file name. [default: ./etc/schema.json]
      
+
      bulk-delete
      
+     Delete documents from index server in bulk
      
 
      • 
 
    • 
-
      -T, --tokenizer-file <TOKENIZER_FILE>
      
-     Tokenizer file. Must specify An existing file name. [default: ./etc/tokenizer.json]
      
+
      commit
      
+     Commit index
      
 
      • 
 
    • 
-
      -t, --indexer-threads <INDEXER_THREADS>
-     Number of indexer threads. [default: 7]
      
+
      rollback
      
+     Rollback index
      
 
      • 
 
    • 
-
      -m, --indexer-memory-size <INDEXER_MEMORY_SIZE>
      
-     Total memory size (in bytes) used by the indexer. [default: 1000000000]
      
+
      merge
      
+     Merge index
      
+
      • 
+
    • 
+
      schema
      
+     Shows index schema that applied
      
+
      • 
+
    • 
+
      search
      
+     Get document from index server
      
+
      • 
+
    • 
+
      status
      
+     Shows system status
      
+
      • 
+
    • 
+
      metrics
      
+     Shows system metrics
      
+
      • 
+
    • 
+
      help
      
+     Prints this message or the help of the given subcommand(s)
      
 
      • 
 
    
-
    ARGS
    
-
      
-
    • <ID>
      
-     Node ID.
      • 
-
    
-
    EXAMPLES
    
-
    To start a server with default options:
    
-
    $ ./bin/bayard start 1
-
    
-
    To start a server with options:
    
-
    $ ./bin/bayard start \
-               --host=0.0.0.0 \
-               --raft-port=7001 \
-               --index-port=5001 \
-               --data-directory=./data/node1 \
-               --schema-file=./etc/schema.json \
-               --tokenizer-file=./etc/tokenizer.json \
-               1
-
    
-
    bayard leave
    
+
    bayard-cli leave
    
 
    DESCRIPTION
    
 
    Delete node from the cluster
    
 
    USAGE
    
-
    bayard leave [OPTIONS] [ID]
    
+
    bayard-cli leave [OPTIONS] [ID]
    
 
    FLAGS
    
 
      
 
    • 
@@ -1183,13 +1178,13 @@ 
      ARGS
      
 
    
 
    EXAMPLES
    
 
    To probe a server with options:
    
-
    $ ./bin/bayard leave --servers=127.0.0.1:5001 3
+
    $ bayard-cli leave --server=127.0.0.1:5001 3
 
    
-
    bayard get
    
+
    bayard-cli get
    
 
    DESCRIPTION
    
 
    Get document from index server
    
 
    USAGE
    
-
    bayard get [OPTIONS] [ID]
    
+
    bayard-cli get [OPTIONS] [ID]
    
 
    FLAGS
    
 
      
 
    • 
@@ -1213,7 +1208,7 @@ 
      ARGS
      
 
    
 
    EXAMPLES
    
 
    To get a document with default options:
    
-
    $ ./bin/bayard get --server=192.168.11.10:5001 1
+
    $ bayard-cli get --server=192.168.11.10:5001 1
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    {
@@ -1241,11 +1236,11 @@ 
    EXAMPLES
    
   ]
 }
 
    
-
    bayard set
    
+
    bayard-cli set
    
 
    DESCRIPTION
    
 
    Set document to index server
    
 
    USAGE
    
-
    bayard bulk-set [OPTIONS] [DOCS]
    
+
    bayard-cli set [OPTIONS] [ARGS]
    
 
    FLAGS
    
 
 
    ARGS
    
 
      
-
    • <DOCS>
      
-     Document containing the unique ID to be indexed.
      • 
+
    • 
+
      <ID>
      
+     A unique ID that identifies the document in the index server.
      
+
      • 
+
    • 
+
      <FIELDS>
      
+     Fields of document to be indexed.
      
+
      • 
 
    
 
    EXAMPLES
    
 
    To put a document:
    
-
    $ cat ./examples/doc_1.json | xargs -0 ./bin/bayard set 1
+
    $ cat ./examples/doc_1.json | xargs -0 bayard-cli set 1
 
    
-
    bayard delete
    
+
    bayard-cli delete
    
 
    DESCRIPTION
    
 
    Delete document from index server
    
 
    USAGE
    
@@ -1299,13 +1300,13 @@ 
    ARGS
    
 
 
    EXAMPLES
    
 
    To delete a document:
    
-
    $ ./bin/bayard delete --server=0.0.0.0:5001 1
+
    $ bayard-cli delete --server=0.0.0.0:5001 1
 
    
-
    bayard bulk-set
    
+
    bayard-cli bulk-set
    
 
    DESCRIPTION
    
 
    Set documents to index server in bulk
    
 
    USAGE
    
-
    bayard set [OPTIONS] [ARGS]
    
+
    bayard-cli bulk-set [OPTIONS] [DOCS]
    
 
    FLAGS
    
 
 
    ARGS
    
 
      
-
    • 
-
      <ID>
      
-     A unique ID that identifies the document in the index server.
      
-
      • 
-
    • 
-
      <FIELDS>
      
-     Fields of document to be indexed.
      
-
      • 
+
    • <DOCS>
      
+     Document containing the unique ID to be indexed.
      • 
 
    
 
    EXAMPLES
    
 
    To put documents in bulk:
    
-
    $ cat ./examples/bulk_put.jsonl | xargs -0 ./bin/bayard bulk-set
+
    $ cat ./examples/bulk_put.jsonl | xargs -0 bayard-cli bulk-set
 
    
-
    bayard bulk-delete
    
+
    bayard-cli bulk-delete
    
 
    DESCRIPTION
    
 
    Delete documents from index server in bulk
    
 
    USAGE
    
-
    bayard set [OPTIONS] [ARGS]
    
+
    bayard-cli bulk-delete [OPTIONS] [DOCS]
    
 
    FLAGS
    
 
 
    ARGS
    
 
      
-
    • 
-
      <ID>
      
-     A unique ID that identifies the document in the index server.
      
-
      • 
-
    • 
-
      <FIELDS>
      
-     Fields of document to be indexed.
      
-
      • 
+
    • <DOCS>
      
+     Document containing the unique ID to be indexed.
      • 
 
    
 
    EXAMPLES
    
 
    To delete documents in bulk:
    
-
    $ cat ./examples/bulk_delete.jsonl | xargs -0 ./bin/bayard bulk-delete
+
    $ cat ./examples/bulk_delete.jsonl | xargs -0 bayard-cli bulk-delete
 
    
-
    bayard commit
    
+
    bayard-cli commit
    
 
    DESCRIPTION
    
 
    Commit index
    
 
    USAGE
    
-
    bayard commit [OPTIONS]
    
+
    bayard-cli commit [OPTIONS]
    
 
    FLAGS
    
 
 
    EXAMPLES
    
 
    To commit an index with options:
    
-
    $ ./bin/bayard commit --servers=127.0.0.1:5001
+
    $ bayard-cli commit --server=127.0.0.1:5001
 
    
-
    bayard rollback
    
+
    bayard-cli rollback
    
 
    DESCRIPTION
    
 
    Rollback index
    
 
    USAGE
    
-
    bayard commit [OPTIONS]
    
+
    bayard-cli rollback [OPTIONS]
    
 
    FLAGS
    
 
 
    EXAMPLES
    
 
    To rollback an index with options:
    
-
    $ ./bin/bayard rollback --servers=127.0.0.1:5001
+
    $ bayard-cli rollback --server=127.0.0.1:5001
 
    
-
    bayard merge
    
+
    bayard-cli merge
    
 
    DESCRIPTION
    
 
    Merge index
    
 
    USAGE
    
-
    bayard merge [OPTIONS]
    
+
    bayard-cli merge [OPTIONS]
    
 
    FLAGS
    
 
 
    EXAMPLES
    
 
    To merge an index with options:
    
-
    $ ./bin/bayard merge --servers=127.0.0.1:5001
+
    $ bayard-cli merge --server=127.0.0.1:5001
 
    
-
    bayard schema
    
+
    bayard-cli schema
    
 
    DESCRIPTION
    
 
    Show index schema
    
 
    USAGE
    
-
    bayard schema [OPTIONS]
    
+
    bayard-cli schema [OPTIONS]
    
 
    FLAGS
    
 
 
    EXAMPLES
    
 
    To show an index schema with options:
    
-
    $ ./bin/bayard schema --servers=127.0.0.1:5001
+
    $ bayard-cli schema --server=127.0.0.1:5001
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    [
@@ -1543,11 +1532,11 @@ 
    EXAMPLES
    
   }
 ]
 
    
-
    bayard search
    
+
    bayard-cli search
    
 
    DESCRIPTION
    
 
    Search documents from index server
    
 
    USAGE
    
-
    bayard search [FLAGS] [OPTIONS] 
    
+
    bayard-cli search [FLAGS] [OPTIONS] 
    
 
    FLAGS
    
 
      
 
    • 
@@ -1597,12 +1586,12 @@ 
      ARGS
      
 
    
 
    EXAMPLES
    
 
    To search documents from the index with options:
    
-
    $ ./bin/bayard search \
-               --server=0.0.0.0:5001 \
-               --facet-field=category \
-               --facet-prefix=/category/search \
-               --facet-prefix=/language \
-               description:rust | jq .
+
    $ bayard-cli search \
+             --server=0.0.0.0:5001 \
+             --facet-field=category \
+             --facet-prefix=/category/search \
+             --facet-prefix=/language \
+             description:rust | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    {
@@ -1672,11 +1661,11 @@ 
    EXAMPLES
    
   }
 }
 
    
-
    bayard status
    
+
    bayard-cli status
    
 
    DESCRIPTION
    
 
    Show system status
    
 
    USAGE
    
-
    bayard status [OPTIONS]
    
+
    bayard-cli status [OPTIONS]
    
 
    FLAGS
    
 
 
    EXAMPLES
    
 
    To show an index schema with options:
    
-
    $ ./bin/bayard status --server=0.0.0.0:5001 | jq .
+
    $ bayard status --server=0.0.0.0:5001 | jq .
 
    
 
    You'll see the result in JSON format. The result of the above command is:
    
 
    {
@@ -1726,11 +1715,11 @@ 
    EXAMPLES
    
   "status": "OK"
 }
 
    
-
    bayard metrics
    
+
    bayard-rest
    
 
    DESCRIPTION
    
-
    Shows system metrics
    
+
    Bayard REST server
    
 
    USAGE
    
-
    bayard metrics [OPTIONS]
    
+
    bayard-rest [OPTIONS]
    
 
    FLAGS
    
 
      
 
    • 
@@ -1744,91 +1733,6 @@ 
      FLAGS
      
 
    
 
    OPTIONS
    
 
      
-
    • -s, --server <IP:PORT>
      
-     Index service address. [default: 127.0.0.1:5000]
      • 
-
    
-
    EXAMPLES
    
-
    To show metrics with default options:
    
-
    $ ./bin/bayard metrics
-
    
-
    To show metrics with options:
    
-
    $ ./bin/bayard metrics --server=127.0.0.1:5001
-
    
-
    You'll see the result in Prometheus exposition format. The result of the above command is:
    
-
    # HELP bayard_applies_total Total number of applies.
-# TYPE bayard_applies_total counter
-bayard_applies_total{func="bulk_set"} 1
-bayard_applies_total{func="commit"} 1
-# HELP bayard_apply_duration_seconds The apply latencies in seconds.
-# TYPE bayard_apply_duration_seconds histogram
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.005"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.01"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.025"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.05"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.1"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.25"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.5"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="1"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="2.5"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="5"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="10"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="+Inf"} 1
-bayard_apply_duration_seconds_sum{func="bulk_set"} 0.001098082
-bayard_apply_duration_seconds_count{func="bulk_set"} 1
-bayard_apply_duration_seconds_bucket{func="commit",le="0.005"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.01"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.025"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.05"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.1"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.25"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.5"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="1"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="2.5"} 1
-bayard_apply_duration_seconds_bucket{func="commit",le="5"} 1
-bayard_apply_duration_seconds_bucket{func="commit",le="10"} 1
-bayard_apply_duration_seconds_bucket{func="commit",le="+Inf"} 1
-bayard_apply_duration_seconds_sum{func="commit"} 1.727736793
-bayard_apply_duration_seconds_count{func="commit"} 1
-
    
-
    bayard-rest
    
-
    DESCRIPTION
    
-
    Manage Bayard REST server
    
-
    USAGE
    
-
    bayard-rest <SUBCOMMAND>
    
-
    FLAGS
    
-
      
-
    • 
-
      -h, --help
      
-     Prints help information.
      
-
      • 
-
    • 
-
      -v, --version
      
-     Prints version information.
      
-
      • 
-
    
-
    SUBCOMMANDS
    
-
      
-
    • start
      
-     Start REST server
      • 
-
    
-
    bayard-rest start
    
-
    DESCRIPTION
    
-
    Start REST server
    
-
    USAGE
    
-
    bayard-rest start [OPTIONS]
    
-
    FLAGS
    
-
      
-
    • 
-
      -h, --help
      
-     Prints help information.
      
-
      • 
-
    • 
-
      -v, --version
      
-     Prints version information.
      
-
      • 
-
    
-
    OPTIONS
    
-
      
 
    • 
 
      -H, --host <HOST>
      
      Node address. [default: 0.0.0.0]
      
@@ -1842,9 +1746,11 @@ 
      OPTIONS
      
      Index service address. [default: 127.0.0.1:5000]
      
 
      • 
 
    
-
    EXAMPLES
    
+
    EXAMPLES
    
 
    To start a server with default options:
    
-
    $ bayard start --host=192.168.1.22 --port=8080 --server=192.168.1.12:5001
+
    $ bayard --host=192.168.1.22 \
+         --port=8001 \
+         --server=192.168.1.12:5001
 
    
 
    REST API
    
 
    The REST API can be used by starting the gateway with the bayard-rest CLI.
    
@@ -1895,10 +1801,6 @@ 
    REST API
    
 
    Status API
    
      Status API shows the cluster that the specified server is joining.
    
 
-
  • 
-
    Metrics API
    
-     Metrics API shows the server metrics of the specified server. The metrics are output in Prometheus exposition format.
    
-
    • 
 
 
    Get document API
    
 
    Get jdocument API gets a document with the specified ID.
    
@@ -1910,7 +1812,7 @@ 
    Path paramete
 
  • <ID>
    
      A unique value that identifies the document in the index.
    • 
 
-
    Examples
    
+
    Examples
    
 
    To get a document:
    
 
    $ curl -X GET 'http://localhost:8000/v1/documents/1' | jq .
 
    
@@ -1972,7 +1874,7 @@ 
    Path para
 
  • <ID>
    
      A unique value that identifies the document in the index.
    • 
 
-
    Examples
    
+
    Examples
    
 
    To delete a document:
    
 
    $ curl -X DELETE 'http://localhost:8000/v1/documents/1'
 
    
@@ -2003,7 +1905,7 @@ 
    Request body<DOCUMENT>
    
      Document(s) expressed in JSONL format
 
-
    Examples
    
+
    Examples
    
 
    To delete documents in bulk:
    
 
    $ curl -X DELETE \
     --header 'Content-Type: application/json' \
@@ -2024,7 +1926,7 @@ 
    Rollback APIRequest
    
 
    GET /v1/rollback
 
    
-
    Examples
    
+
    Examples
    
 
    To rollback an index:
    
 
    $ curl -X GET 'http://localhost:8000/v1/rollback'
 
    
@@ -2033,7 +1935,7 @@ 
    Merge API
    
 
    Request
    
 
    GET /v1/merge
 
    
-
    Examples
    
+
    Examples
    
 
    To merge segments in the index:
    
 
    $ curl -X GET 'http://localhost:8000/v1/merge'
 
    
@@ -2042,7 +1944,7 @@ 
    Schema API
    
 
    Request
    
 
    GET /v1/schema
 
    
-
    Examples
    
+
    Examples
    
 
    To show the index schema:
    
 
    $ curl -X POST 'http://localhost:8000/v1/schema' | jq .
 
    
@@ -2421,7 +2323,7 @@ 
    Status API
    
 
    Request
    
 
    GET /v1/status
 
    
-
    Examples
    
+
    Examples
    
 
    To show peers of the cluster:
    
 
    $ curl -X GET 'http://localhost:8000/v1/status'
 
    
@@ -2453,51 +2355,6 @@ 
    Examples
    
   ],
   "status": "OK"
 }
-
    
-
    Metrics API
    
-
    Metrics API shows the server metrics of the specified server. The metrics are output in Prometheus exposition format.
    
-
    Request
    
-
    GET /v1/metrics
-
    
-
    Examples
    
-
    To show metrics:
    
-
    $ curl -X GET 'http://localhost:8000/v1/metrics'
-
    
-
    You'll see the result in Prometheus exposition format. The result of the above command is:
    
-
    # HELP bayard_applies_total Total number of applies.
-# TYPE bayard_applies_total counter
-bayard_applies_total{func="bulk_set"} 1
-bayard_applies_total{func="commit"} 1
-# HELP bayard_apply_duration_seconds The apply latencies in seconds.
-# TYPE bayard_apply_duration_seconds histogram
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.005"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.01"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.025"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.05"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.1"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.25"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.5"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="1"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="2.5"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="5"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="10"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="+Inf"} 1
-bayard_apply_duration_seconds_sum{func="bulk_set"} 0.001098082
-bayard_apply_duration_seconds_count{func="bulk_set"} 1
-bayard_apply_duration_seconds_bucket{func="commit",le="0.005"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.01"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.025"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.05"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.1"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.25"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.5"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="1"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="2.5"} 1
-bayard_apply_duration_seconds_bucket{func="commit",le="5"} 1
-bayard_apply_duration_seconds_bucket{func="commit",le="10"} 1
-bayard_apply_duration_seconds_bucket{func="commit",le="+Inf"} 1
-bayard_apply_duration_seconds_sum{func="commit"} 1.727736793
-bayard_apply_duration_seconds_count{func="commit"} 1
 
    
 
                     
diff --git a/docs/reference.html b/docs/reference.html
index 3012039..46a24e7 100644
--- a/docs/reference.html
+++ b/docs/reference.html
@@ -81,7 +81,7 @@
 
         
diff --git a/docs/reference/api.html b/docs/reference/api.html
index 012e8df..67b5069 100644
--- a/docs/reference/api.html
+++ b/docs/reference/api.html
@@ -81,7 +81,7 @@
 
         
@@ -197,10 +197,6 @@ 
    REST API
    
 
    Status API
    
      Status API shows the cluster that the specified server is joining.
    
 
-
  • 
-
    Metrics API
    
-     Metrics API shows the server metrics of the specified server. The metrics are output in Prometheus exposition format.
    
-
    • 
 
 
                     
@@ -208,7 +204,7 @@ 
    REST API
    
                     
 
         
diff --git a/docs/reference/cli.html b/docs/reference/cli.html
index 643205e..72e3df9 100644
--- a/docs/reference/cli.html
+++ b/docs/reference/cli.html
@@ -81,7 +81,7 @@
 
         
@@ -154,11 +154,15 @@ 
    
 
  • 
 
    bayard
    
-     Manage Bayard index server
    
+     Bayard server
    
+
    • 
+
  • 
+
    bayard-cli
    
+     Bayard command-line interface
    
 
    • 
 
  • 
 
    bayard-rest
    
-     Manage Bayard REST server
    
+     Bayard REST server
    
 
    • 
 
 
diff --git a/docs/reference/api/metrics.html b/docs/reference/cli/bayard-cli.html
similarity index 59%
rename from docs/reference/api/metrics.html
rename to docs/reference/cli/bayard-cli.html
index 60d8a8b..e328d70 100644
--- a/docs/reference/api/metrics.html
+++ b/docs/reference/cli/bayard-cli.html
@@ -3,7 +3,7 @@
     
         
         
-        Metrics API - Bayard
+        bayard-cli - Bayard
         
 
         
@@ -81,7 +81,7 @@
 
         
@@ -148,63 +148,97 @@ 
    Bayard
    
 
                 
    
                     
    
-                        
    Metrics API
    
-
    Metrics API shows the server metrics of the specified server. The metrics are output in Prometheus exposition format.
    
-
    Request
    
-
    GET /v1/metrics
-
    
-
    Examples
    
-
    To show metrics:
    
-
    $ curl -X GET 'http://localhost:8000/v1/metrics'
-
    
-
    You'll see the result in Prometheus exposition format. The result of the above command is:
    
-
    # HELP bayard_applies_total Total number of applies.
-# TYPE bayard_applies_total counter
-bayard_applies_total{func="bulk_set"} 1
-bayard_applies_total{func="commit"} 1
-# HELP bayard_apply_duration_seconds The apply latencies in seconds.
-# TYPE bayard_apply_duration_seconds histogram
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.005"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.01"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.025"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.05"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.1"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.25"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="0.5"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="1"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="2.5"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="5"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="10"} 1
-bayard_apply_duration_seconds_bucket{func="bulk_set",le="+Inf"} 1
-bayard_apply_duration_seconds_sum{func="bulk_set"} 0.001098082
-bayard_apply_duration_seconds_count{func="bulk_set"} 1
-bayard_apply_duration_seconds_bucket{func="commit",le="0.005"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.01"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.025"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.05"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.1"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.25"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="0.5"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="1"} 0
-bayard_apply_duration_seconds_bucket{func="commit",le="2.5"} 1
-bayard_apply_duration_seconds_bucket{func="commit",le="5"} 1
-bayard_apply_duration_seconds_bucket{func="commit",le="10"} 1
-bayard_apply_duration_seconds_bucket{func="commit",le="+Inf"} 1
-bayard_apply_duration_seconds_sum{func="commit"} 1.727736793
-bayard_apply_duration_seconds_count{func="commit"} 1
-
    
+                        
    bayard-cli
    
+
    DESCRIPTION
    
+
    Bayard command-line interface
    
+
    USAGE
    
+
    bayard-cli <SUBCOMMAND>
    
+
    FLAGS
    
+
      
+
    • 
+
      -h, --help
      
+     Prints help information.
      
+
      • 
+
    • 
+
      -v, --version
      
+     Prints version information.
      
+
      • 
+
    
+
    SUBCOMMANDS
    
+
      
+
    • 
+
      leave
      
+     Delete document from index server
      
+
      • 
+
    • 
+
      get
      
+     Get document from index server
      
+
      • 
+
    • 
+
      set
      
+     Set document to index server
      
+
      • 
+
    • 
+
      delete
      
+     Delete document from index server
      
+
      • 
+
    • 
+
      bulk-set
      
+     Set documents to index server in bulk
      
+
      • 
+
    • 
+
      bulk-delete
      
+     Delete documents from index server in bulk
      
+
      • 
+
    • 
+
      commit
      
+     Commit index
      
+
      • 
+
    • 
+
      rollback
      
+     Rollback index
      
+
      • 
+
    • 
+
      merge
      
+     Merge index
      
+
      • 
+
    • 
+
      schema
      
+     Shows index schema that applied
      
+
      • 
+
    • 
+
      search
      
+     Get document from index server
      
+
      • 
+
    • 
+
      status
      
+     Shows system status
      
+
      • 
+
    • 
+
      metrics
      
+     Shows system metrics
      
+
      • 
+
    • 
+
      help
      
+     Prints this message or the help of the given subcommand(s)
      
+
      • 
+
    
 
                     
    
 
                     
@@ -213,12 +247,16 @@ 
    Examples
    
 
             
 
         
    
diff --git a/docs/reference/cli/bayard/bulk-delete.html b/docs/reference/cli/bayard-cli/bulk-delete.html
similarity index 69%
rename from docs/reference/cli/bayard/bulk-delete.html
rename to docs/reference/cli/bayard-cli/bulk-delete.html
index b523ea8..d5200c5 100644
--- a/docs/reference/cli/bayard/bulk-delete.html
+++ b/docs/reference/cli/bayard-cli/bulk-delete.html
@@ -81,7 +81,7 @@
 
         
@@ -148,11 +148,11 @@ 
    Bayard
    
 
                 
    
                     
    
-                        
    bayard bulk-delete
    
+                        
    bayard-cli bulk-delete
    
 
    DESCRIPTION
    
 
    Delete documents from index server in bulk
    
 
    USAGE
    
-
    bayard set [OPTIONS] [ARGS]
    
+
    bayard-cli bulk-delete [OPTIONS] [DOCS]
    
 
    FLAGS
    
 
 
    ARGS
    
 
      
-
    • 
-
      <ID>
      
-     A unique ID that identifies the document in the index server.
      
-
      • 
-
    • 
-
      <FIELDS>
      
-     Fields of document to be indexed.
      
-
      • 
+
    • <DOCS>
      
+     Document containing the unique ID to be indexed.
      • 
 
    
 
    EXAMPLES
    
 
    To delete documents in bulk:
    
-
    $ cat ./examples/bulk_delete.jsonl | xargs -0 ./bin/bayard bulk-delete
+
    $ cat ./examples/bulk_delete.jsonl | xargs -0 bayard-cli bulk-delete
 
    
 
                     
    
@@ -190,13 +184,13 @@ 
    EXAMPLES