mosuka · mosuka · Jan 3, 2020 · Jan 3, 2020
diff --git a/Cargo.toml b/Cargo.toml
@@ -31,6 +31,7 @@ iron = "0.6.1"
 job_scheduler = "1.1.0"
 log = "0.4.8"
 logger = "0.4.0"
+num_cpus = "1.11.1"
 persistent = "0.4.0"
 prometheus = "0.7.0"
 protobuf = "2.8.0"

diff --git a/docs/print.html b/docs/print.html
@@ -770,19 +770,29 @@ <h2><a class="header" href="#flags-1" id="flags-1">FLAGS</a></h2>
 </code></pre>
 <h2><a class="header" href="#options-1" id="options-1">OPTIONS</a></h2>
 <pre><code>-i, --id &lt;ID&gt;
-        Server ID. Must specify a numeric ID that is unique within the cluster. If not specified, use the default ID. [default: 1]
+        Server ID. Must specify a numeric ID that is unique within the cluster. If not specified, use the default
+        ID. [default: 1]
 -H, --host &lt;HOST&gt;
-        Host address. Must specify the host name or IP address. If not specified, use the default address. [default: 0.0.0.0]
+        Host address. Must specify the host name or IP address. If not specified, use the default address. [default:
+        0.0.0.0]
 -P, --port &lt;PORT&gt;
-        Port number. This port is used for communication via gRPC. If not specified, use the default port. [default: 5000]
+        Port number. This port is used for communication via gRPC. If not specified, use the default port. [default:
+        5000]
 -p, --peers &lt;ID=IP:PORT&gt;...
-        Server ID and addresses in an existing cluster separated by &quot;,&quot;. If specified, the server will join the cluster.
+        Server ID and addresses in an existing cluster separated by &quot;,&quot;. If specified, the server will join the
+        cluster.
 -d, --data-directory &lt;DATA_DIRECTORY&gt;
-        Data directory. Stores index, snapshots, and raft logs. If not specified, use the default directory. [default: ./data]
+        Data directory. Stores index, snapshots, and raft logs. If not specified, use the default directory.
+        [default: ./data]
 -s, --schema-file &lt;SCHEMA_FILE&gt;
-        Schema file. Must specify An existing file name. If not specified, use the default schema file. [default: ./etc/schema.json]
--u, --unique-key-field-name &lt;UNIQUE_KEY_FIELD_NAME&gt;
-        Unique key field name. Specify the field name to be treated as a unique key in the field defined in the schema. If not specified, use the default unique key field name. [default: id]
+        Schema file. Must specify An existing file name. If not specified, use the default schema file. [default:
+        ./etc/schema.json]
+-t, --indexer-threads &lt;INDEXER_THREADS&gt;
+        Number of indexer threads. If not specified, number of CPU cores - 1 will be used. [default: 7]
+
+-m, --indexer-memory-size &lt;INDEXER_MEMORY_SIZE&gt;
+        Total memory size (in bytes) used by the indexer. It will be split for the different thread. If not
+        specified, use the default. [default: 1000000000]
 </code></pre>
 <h2><a class="header" href="#examples-1" id="examples-1">EXAMPLES</a></h2>
 <p>To start a server with default options:</p>
@@ -794,8 +804,7 @@ <h2><a class="header" href="#examples-1" id="examples-1">EXAMPLES</a></h2>
       --host=0.0.0.0 \
       --port=5001 \
       --data-directory=./data/1 \
-      --schema-file=./etc/schema.json \
-      --unique-key-field-name=id
+      --schema-file=./etc/schema.json
 </code></pre>
 <h1><a class="header" href="#bayard-probe" id="bayard-probe">bayard probe</a></h1>
 <p>The <code>bayard probe</code> CLI probes the server.</p>

diff --git a/docs/reference/bayard/serve.html b/docs/reference/bayard/serve.html
@@ -159,19 +159,29 @@ <h2><a class="header" href="#flags" id="flags">FLAGS</a></h2>
 </code></pre>
 <h2><a class="header" href="#options" id="options">OPTIONS</a></h2>
 <pre><code>-i, --id &lt;ID&gt;
-        Server ID. Must specify a numeric ID that is unique within the cluster. If not specified, use the default ID. [default: 1]
+        Server ID. Must specify a numeric ID that is unique within the cluster. If not specified, use the default
+        ID. [default: 1]
 -H, --host &lt;HOST&gt;
-        Host address. Must specify the host name or IP address. If not specified, use the default address. [default: 0.0.0.0]
+        Host address. Must specify the host name or IP address. If not specified, use the default address. [default:
+        0.0.0.0]
 -P, --port &lt;PORT&gt;
-        Port number. This port is used for communication via gRPC. If not specified, use the default port. [default: 5000]
+        Port number. This port is used for communication via gRPC. If not specified, use the default port. [default:
+        5000]
 -p, --peers &lt;ID=IP:PORT&gt;...
-        Server ID and addresses in an existing cluster separated by &quot;,&quot;. If specified, the server will join the cluster.
+        Server ID and addresses in an existing cluster separated by &quot;,&quot;. If specified, the server will join the
+        cluster.
 -d, --data-directory &lt;DATA_DIRECTORY&gt;
-        Data directory. Stores index, snapshots, and raft logs. If not specified, use the default directory. [default: ./data]
+        Data directory. Stores index, snapshots, and raft logs. If not specified, use the default directory.
+        [default: ./data]
 -s, --schema-file &lt;SCHEMA_FILE&gt;
-        Schema file. Must specify An existing file name. If not specified, use the default schema file. [default: ./etc/schema.json]
--u, --unique-key-field-name &lt;UNIQUE_KEY_FIELD_NAME&gt;
-        Unique key field name. Specify the field name to be treated as a unique key in the field defined in the schema. If not specified, use the default unique key field name. [default: id]
+        Schema file. Must specify An existing file name. If not specified, use the default schema file. [default:
+        ./etc/schema.json]
+-t, --indexer-threads &lt;INDEXER_THREADS&gt;
+        Number of indexer threads. If not specified, number of CPU cores - 1 will be used. [default: 7]
+
+-m, --indexer-memory-size &lt;INDEXER_MEMORY_SIZE&gt;
+        Total memory size (in bytes) used by the indexer. It will be split for the different thread. If not
+        specified, use the default. [default: 1000000000]
 </code></pre>
 <h2><a class="header" href="#examples" id="examples">EXAMPLES</a></h2>
 <p>To start a server with default options:</p>
@@ -183,8 +193,7 @@ <h2><a class="header" href="#examples" id="examples">EXAMPLES</a></h2>
       --host=0.0.0.0 \
       --port=5001 \
       --data-directory=./data/1 \
-      --schema-file=./etc/schema.json \
-      --unique-key-field-name=id
+      --schema-file=./etc/schema.json
 </code></pre>
 
                     </main>

diff --git a/docs/searchindex.js b/docs/searchindex.js
diff --git a/docs/searchindex.json b/docs/searchindex.json
diff --git a/docs_src/reference/bayard/serve.md b/docs_src/reference/bayard/serve.md
@@ -14,19 +14,29 @@ The `bayard serve` CLI starts the server.
 ## OPTIONS
 
     -i, --id <ID>
-            Server ID. Must specify a numeric ID that is unique within the cluster. If not specified, use the default ID. [default: 1]
+            Server ID. Must specify a numeric ID that is unique within the cluster. If not specified, use the default
+            ID. [default: 1]
     -H, --host <HOST>
-            Host address. Must specify the host name or IP address. If not specified, use the default address. [default: 0.0.0.0]
+            Host address. Must specify the host name or IP address. If not specified, use the default address. [default:
+            0.0.0.0]
     -P, --port <PORT>
-            Port number. This port is used for communication via gRPC. If not specified, use the default port. [default: 5000]
+            Port number. This port is used for communication via gRPC. If not specified, use the default port. [default:
+            5000]
     -p, --peers <ID=IP:PORT>...
-            Server ID and addresses in an existing cluster separated by ",". If specified, the server will join the cluster.
+            Server ID and addresses in an existing cluster separated by ",". If specified, the server will join the
+            cluster.
     -d, --data-directory <DATA_DIRECTORY>
-            Data directory. Stores index, snapshots, and raft logs. If not specified, use the default directory. [default: ./data]
+            Data directory. Stores index, snapshots, and raft logs. If not specified, use the default directory.
+            [default: ./data]
     -s, --schema-file <SCHEMA_FILE>
-            Schema file. Must specify An existing file name. If not specified, use the default schema file. [default: ./etc/schema.json]
-    -u, --unique-key-field-name <UNIQUE_KEY_FIELD_NAME>
-            Unique key field name. Specify the field name to be treated as a unique key in the field defined in the schema. If not specified, use the default unique key field name. [default: id]
+            Schema file. Must specify An existing file name. If not specified, use the default schema file. [default:
+            ./etc/schema.json]
+    -t, --indexer-threads <INDEXER_THREADS>
+            Number of indexer threads. If not specified, number of CPU cores - 1 will be used. [default: 7]
+
+    -m, --indexer-memory-size <INDEXER_MEMORY_SIZE>
+            Total memory size (in bytes) used by the indexer. It will be split for the different thread. If not
+            specified, use the default. [default: 1000000000]
 
 ## EXAMPLES
 
@@ -44,6 +54,5 @@ $ ./bin/bayard serve \
       --host=0.0.0.0 \
       --port=5001 \
       --data-directory=./data/1 \
-      --schema-file=./etc/schema.json \
-      --unique-key-field-name=id
+      --schema-file=./etc/schema.json
 ```
diff --git a/proto/indexrpcpb.proto b/proto/indexrpcpb.proto
@@ -6,19 +6,21 @@ import "eraftpb.proto";
 enum RespErr {
     OK = 0;
     ErrWrongLeader = 1;
-    ErrProbeFailed = 2;
-    ErrPeerFailed = 3;
-    ErrMetricsFailed = 4;
-    ErrGetFailed = 5;
-    ErrPutFailed = 6;
-    ErrDeleteFailed = 7;
-    ErrBulkPutFailed = 8;
-    ErrBulkDeleteFailed = 9;
-    ErrCommitFailed = 10;
-    ErrRollbackFailed = 11;
-    ErrMergeFailed = 12;
-    ErrSearchFailed = 13;
-    ErrSchemaFailed = 14;
+    ErrTimeout = 2;
+    ErrDisconnected = 3;
+    ErrProbeFailed = 4;
+    ErrPeerFailed = 5;
+    ErrMetricsFailed = 6;
+    ErrGetFailed = 7;
+    ErrPutFailed = 8;
+    ErrDeleteFailed = 9;
+    ErrBulkPutFailed = 10;
+    ErrBulkDeleteFailed = 11;
+    ErrCommitFailed = 12;
+    ErrRollbackFailed = 13;
+    ErrMergeFailed = 14;
+    ErrSearchFailed = 15;
+    ErrSchemaFailed = 16;
 }
 
 enum ReqType {

diff --git a/src/client/client.rs b/src/client/client.rs
@@ -560,7 +560,13 @@ impl Clerk {
                     resp
                 });
             match reply.err {
-                RespErr::OK => return reply.value,
+                RespErr::OK => {
+                    debug!("commit succeeded");
+                    return reply.value;
+                }
+                RespErr::ErrWrongLeader => error!("wrong leader"),
+                RespErr::ErrTimeout => error!("timeout"),
+                RespErr::ErrDisconnected => error!("disconnected"),
                 _ => error!("failed to commit index"),
             }
             self.leader_id = (self.leader_id + 1) % self.servers.len();

diff --git a/src/cmd/serve.rs b/src/cmd/serve.rs
@@ -24,8 +24,27 @@ pub fn run_serve_cli(matches: &ArgMatches) -> Result<(), String> {
     }
     let data_directory = matches.value_of("DATA_DIRECTORY").unwrap();
     let schema_file = matches.value_of("SCHEMA_FILE").unwrap();
+    let indexer_threads = matches
+        .value_of("INDEXER_THREADS")
+        .unwrap()
+        .parse::<usize>()
+        .unwrap();
+    let indexer_memory_size = matches
+        .value_of("INDEXER_MEMORY_SIZE")
+        .unwrap()
+        .parse::<usize>()
+        .unwrap();
 
-    IndexServer::start_server(id, host, port, peers_addr, data_directory, schema_file);
+    IndexServer::start_server(
+        id,
+        host,
+        port,
+        peers_addr,
+        data_directory,
+        schema_file,
+        indexer_threads,
+        indexer_memory_size,
+    );
 
     Ok(())
 }
diff --git a/src/main.rs b/src/main.rs
@@ -1,6 +1,7 @@
 extern crate bayard;
 #[macro_use]
 extern crate clap;
+extern crate num_cpus;
 
 use std::io::Write;
 
@@ -23,12 +24,19 @@ use bayard::cmd::search::run_search_cli;
 use bayard::cmd::serve::run_serve_cli;
 
 fn main() {
+    let cpus = num_cpus::get().to_owned();
+    let threads;
+    if cpus > 1 {
+        threads = format!("{}", cpus - 1);
+    } else {
+        threads = format!("{}", cpus);
+    }
+
     let app = App::new(crate_name!())
         .setting(AppSettings::DeriveDisplayOrder)
         .setting(AppSettings::SubcommandRequiredElseHelp)
         .version(crate_version!())
         .author(crate_authors!())
-//        .about(crate_description!())
         .about("The `bayard` CLI manages server, cluster and index.")
         .help_message("Prints help information.")
         .version_message("Prints version information.")
@@ -100,15 +108,24 @@ fn main() {
                         .default_value("./etc/schema.json")
                         .takes_value(true),
                 )
-//                .arg(
-//                    Arg::with_name("UNIQUE_KEY_FIELD_NAME")
-//                        .help("Unique key field name. Specify the field name to be treated as a unique key in the field defined in the schema. If not specified, use the default unique key field name.")
-//                        .short("u")
-//                        .long("unique-key-field-name")
-//                        .value_name("UNIQUE_KEY_FIELD_NAME")
-//                        .default_value("_id")
-//                        .takes_value(true),
-//                )
+                .arg(
+                    Arg::with_name("INDEXER_THREADS")
+                        .help("Number of indexer threads. If not specified, number of CPU cores - 1 will be used.")
+                        .short("t")
+                        .long("indexer-threads")
+                        .value_name("INDEXER_THREADS")
+                        .default_value(&threads)
+                        .takes_value(true),
+                )
+                .arg(
+                    Arg::with_name("INDEXER_MEMORY_SIZE")
+                        .help("Total memory size (in bytes) used by the indexer. It will be split for the different thread. If not specified, use the default.")
+                        .short("m")
+                        .long("indexer-memory-size")
+                        .value_name("INDEXER_MEMORY_SIZE")
+                        .default_value("1000000000")
+                        .takes_value(true),
+                )
         )
         .subcommand(
             SubCommand::with_name("probe")