From 692671504969d476097e61bcb03b104547005d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20R=C3=B3=C5=BCa=C5=84ski?= Date: Tue, 19 Sep 2023 10:34:46 +0200 Subject: [PATCH] Post Service POC --- .gitmodules | 3 + Cargo.lock | 555 ++++++++++++++++++++++++++++++++++++- Cargo.toml | 2 +- service/Cargo.toml | 29 ++ service/api | 1 + service/build.rs | 4 + service/src/client.rs | 116 ++++++++ service/src/main.rs | 175 ++++++++++++ service/src/service.rs | 120 ++++++++ service/src/test_server.rs | 143 ++++++++++ 10 files changed, 1137 insertions(+), 11 deletions(-) create mode 100644 .gitmodules create mode 100644 service/Cargo.toml create mode 160000 service/api create mode 100644 service/build.rs create mode 100644 service/src/client.rs create mode 100644 service/src/main.rs create mode 100644 service/src/service.rs create mode 100644 service/src/test_server.rs diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..3d4297c5 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "service/api"] + path = service/api + url = https://github.com/spacemeshos/api.git diff --git a/Cargo.lock b/Cargo.lock index 48453e25..8561f1c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,6 +118,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + [[package]] name = "arrayref" version = "0.3.7" @@ -130,6 +136,39 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + +[[package]] +name = "async-trait" +version = "0.1.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + [[package]] name = "atty" version = "0.2.14" @@ -147,6 +186,51 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -292,6 +376,12 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + [[package]] name = "cast" version = "0.3.0" @@ -428,20 +518,19 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.1" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c8d502cbaec4595d2e7d5f61e318f05417bd2b66fdc3809498f0d3fdf0bea27" +checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.4.1" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5891c7bc0edb3e1c2204fc5e94009affabeb1821c9e5fdc3959536c5c0bb984d" +checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" dependencies = [ "anstream", "anstyle", @@ -451,9 +540,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" dependencies = [ "heck", "proc-macro2", @@ -521,7 +610,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.4.1", + "clap 4.4.4", "criterion-plot", "is-terminal", "itertools", @@ -802,6 +891,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "float-cmp" version = "0.9.0" @@ -963,6 +1058,25 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "h2" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 1.9.3", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "1.8.2" @@ -1008,12 +1122,82 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.9", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "iana-time-zone" version = "0.1.57" @@ -1113,7 +1297,7 @@ name = "initializer" version = "0.4.4" dependencies = [ "base64 0.21.3", - "clap 4.4.1", + "clap 4.4.4", "env_logger", "eyre", "post-rs", @@ -1223,6 +1407,12 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "matchit" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" + [[package]] name = "memchr" version = "2.6.2" @@ -1247,6 +1437,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1262,6 +1458,17 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + [[package]] name = "mockall" version = "0.11.4" @@ -1289,6 +1496,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "nix" version = "0.26.4" @@ -1491,6 +1704,42 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap 2.0.0", +] + +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -1675,7 +1924,7 @@ dependencies = [ name = "profiler" version = "0.4.4" dependencies = [ - "clap 4.4.1", + "clap 4.4.4", "env_logger", "eyre", "hex", @@ -1708,6 +1957,60 @@ dependencies = [ "unarray", ] +[[package]] +name = "prost" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" +dependencies = [ + "bytes", + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.29", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.29", +] + +[[package]] +name = "prost-types" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" +dependencies = [ + "prost", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -1973,6 +2276,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + [[package]] name = "rusty-fork" version = "0.3.0" @@ -2101,6 +2410,24 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "service" +version = "0.1.0" +dependencies = [ + "async-stream", + "clap 4.4.4", + "env_logger", + "eyre", + "hex", + "log", + "post-rs", + "prost", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", +] + [[package]] name = "shlex" version = "1.1.0" @@ -2122,6 +2449,26 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2197,6 +2544,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "tap" version = "1.0.1" @@ -2305,6 +2658,69 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tokio" +version = "1.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "pin-project-lite", + "socket2 0.5.4", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml" version = "0.5.11" @@ -2331,6 +2747,116 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5469afaf78a11265c343a88969045c1568aa8ecc6c787dbf756e92e70f199861" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.3", + "bytes", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b477abbe1d18c0b08f56cd01d1bc288668c5b5cfd19b2ae1886bbf599c546f1" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn 2.0.29", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + +[[package]] +name = "tracing-core" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "typenum" version = "1.16.0" @@ -2398,6 +2924,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 2f324ff9..1dcf3a14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "ffi", "scrypt-ocl", "initializer", "profiler"] +members = [".", "ffi", "scrypt-ocl", "initializer", "profiler", "service"] [package] name = "post-rs" diff --git a/service/Cargo.toml b/service/Cargo.toml new file mode 100644 index 00000000..6e2e005e --- /dev/null +++ b/service/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "service" +version = "0.1.0" +edition = "2021" + +[[bin]] # Bin to run the HelloWorld gRPC server +name = "test-server" +path = "src/test_server.rs" + +[dependencies] +post-rs = { path = "../" } +prost = "0.12.1" +tonic = "0.10.0" +tokio = { version = "1.0", features = [ + "rt-multi-thread", + "macros", + "sync", + "time", +] } +tokio-stream = "0.1" +async-stream = "0.3.5" +log = "0.4.20" +eyre = "0.6.8" +env_logger = "0.10.0" +clap = { version = "4.4.4", features = ["derive"] } +hex = "0.4.3" + +[build-dependencies] +tonic-build = "0.10.0" diff --git a/service/api b/service/api new file mode 160000 index 00000000..3cdeeff6 --- /dev/null +++ b/service/api @@ -0,0 +1 @@ +Subproject commit 3cdeeff627f1b31ec5becd46b6109784de749892 diff --git a/service/build.rs b/service/build.rs new file mode 100644 index 00000000..b4eb8a0a --- /dev/null +++ b/service/build.rs @@ -0,0 +1,4 @@ +fn main() -> Result<(), Box> { + tonic_build::configure().compile(&["api/spacemesh/v1/post.proto"], &["api"])?; + Ok(()) +} diff --git a/service/src/client.rs b/service/src/client.rs new file mode 100644 index 00000000..be3f50a9 --- /dev/null +++ b/service/src/client.rs @@ -0,0 +1,116 @@ +pub(crate) use spacemesh_v1::post_service_client::PostServiceClient; +use spacemesh_v1::service_response; +use spacemesh_v1::ServiceResponse; +use spacemesh_v1::{GenProofResponse, GenProofStatus, Proof, ProofMetadata}; +use tokio::sync::{mpsc, oneshot}; +use tonic::transport::Channel; +use tonic::Request; + +use crate::service::Command; +use spacemesh_v1::node_request; + +pub mod spacemesh_v1 { + tonic::include_proto!("spacemesh.v1"); +} + +pub(crate) async fn run( + mut client: PostServiceClient, + cmds: mpsc::Sender, +) -> eyre::Result<()> { + let (tx, mut rx) = mpsc::channel::(1); + let outbound = async_stream::stream! { + while let Some(msg) = rx.recv().await { + yield msg; + } + }; + + let response = client.register(Request::new(outbound)).await?; + let mut inbound = response.into_inner(); + + while let Some(request) = inbound.message().await? { + log::debug!("Got request from node: {request:?}"); + match request.kind { + Some(node_request::Kind::GenProof(req)) => { + let (cmd_response, rx) = oneshot::channel(); + let Ok(challenge) = req.challenge.try_into() else { + log::error!("invalid challenge"); + tx.send(ServiceResponse { + kind: Some(service_response::Kind::GenProof(GenProofResponse { + status: GenProofStatus::Error as i32, + ..Default::default() + })), + }) + .await?; + continue; + }; + + // Forward the request to the service + cmds.send(Command::GenProof { + challenge, + response: cmd_response, + }) + .await?; + + // Process the response from the service + let resp = match rx.await? { + Ok(Some(resp)) => { + log::info!("proof generation finished"); + ServiceResponse { + kind: Some(service_response::Kind::GenProof(GenProofResponse { + proof: Some(Proof { + nonce: resp.0.nonce, + indices: resp.0.indices.into_owned(), + pow: resp.0.pow, + }), + metadata: Some(ProofMetadata { + challenge: resp.1.challenge.to_vec(), + node_id: Some(spacemesh_v1::SmesherId { + id: resp.1.node_id.to_vec(), + }), + commitment_atx_id: Some(spacemesh_v1::ActivationId { + id: resp.1.commitment_atx_id.to_vec(), + }), + num_units: resp.1.num_units, + labels_per_unit: resp.1.labels_per_unit, + }), + status: GenProofStatus::Ok as i32, + })), + } + } + Ok(None) => { + log::info!("proof generation in progress"); + ServiceResponse { + kind: Some(service_response::Kind::GenProof(GenProofResponse { + status: GenProofStatus::Ok as i32, + ..Default::default() + })), + } + } + Err(e) => { + log::error!("failed to generate proof: {e:?}"); + ServiceResponse { + kind: Some(service_response::Kind::GenProof(GenProofResponse { + status: GenProofStatus::Error as i32, + ..Default::default() + })), + } + } + }; + + tx.send(resp).await?; + } + None => { + log::warn!("Got a request with no kind"); + tx.send(ServiceResponse { + kind: Some(service_response::Kind::GenProof(GenProofResponse { + status: GenProofStatus::Error as i32, + ..Default::default() + })), + }) + .await? + } + } + } + + Ok(()) +} diff --git a/service/src/main.rs b/service/src/main.rs new file mode 100644 index 00000000..52736084 --- /dev/null +++ b/service/src/main.rs @@ -0,0 +1,175 @@ +use std::{path::PathBuf, time::Duration}; + +use clap::{Args, Parser, ValueEnum}; +use eyre::Context; +use post::pow::randomx::RandomXFlag; +use tokio::{sync::mpsc, time::sleep}; + +mod client; +mod service; + +/// Post Service +#[derive(Parser, Debug)] +#[command(version, about)] +struct Cli { + /// Directory of POST data + #[arg(short, long)] + dir: PathBuf, + + /// Node address to connect to + #[arg(short, long)] + address: String, + + #[command(flatten, next_help_heading = "POST configuration")] + post_config: PostConfig, + + #[command(flatten, next_help_heading = "POST settings")] + post_settings: PostSettings, +} + +#[derive(Args, Debug)] +/// POST configuration - network parameters +struct PostConfig { + /// K1 specifies the difficulty for a label to be a candidate for a proof. + #[arg(long, default_value = "26")] + k1: u32, + /// K2 is the number of labels below the required difficulty required for a proof. + #[arg(long, default_value = "37")] + k2: u32, + /// K3 is the size of the subset of proof indices that is validated. + #[arg(long, default_value = "37")] + k3: u32, + /// Difficulty for the nonce proof of work. Lower values increase difficulty of finding + /// `pow` for [Proof][crate::prove::Proof]. + #[arg( + long, + default_value = "000dfb23b0979b4b000000000000000000000000000000000000000000000000", + value_parser(parse_difficulty) + )] + pow_difficulty: [u8; 32], + + /// Scrypt parameters for initialization + #[command(flatten)] + scrypt: ScryptParams, +} + +#[derive(Args, Debug)] +struct ScryptParams { + /// Scrypt N parameter + #[arg(short, default_value_t = 8192)] + n: usize, + + /// Scrypt R parameter + #[arg(short, default_value_t = 1)] + r: usize, + + /// Scrypt P parameter + #[arg(short, default_value_t = 1)] + p: usize, +} +#[derive(Args, Debug)] +/// POST proof generation settings +struct PostSettings { + /// Number of threads to use. + /// '0' means use all available threads + #[arg(long, default_value_t = 1)] + threads: usize, + + /// Number of nonces to attempt in single pass over POS data. + /// + /// Each group of 16 nonces requires a separate PoW. Must be a multiple of 16. + /// + /// Higher value gives a better chance to find a proof within less passes over the POS data, + /// but also slows down the process. + #[arg(long, default_value_t = 128, value_parser(parse_nonces))] + nonces: usize, + + /// Modes of operation for RandomX. + /// + /// They are interchangeable as they give the same results but have different + /// purpose and memory requirements. + #[arg(long, default_value_t = RandomXMode::Fast)] + randomx_mode: RandomXMode, +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq, ValueEnum)] +enum RandomXMode { + /// Fast mode for proving. Requires 2080 MiB of memory. + Fast, + /// Light mode for verification. Requires only 256 MiB of memory, but runs significantly slower + Light, +} + +impl std::fmt::Display for RandomXMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.to_possible_value().unwrap().get_name().fmt(f) + } +} + +impl From for RandomXFlag { + fn from(val: RandomXMode) -> Self { + match val { + RandomXMode::Fast => RandomXFlag::get_recommended_flags() | RandomXFlag::FLAG_FULL_MEM, + RandomXMode::Light => RandomXFlag::get_recommended_flags(), + } + } +} + +fn parse_nonces(arg: &str) -> eyre::Result { + let nonces = arg.parse()?; + eyre::ensure!(nonces % 16 == 0, "nonces must be multiple of 16"); + eyre::ensure!(nonces / 16 <= 256, format!("max nonces is {}", 256 * 16)); + Ok(nonces) +} + +fn parse_difficulty(arg: &str) -> eyre::Result<[u8; 32]> { + hex::decode(arg)? + .as_slice() + .try_into() + .wrap_err("invalid difficulty length") +} + +#[tokio::main] +async fn main() -> eyre::Result<()> { + let args = Cli::parse(); + + let env = env_logger::Env::default().filter_or("RUST_LOG", "info"); + env_logger::init_from_env(env); + + let (tx, rx) = mpsc::channel(1); + + let service = service::PostService::new( + rx, + args.dir, + post::config::Config { + k1: args.post_config.k1, + k2: args.post_config.k2, + k3: args.post_config.k3, + pow_difficulty: args.post_config.pow_difficulty, + scrypt: post::ScryptParams::new( + args.post_config.scrypt.n.ilog2() as u8 - 1, + args.post_config.scrypt.r.ilog2() as u8, + args.post_config.scrypt.p.ilog2() as u8, + ), + }, + args.post_settings.nonces, + args.post_settings.threads, + args.post_settings.randomx_mode.into(), + )?; + tokio::spawn(service.run()); + + loop { + let client = loop { + match client::PostServiceClient::connect(args.address.clone()).await { + Ok(client) => break client, + Err(e) => { + log::error!("failed to connect to node: {e}"); + sleep(Duration::from_secs(1)).await; + } + } + }; + let res = client::run(client, tx.clone()).await; + log::info!("client exited: {res:?}"); + } + // Ok(()) +} diff --git a/service/src/service.rs b/service/src/service.rs new file mode 100644 index 00000000..20db2a2b --- /dev/null +++ b/service/src/service.rs @@ -0,0 +1,120 @@ +use std::path::PathBuf; + +use eyre::Context; +use post::{metadata::ProofMetadata, pow::randomx::RandomXFlag, prove::Proof}; +use tokio::sync::{mpsc, oneshot}; + +#[derive(Debug)] +pub(crate) enum Command { + GenProof { + challenge: [u8; 32], + response: oneshot::Sender, ProofMetadata)>>>, + }, +} + +#[derive(Debug)] +pub(crate) struct PostService { + id: [u8; 32], + datadir: PathBuf, + cfg: post::config::Config, + nonces: usize, + threads: usize, + pow_flags: RandomXFlag, + rx: mpsc::Receiver, + proof_generation: Option>>>, +} + +impl PostService { + pub(crate) fn new( + rx: mpsc::Receiver, + datadir: PathBuf, + cfg: post::config::Config, + nonces: usize, + threads: usize, + pow_flags: RandomXFlag, + ) -> eyre::Result { + let metadata = + post::metadata::load(&datadir).wrap_err("loading metadata. Is POST initialized?")?; + let id = metadata.node_id; + + Ok(Self { + id, + rx, + proof_generation: None, + datadir, + cfg, + nonces, + threads, + pow_flags, + }) + } + + pub(crate) async fn run(mut self) -> eyre::Result<()> { + log::info!("starting PostService"); + while let Some(cmd) = self.rx.recv().await { + log::info!("got {cmd:?}"); + match cmd { + Command::GenProof { + challenge, + response, + } => { + log::info!("got GenProof command"); + match &mut self.proof_generation { + Some(handle) => { + if handle.is_finished() { + log::info!("proof generation is finished"); + let result = handle.await?; + self.proof_generation = None; + match result { + Ok(proof) => { + let metadata = post::metadata::load(&self.datadir).unwrap(); + + _ = response.send(Ok(Some(( + proof, + ProofMetadata { + challenge, + node_id: metadata.node_id, + commitment_atx_id: metadata.commitment_atx_id, + num_units: metadata.num_units, + labels_per_unit: metadata.labels_per_unit, + }, + )))); + } + Err(e) => { + _ = response.send(Err(e)); + } + } + } else { + log::info!("proof generation in progress"); + _ = response.send(Ok(None)); + } + } + None => { + log::info!("starting proof generation"); + let pow_flags = self.pow_flags; + let cfg = self.cfg; + let datadir = self.datadir.clone(); + let node_id = self.id; + let nonces = self.nonces; + let threads = self.threads; + self.proof_generation = Some(tokio::task::spawn_blocking(move || { + post::prove::generate_proof( + &datadir, + &challenge, + cfg, + nonces, + threads, + pow_flags, + Some(node_id), + ) + })); + // in progress + _ = response.send(Ok(None)); + } + } + } + } + } + Ok(()) + } +} diff --git a/service/src/test_server.rs b/service/src/test_server.rs new file mode 100644 index 00000000..1d2568a9 --- /dev/null +++ b/service/src/test_server.rs @@ -0,0 +1,143 @@ +use std::pin::Pin; +use std::sync::Mutex; +use std::time::Duration; + +use tokio::sync::{broadcast, mpsc, oneshot}; +use tokio::time::sleep; +use tokio_stream::{Stream, StreamExt}; +use tonic::{transport::Server, Request, Response, Status}; + +use spacemesh_v1::post_service_server::{PostService, PostServiceServer}; +use spacemesh_v1::{NodeRequest, ServiceResponse}; + +use spacemesh_v1::node_request; +use spacemesh_v1::GenProofRequest; +pub mod spacemesh_v1 { + tonic::include_proto!("spacemesh.v1"); +} + +struct TestNodeRequest { + request: NodeRequest, + response: oneshot::Sender, +} + +#[derive(Debug)] +pub struct TestPostService { + registered: Mutex>>, +} + +impl TestPostService { + fn new() -> Self { + Self { + registered: Mutex::new(broadcast::channel(1).0), + } + } + fn wait_for_connection(&mut self) -> broadcast::Receiver> { + self.registered.lock().unwrap().subscribe() + } +} + +#[tonic::async_trait] +impl PostService for TestPostService { + type RegisterStream = Pin> + Send + 'static>>; + + async fn register( + &self, + request: Request>, + ) -> Result, Status> { + log::info!("Post Service connected: {:?}", request); + let mut stream = request.into_inner(); + + let (tx, mut rx) = tokio::sync::mpsc::channel(1); + self.registered + .lock() + .unwrap() + .send(tx) + .expect("nobody is interested in post service registered"); + + let output = async_stream::try_stream! { + while let Some(req) = rx.recv().await { + yield req.request; + if let Some(Ok(response)) = stream.next().await { + _ = req.response.send(response); + } else { + log::info!("stream closed"); + return; + } + } + }; + + Ok(Response::new(Box::pin(output) as Self::RegisterStream)) + } +} + +#[tokio::main] +async fn main() -> eyre::Result<()> { + let env = env_logger::Env::default().filter_or("RUST_LOG", "info"); + env_logger::init_from_env(env); + + let addr = "[::1]:50051".parse()?; + + let mut test_node = TestPostService::new(); + + let mut reg = test_node.wait_for_connection(); + + let _handle = tokio::spawn( + Server::builder() + .add_service(PostServiceServer::new(test_node)) + .serve(addr), + ); + + loop { + // wait for the connection to be established + let tx = reg.recv().await?; + + loop { + let (resp_tx, resp_rx) = oneshot::channel(); + if let Err(e) = tx + .send(TestNodeRequest { + request: NodeRequest { + kind: Some(node_request::Kind::GenProof(GenProofRequest { + challenge: vec![0xCA; 32], + })), + }, + response: resp_tx, + }) + .await + { + log::error!("post service disconnected: {:?}", e); + break; + } + + let resp = resp_rx.await?; + match resp.kind { + Some(spacemesh_v1::service_response::Kind::GenProof(resp)) => { + log::debug!("Got GenProof response: {resp:?}"); + match resp.status() { + spacemesh_v1::GenProofStatus::Ok => { + if let Some(proof) = resp.proof { + log::info!("POST proof generation finished, proof: {:?}", proof); + break; + } + log::info!("POST proof generation in progress"); + } + spacemesh_v1::GenProofStatus::Unspecified => { + log::error!("unspecified status"); + } + spacemesh_v1::GenProofStatus::Error => { + log::error!("POST proof generation error"); + break; + } + } + } + _ => { + log::error!("Got unexpected response: {:?}", resp); + } + } + sleep(Duration::from_secs(5)).await; + } + } + + // _ = handle.await?; + // Ok(()) +}