From dba39047ae8faca16f62606ed5cb8f8a75b993b9 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Fri, 7 Oct 2022 05:51:40 +0800 Subject: [PATCH] fix major bugs in p2p PR --- .gitignore | 1 - Cargo.lock | 122 +++++++++--------- core/src/lib.rs | 8 +- crates/p2p/Cargo.toml | 4 +- crates/p2p/examples/basic.rs | 1 + crates/p2p/src/discovery/global_discovery.rs | 2 +- crates/p2p/src/discovery/mdns.rs | 2 +- crates/p2p/src/lib.rs | 2 +- crates/p2p/src/network_manager/nm.rs | 2 +- crates/p2p/src/network_manager/nm_config.rs | 2 +- crates/p2p/src/network_manager/nm_internal.rs | 2 +- crates/p2p/src/network_manager/nm_server.rs | 2 +- crates/p2p/src/p2p_manager.rs | 2 +- crates/p2p/src/peer/peer.rs | 2 +- crates/p2p/src/peer/peer_candidate.rs | 5 +- crates/p2p/src/peer/peer_metadata.rs | 2 +- crates/p2p/tunnel/Cargo.toml | 6 +- crates/p2p/tunnel/fly.toml | 2 +- crates/p2p/tunnel/src/main.rs | 50 +++---- crates/p2p/tunnel/utils/Cargo.toml | 2 +- 20 files changed, 110 insertions(+), 111 deletions(-) diff --git a/.gitignore b/.gitignore index fc00b6ad0..1a7673277 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,6 @@ apps/*/stats.html docs/public/*.st docs/public/*.toml dev.db -stats.html !cli/cmd/turbo cli/npm/turbo-android-arm64/bin diff --git a/Cargo.lock b/Cargo.lock index 212afe86f..3724a84a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3396,30 +3396,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "p2p" -version = "0.1.0" -dependencies = [ - "bip39", - "ctrlc", - "dashmap", - "futures-util", - "if-watch", - "mdns-sd", - "quinn", - "rcgen", - "rmp-serde", - "rustls", - "serde", - "spake2", - "specta 0.0.2", - "thiserror", - "tokio", - "tracing", - "ts-rs", - "tunnel-utils", -] - [[package]] name = "pango" version = "0.15.10" @@ -4987,6 +4963,30 @@ dependencies = [ "webp", ] +[[package]] +name = "sd-p2p" +version = "0.1.0" +dependencies = [ + "bip39", + "ctrlc", + "dashmap", + "futures-util", + "if-watch", + "mdns-sd", + "quinn", + "rcgen", + "rmp-serde", + "rustls", + "sd-tunnel-utils", + "serde", + "spake2", + "specta 0.0.2", + "thiserror", + "tokio", + "tracing", + "ts-rs", +] + [[package]] name = "sd-sync" version = "0.1.0" @@ -4998,6 +4998,43 @@ dependencies = [ "uuid 1.1.2", ] +[[package]] +name = "sd-tunnel" +version = "0.1.0" +dependencies = [ + "base64 0.13.0", + "bb8-redis", + "dotenv", + "futures", + "metrics 0.19.0", + "metrics-exporter-prometheus", + "quinn", + "rcgen", + "rmp-serde", + "rustls", + "sd-tunnel-utils", + "serde", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "sd-tunnel-utils" +version = "0.1.0" +dependencies = [ + "quinn", + "ring 0.16.20", + "rmp", + "rmp-serde", + "rustls", + "serde", + "specta 0.0.2", + "thiserror", + "ts-rs", +] + [[package]] name = "security-framework" version = "2.7.0" @@ -6434,43 +6471,6 @@ dependencies = [ "utf-8", ] -[[package]] -name = "tunnel" -version = "0.1.0" -dependencies = [ - "base64 0.13.0", - "bb8-redis", - "dotenv", - "futures", - "metrics 0.19.0", - "metrics-exporter-prometheus", - "quinn", - "rcgen", - "rmp-serde", - "rustls", - "serde", - "thiserror", - "tokio", - "tracing", - "tracing-subscriber", - "tunnel-utils", -] - -[[package]] -name = "tunnel-utils" -version = "0.1.0" -dependencies = [ - "quinn", - "ring 0.16.20", - "rmp", - "rmp-serde", - "rustls", - "serde", - "specta 0.0.2", - "thiserror", - "ts-rs", -] - [[package]] name = "typenum" version = "1.15.0" diff --git a/core/src/lib.rs b/core/src/lib.rs index 1efe8bc30..fae1da894 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -53,10 +53,10 @@ impl Node { // dbg!(get_object_kind_from_extension("png")); - let (non_blocking, _guard) = tracing_appender::non_blocking(rolling::daily( - Path::new(&data_dir).join("logs"), - "log", - )); + // let (non_blocking, _guard) = tracing_appender::non_blocking(rolling::daily( + // Path::new(&data_dir).join("logs"), + // "log", + // )); // TODO: Make logs automatically delete after x time https://github.com/tokio-rs/tracing/pull/2169 tracing_subscriber::registry() diff --git a/crates/p2p/Cargo.toml b/crates/p2p/Cargo.toml index 1f064147d..d6df53c99 100644 --- a/crates/p2p/Cargo.toml +++ b/crates/p2p/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "p2p" +name = "sd-p2p" version = "0.1.0" edition = "2021" [dependencies] -tunnel-utils = { path = "./tunnel/utils" } +sd-tunnel-utils = { path = "./tunnel/utils" } dashmap = "5.3.4" rcgen = "0.9.2" diff --git a/crates/p2p/examples/basic.rs b/crates/p2p/examples/basic.rs index e5a2df5ba..5f3f0e4db 100644 --- a/crates/p2p/examples/basic.rs +++ b/crates/p2p/examples/basic.rs @@ -28,6 +28,7 @@ impl P2PManager for SdP2PManager { PeerMetadata { name: self.peer_name.clone(), version: Some(env!("CARGO_PKG_VERSION").into()), + operating_system: todo!(), } } diff --git a/crates/p2p/src/discovery/global_discovery.rs b/crates/p2p/src/discovery/global_discovery.rs index 5dc722132..8dd48f6f8 100644 --- a/crates/p2p/src/discovery/global_discovery.rs +++ b/crates/p2p/src/discovery/global_discovery.rs @@ -1,8 +1,8 @@ /// The functions in this file are predominantly useless in the current system. This will be fixed in a future PR's. use std::sync::Arc; +use sd_tunnel_utils::{Client, Message}; use tracing::warn; -use tunnel_utils::{Client, Message}; use crate::{NetworkManager, NetworkManagerError, P2PManager}; diff --git a/crates/p2p/src/discovery/mdns.rs b/crates/p2p/src/discovery/mdns.rs index 0c72a1256..17910f316 100644 --- a/crates/p2p/src/discovery/mdns.rs +++ b/crates/p2p/src/discovery/mdns.rs @@ -1,8 +1,8 @@ use std::{net::Ipv4Addr, sync::Arc}; use mdns_sd::{Receiver, ServiceDaemon, ServiceEvent, ServiceInfo}; +use sd_tunnel_utils::PeerId; use tracing::warn; -use tunnel_utils::PeerId; use crate::{NetworkManager, NetworkManagerError, P2PManager, PeerCandidate, PeerMetadata}; diff --git a/crates/p2p/src/lib.rs b/crates/p2p/src/lib.rs index 21820a660..ad147eae5 100644 --- a/crates/p2p/src/lib.rs +++ b/crates/p2p/src/lib.rs @@ -8,7 +8,7 @@ pub(crate) use discovery::*; pub use network_manager::*; pub use p2p_manager::*; pub use peer::*; -pub use tunnel_utils::{read_value, write_value, PeerId}; +pub use sd_tunnel_utils::{read_value, write_value, PeerId}; pub use utils::*; /// We reexport some types from `quinn` to avoid the user needing to add `quinn` and keep its version in sync with the p2p library. diff --git a/crates/p2p/src/network_manager/nm.rs b/crates/p2p/src/network_manager/nm.rs index 9d3e84fe8..4b57a2a2e 100644 --- a/crates/p2p/src/network_manager/nm.rs +++ b/crates/p2p/src/network_manager/nm.rs @@ -10,11 +10,11 @@ use dashmap::{DashMap, DashSet}; use futures_util::future::join_all; use quinn::{Chunk, Endpoint, NewConnection, RecvStream, SendStream, ServerConfig}; use rustls::{Certificate, PrivateKey}; +use sd_tunnel_utils::{quic, write_value, PeerId, UtilError}; use spake2::{Ed25519Group, Password, Spake2}; use thiserror::Error; use tokio::sync::{mpsc, oneshot}; use tracing::{debug, error, warn}; -use tunnel_utils::{quic, write_value, PeerId, UtilError}; use crate::{ ConnectError, ConnectionEstablishmentPayload, ConnectionType, Identity, NetworkManagerConfig, diff --git a/crates/p2p/src/network_manager/nm_config.rs b/crates/p2p/src/network_manager/nm_config.rs index 775ed2be0..c7dea7555 100644 --- a/crates/p2p/src/network_manager/nm_config.rs +++ b/crates/p2p/src/network_manager/nm_config.rs @@ -1,6 +1,6 @@ use std::collections::HashSet; -use tunnel_utils::PeerId; +use sd_tunnel_utils::PeerId; /// Stores configuration which is given to the [crate::NetworkManager] at startup so it can resume from it's previous state. #[derive(Clone)] diff --git a/crates/p2p/src/network_manager/nm_internal.rs b/crates/p2p/src/network_manager/nm_internal.rs index 5276f1072..fba79b12e 100644 --- a/crates/p2p/src/network_manager/nm_internal.rs +++ b/crates/p2p/src/network_manager/nm_internal.rs @@ -7,10 +7,10 @@ use std::{ use futures_util::StreamExt; use if_watch::{IfEvent, IfWatcher}; use quinn::{ClientConfig, Incoming, NewConnection, VarInt}; +use sd_tunnel_utils::{quic::client_config, PeerId}; use thiserror::Error; use tokio::{select, sync::mpsc, time::sleep}; use tracing::{debug, error, warn}; -use tunnel_utils::{quic::client_config, PeerId}; use crate::{ ConnectionType, DiscoveryStack, NetworkManager, NetworkManagerError, P2PManager, Peer, diff --git a/crates/p2p/src/network_manager/nm_server.rs b/crates/p2p/src/network_manager/nm_server.rs index aba1f5904..e543a8312 100644 --- a/crates/p2p/src/network_manager/nm_server.rs +++ b/crates/p2p/src/network_manager/nm_server.rs @@ -3,10 +3,10 @@ use std::{sync::Arc, time::Duration}; use futures_util::StreamExt; use quinn::{Connecting, NewConnection, VarInt}; use rustls::Certificate; +use sd_tunnel_utils::{read_value, write_value, PeerId}; use spake2::{Ed25519Group, Password, Spake2}; use tokio::{sync::oneshot, time::sleep}; use tracing::{debug, error, info, warn}; -use tunnel_utils::{read_value, write_value, PeerId}; use crate::{ ConnectionEstablishmentPayload, ConnectionType, NetworkManager, P2PManager, diff --git a/crates/p2p/src/p2p_manager.rs b/crates/p2p/src/p2p_manager.rs index 144ccd74e..a25a0252f 100644 --- a/crates/p2p/src/p2p_manager.rs +++ b/crates/p2p/src/p2p_manager.rs @@ -1,8 +1,8 @@ use std::{collections::HashMap, future::Future, pin::Pin}; use quinn::{RecvStream, SendStream}; +use sd_tunnel_utils::PeerId; use tokio::sync::oneshot; -use tunnel_utils::PeerId; use crate::{NetworkManager, Peer, PeerMetadata}; diff --git a/crates/p2p/src/peer/peer.rs b/crates/p2p/src/peer/peer.rs index 5ea9830cb..9f688e5aa 100644 --- a/crates/p2p/src/peer/peer.rs +++ b/crates/p2p/src/peer/peer.rs @@ -5,8 +5,8 @@ use std::{ use futures_util::StreamExt; use quinn::{ApplicationClose, Connection, IncomingBiStreams}; +use sd_tunnel_utils::PeerId; use tracing::{debug, error}; -use tunnel_utils::PeerId; use crate::{NetworkManager, P2PManager, PeerMetadata}; diff --git a/crates/p2p/src/peer/peer_candidate.rs b/crates/p2p/src/peer/peer_candidate.rs index b67ee3c6f..2ba387427 100644 --- a/crates/p2p/src/peer/peer_candidate.rs +++ b/crates/p2p/src/peer/peer_candidate.rs @@ -1,8 +1,7 @@ use std::net::Ipv4Addr; +use sd_tunnel_utils::PeerId; use serde::{Deserialize, Serialize}; -use specta::Type; -use tunnel_utils::PeerId; use crate::PeerMetadata; @@ -10,7 +9,7 @@ use crate::PeerMetadata; /// It is called a candidate as it contains all of the information required to connection and pair with the peer. /// A peer candidate discovered through mDNS may have been modified by an attacker on your local network but this is deemed acceptable as the attacker can only modify primitive metadata such a name or Spacedrive version which is used for pairing. /// When we initiated communication with the device we will ensure we are talking to the correct device using PAKE (specially SPAKE2) for pairing and verifying the TLS certificate for general communication. -#[derive(Debug, Clone, Type, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] // TODO: Type pub struct PeerCandidate { pub id: PeerId, pub metadata: PeerMetadata, diff --git a/crates/p2p/src/peer/peer_metadata.rs b/crates/p2p/src/peer/peer_metadata.rs index 6e3fe3abb..b8f909585 100644 --- a/crates/p2p/src/peer/peer_metadata.rs +++ b/crates/p2p/src/peer/peer_metadata.rs @@ -1,8 +1,8 @@ use std::{collections::HashMap, env, str::FromStr}; +use sd_tunnel_utils::PeerId; use serde::{Deserialize, Serialize}; use specta::Type; -use tunnel_utils::PeerId; /// Represents the operating system which the remote peer is running. /// This is not used internally and predominantly is designed to be used for display purposes by the embedding application. diff --git a/crates/p2p/tunnel/Cargo.toml b/crates/p2p/tunnel/Cargo.toml index 28e81c66e..cdbe7860e 100644 --- a/crates/p2p/tunnel/Cargo.toml +++ b/crates/p2p/tunnel/Cargo.toml @@ -1,11 +1,11 @@ [package] -name = "tunnel" +name = "sd-tunnel" version = "0.1.0" edition = "2021" -default-run = "tunnel" +default-run = "sd-tunnel" [dependencies] -tunnel-utils = { path = "./utils" } +sd-tunnel-utils = { path = "./utils" } base64 = "0.13.0" dotenv = "0.15.0" diff --git a/crates/p2p/tunnel/fly.toml b/crates/p2p/tunnel/fly.toml index b48236dcb..b696664a0 100644 --- a/crates/p2p/tunnel/fly.toml +++ b/crates/p2p/tunnel/fly.toml @@ -1,4 +1,4 @@ -app = "sdtunnel" +app = "sd-tunnel" [env] SD_PORT = 9000 diff --git a/crates/p2p/tunnel/src/main.rs b/crates/p2p/tunnel/src/main.rs index 163297373..63de3c82f 100644 --- a/crates/p2p/tunnel/src/main.rs +++ b/crates/p2p/tunnel/src/main.rs @@ -42,37 +42,37 @@ async fn main() { .init(); let certificate = match env::var("SD_ROOT_CERTIFICATE") { - Ok(certificate) => rustls::Certificate( - decode(certificate).expect("Error decoding 'SD_ROOT_CERTIFICATE'"), - ), + Ok(certificate) => { + rustls::Certificate(decode(certificate).expect("Error decoding 'SD_ROOT_CERTIFICATE'")) + } Err(_) => { error!("Error: 'SD_ROOT_CERTIFICATE' env var is not set!"); return; - }, + } }; let priv_key = match env::var("SD_ROOT_CERTIFICATE_KEY") { - Ok(key) => rustls::PrivateKey( - decode(key).expect("Error decoding 'SD_ROOT_CERTIFICATE_KEY'"), - ), + Ok(key) => { + rustls::PrivateKey(decode(key).expect("Error decoding 'SD_ROOT_CERTIFICATE_KEY'")) + } Err(_) => { error!("Error: 'SD_ROOT_CERTIFICATE_KEY' env var is not set!"); return; - }, + } }; let redis_url = match env::var("SD_REDIS_URL") { Ok(redis_url) => redis_url, Err(_) => { error!("Error: 'SD_REDIS_URL' env var is not set!"); return; - }, + } }; let server_port = env::var("SD_PORT") .map(|port| port.parse::().unwrap_or(9000)) .unwrap_or(9000); let bind_addr = env::var("SD_BIND_ADDR").unwrap_or(Ipv4Addr::UNSPECIFIED.to_string()); - let manager = RedisConnectionManager::new(redis_url) - .expect("Error creating Redis connection manager!"); + let manager = + RedisConnectionManager::new(redis_url).expect("Error creating Redis connection manager!"); let redis_pool = Pool::builder() .build(manager) .await @@ -141,12 +141,12 @@ async fn handle_connection( error!("Error: peer has multiple client certificates!"); increment_counter!("spacetunnel_connections_invalid"); return Ok(()); - }, + } Err(_) => { error!("Error: peer did not provide a client certificates!"); increment_counter!("spacetunnel_connections_invalid"); return Ok(()); - }, + } }; info!( "established connection with peer '{}' from addr '{}'", @@ -164,9 +164,12 @@ async fn handle_connection( error_code, reason, })) => { - debug!("closed connection with peer '{}' with error_code '{}' and reason '{:?}' ", peer_id, error_code, reason); + debug!( + "closed connection with peer '{}' with error_code '{}' and reason '{:?}' ", + peer_id, error_code, reason + ); return Ok(()); - }, + } Err(e) => return Err(e.into()), Ok(s) => s, }; @@ -189,11 +192,11 @@ async fn handle_connection( match Message::Error(MessageError::InternalServerErr).encode() { Ok(msg) => { let _ = tx.write_all(&msg).await; - }, + } Err(e) => { error!("Error encoding error error message: {}", e.to_string()); increment_counter!("spacetunnel_stream_errored"); - }, + } } } else { debug!("closed stream from peer '{}'", peer_id); @@ -215,7 +218,7 @@ async fn handle_stream( error!("Error getting Redis connection: {}", err); increment_counter!("spacetunnel_redis_error", "error_src" => "get"); return Ok(()); - }, + } }; while let Some(chunk) = recv.read_chunk(MAX_MESSAGE_SIZE, true).await? { @@ -241,7 +244,7 @@ async fn handle_stream( Message::ClientAnnouncementOk } - }, + } Message::QueryClientAnnouncement(peer_ids) => { increment_counter!("spacetunnel_discovery_announcement_queries"); @@ -253,15 +256,12 @@ async fn handle_stream( "Client requested too many client announcements '{}'", peer_ids.len() ); - increment_counter!( - "spacetunnel_discovery_announcement_queries_invalid" - ); + increment_counter!("spacetunnel_discovery_announcement_queries_invalid"); Message::Error(MessageError::InvalidReqErr) } else { let mut peers = Vec::with_capacity(peer_ids.len()); for peer_id in peer_ids.iter() { - let redis_key = - format!("peer:announcement:{}", peer_id.to_string()); + let redis_key = format!("peer:announcement:{}", peer_id.to_string()); let resp: HashMap = cmd("HGETALL") .arg(&redis_key) @@ -280,7 +280,7 @@ async fn handle_stream( } Message::QueryClientAnnouncementResponse(peers) } - }, + } Message::ClientAnnouncementOk | Message::QueryClientAnnouncementResponse { .. } | Message::Error(_) => Message::Error(MessageError::InvalidReqErr), diff --git a/crates/p2p/tunnel/utils/Cargo.toml b/crates/p2p/tunnel/utils/Cargo.toml index 89faa0252..0800d7974 100644 --- a/crates/p2p/tunnel/utils/Cargo.toml +++ b/crates/p2p/tunnel/utils/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "tunnel-utils" +name = "sd-tunnel-utils" version = "0.1.0" edition = "2021"