[ENG-674] fix invalidation (#891)

* reenable the invalidate requests system

* a tester for the invalidate system

* watch locations in Library::load

---------

Co-authored-by: Brendan Allan <brendonovich@outlook.com>

apps/server/Cargo.toml

@@ -9,7 +9,7 @@ edition.workspace = true
 assets = []
 
 [dependencies]
-sd-core = { path = "../../core", features = ["ffmpeg"] }
+sd-core = { path = "../../core", features = ["ffmpeg", "location-watcher"] }
 rspc = { workspace = true, features = ["axum"] }
 httpz = { workspace = true, features = ["axum"] }
 axum = "0.6.18"

apps/web/src/demoData.json

@@ -173,10 +173,10 @@
 				"fetchStatus": "fetching"
 			},
 			"queryKey": [
-				"library.getStatistics",
+				"library.statistics",
 				{ "library_id": "dc1c41b9-65c6-4a9d-a418-79e628b3ac59", "arg": null }
 			],
-			"queryHash": "[\"library.getStatistics\",{\"arg\":null,\"library_id\":\"dc1c41b9-65c6-4a9d-a418-79e628b3ac59\"}]"
+			"queryHash": "[\"library.statistics\",{\"arg\":null,\"library_id\":\"dc1c41b9-65c6-4a9d-a418-79e628b3ac59\"}]"
 		}
 	]
 }

core/src/api/libraries.rs

@@ -110,7 +110,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 						.get_library(new_library.uuid)
 						.await
 						.expect("We just created the library. Where do it be?"),
-					"library.getStatistics"
+					"library.statistics"
 				);
 
 				Ok(new_library)

core/src/api/utils/invalidate.rs

@@ -119,22 +119,22 @@ macro_rules! invalidate_query {
 	($ctx:expr, $key:literal) => {{
 		let ctx: &crate::library::Library = &$ctx; // Assert the context is the correct type
 
-		// #[cfg(debug_assertions)]
-		// {
-		// 	#[ctor::ctor]
-		// 	fn invalidate() {
-		// 		crate::api::utils::INVALIDATION_REQUESTS
-		// 			.lock()
-		// 			.unwrap()
-		// 			.queries
-		// 			.push(crate::api::utils::InvalidationRequest {
-		// 				key: $key,
-		// 				arg_ty: None,
-		// 				result_ty: None,
-  //           			macro_src: concat!(file!(), ":", line!()),
-		// 			})
-		// 	}
-		// }
+		#[cfg(debug_assertions)]
+		{
+			#[ctor::ctor]
+			fn invalidate() {
+				crate::api::utils::INVALIDATION_REQUESTS
+					.lock()
+					.unwrap()
+					.queries
+					.push(crate::api::utils::InvalidationRequest {
+						key: $key,
+						arg_ty: None,
+						result_ty: None,
+            			macro_src: concat!(file!(), ":", line!()),
+					})
+			}
+		}
 
 		// The error are ignored here because they aren't mission critical. If they fail the UI might be outdated for a bit.
 		ctx.emit(crate::api::CoreEvent::InvalidateOperation(
@@ -145,25 +145,25 @@ macro_rules! invalidate_query {
 		let _: $arg_ty = $arg; // Assert the type the user provided is correct
 		let ctx: &crate::library::Library = &$ctx; // Assert the context is the correct type
 
-		// #[cfg(debug_assertions)]
-		// {
-		// 	#[ctor::ctor]
-		// 	fn invalidate() {
-		// 		crate::api::utils::INVALIDATION_REQUESTS
-		// 			.lock()
-		// 			.unwrap()
-		// 			.queries
-		// 			.push(crate::api::utils::InvalidationRequest {
-		// 				key: $key,
-		// 				arg_ty: Some(<$arg_ty as rspc::internal::specta::Type>::reference(rspc::internal::specta::DefOpts {
-  //                           parent_inline: false,
-  //                           type_map: &mut rspc::internal::specta::TypeDefs::new(),
-  //                       }, &[])),
-		// 				result_ty: None,
-  //                       macro_src: concat!(file!(), ":", line!()),
-		// 			})
-		// 	}
-		// }
+		#[cfg(debug_assertions)]
+		{
+			#[ctor::ctor]
+			fn invalidate() {
+				crate::api::utils::INVALIDATION_REQUESTS
+					.lock()
+					.unwrap()
+					.queries
+					.push(crate::api::utils::InvalidationRequest {
+						key: $key,
+						arg_ty: Some(<$arg_ty as rspc::internal::specta::Type>::reference(rspc::internal::specta::DefOpts {
+                            parent_inline: false,
+                            type_map: &mut rspc::internal::specta::TypeDefs::new(),
+                        }, &[])),
+						result_ty: None,
+                        macro_src: concat!(file!(), ":", line!()),
+					})
+			}
+		}
 
 		// The error are ignored here because they aren't mission critical. If they fail the UI might be outdated for a bit.
 		let _ = serde_json::to_value($arg)
@@ -224,7 +224,27 @@ pub(crate) fn mount_invalidate() -> AlphaRouter<Ctx> {
 	let manager_thread_active = Arc::new(AtomicBool::new(false));
 
 	// TODO: Scope the invalidate queries to a specific library (filtered server side)
-	R.router().procedure("listen", {
+	let mut r = R.router();
+
+	#[cfg(debug_assertions)]
+	{
+		let count = Arc::new(std::sync::atomic::AtomicU16::new(0));
+
+		r = r
+			.procedure(
+				"test-invalidate",
+				R.query(move |_, _: ()| count.fetch_add(1, Ordering::SeqCst)),
+			)
+			.procedure(
+				"test-invalidate-mutation",
+				R.with2(super::library()).mutation(|(_, library), _: ()| {
+					invalidate_query!(library, "invalidation.test-invalidate");
+					Ok(())
+				}),
+			);
+	}
+
+	r.procedure("listen", {
 		R.subscription(move |ctx, _: ()| {
 			// This thread is used to deal with batching and deduplication.
 			// Their is only ever one of these management threads per Node but we spawn it like this so we can steal the event bus from the rspc context.

core/src/library/manager.rs

@@ -1,8 +1,9 @@
 use crate::{
 	invalidate_query,
+	location::LocationManagerError,
 	node::Platform,
 	object::orphan_remover::OrphanRemoverActor,
-	prisma::{node, PrismaClient},
+	prisma::{location, node, PrismaClient},
 	sync::{SyncManager, SyncMessage},
 	util::{
 		db::{load_and_migrate, MigrationError},
@@ -68,6 +69,8 @@ pub enum LibraryManagerError {
 	InvalidConfig(String),
 	#[error(transparent)]
 	NonUtf8Path(#[from] NonUtf8PathError),
+	#[error("failed to watch locations: {0}")]
+	LocationWatcher(#[from] LocationManagerError),
 }
 
 impl From<LibraryManagerError> for rspc::Error {
@@ -425,6 +428,20 @@ impl LibraryManager {
 			node_context,
 		};
 
+		for location in library
+			.db
+			.location()
+			.find_many(vec![location::node_id::equals(node_data.id)])
+			.exec()
+			.await?
+		{
+			library
+				.node_context
+				.location_manager
+				.add(location.id, library.clone())
+				.await?;
+		}
+
 		if let Err(e) = library
 			.node_context
 			.jobs

crates/p2p/examples/basic.rs

@@ -119,7 +119,7 @@ async fn main() {
 				sleep(Duration::from_secs(3)).await;
 				manager
 					.broadcast(
-						format!("Hello World From {}", keypair.public().to_peer_id())
+						format!("Hello World From {}", keypair.peer_id())
 							.as_bytes()
 							.to_vec(),
 					)

interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx

@@ -1,4 +1,4 @@
-import { getDebugState, useBridgeQuery, useDebugState } from '@sd/client';
+import { getDebugState, useBridgeQuery, useDebugState, useLibraryMutation } from '@sd/client';
 import { Button, Popover, Select, SelectOption, Switch } from '@sd/ui';
 import { usePlatform } from '~/util/Platform';
 import Setting from '../../settings/Setting';
@@ -97,6 +97,7 @@ export default () => {
 						<SelectOption value="enabled">Enabled</SelectOption>
 					</Select>
 				</Setting>
+				<InvalidateDebugPanel />
 
 				{/* {platform.showDevtools && (
 					<SettingContainer
@@ -115,3 +116,22 @@ export default () => {
 		</Popover>
 	);
 };
+
+function InvalidateDebugPanel() {
+	const { data: count } = useBridgeQuery(['invalidation.test-invalidate']);
+	const { mutate } = useLibraryMutation(['invalidation.test-invalidate-mutation']);
+
+	return (
+		<Setting
+			mini
+			title="Invalidate Debug Panel"
+			description={`Pressing the button issues an invalidate to the query rendering this number: ${count}`}
+		>
+			<div className="mt-2">
+				<Button size="sm" variant="gray" onClick={() => mutate(null)}>
+					Invalidate
+				</Button>
+			</div>
+		</Setting>
+	);
+}

packages/client/src/core.ts

@@ -6,6 +6,7 @@ export type Procedures = {
         { key: "buildInfo", input: never, result: BuildInfo } | 
         { key: "categories.list", input: LibraryArgs<null>, result: { [key in Category]: number } } | 
         { key: "files.get", input: LibraryArgs<GetArgs>, result: { id: number; pub_id: number[]; kind: number; key_id: number | null; hidden: boolean; favorite: boolean; important: boolean; has_thumbnail: boolean; has_thumbstrip: boolean; has_video_preview: boolean; ipfs_id: string | null; note: string | null; date_created: string; date_accessed: string | null; file_paths: FilePath[]; media_data: MediaData | null } | null } | 
+        { key: "invalidation.test-invalidate", input: never, result: number } | 
         { key: "jobs.getHistory", input: LibraryArgs<null>, result: JobReport[] } | 
         { key: "jobs.getRunning", input: LibraryArgs<null>, result: JobReport[] } | 
         { key: "keys.getDefault", input: LibraryArgs<null>, result: string | null } | 
@@ -46,6 +47,7 @@ export type Procedures = {
         { key: "files.setFavorite", input: LibraryArgs<SetFavoriteArgs>, result: null } | 
         { key: "files.setNote", input: LibraryArgs<SetNoteArgs>, result: null } | 
         { key: "files.updateAccessTime", input: LibraryArgs<number>, result: null } | 
+        { key: "invalidation.test-invalidate-mutation", input: LibraryArgs<null>, result: null } | 
         { key: "jobs.clear", input: LibraryArgs<string>, result: null } | 
         { key: "jobs.clearAll", input: LibraryArgs<null>, result: null } | 
         { key: "jobs.generateThumbsForLocation", input: LibraryArgs<GenerateThumbsForLocationArgs>, result: null } |