[ENG-1095] HEIF* and PDF thumbnail orientation (#1815)

* fix horizontal pdf rendering

* don't rotate certain formats (heif*)

* return `u32` from scaling function

* add `avifs` as a valid extension

* add avifs as a valid MIME type

* re-categorize `avifs` as a video (sequence)

* Fix mime-types

* Add Hif extension
 - Cannon uses a special extension for heif in its cameras: https://github.com/digipres/digipres.github.io/blob/master/_sources/registries/fdd/fddXML/fdd000525.xml#L213-L221

* hif is actually a heic

---------

Co-authored-by: Vítor Vasconcellos <vasconcellos.dev@gmail.com>

core/src/custom_uri/mod.rs

@@ -423,11 +423,22 @@ async fn infer_the_mime_type(
 		"webp" => "image/webp",
 		// PDF document
 		"pdf" => "application/pdf",
-		// HEIF/HEIC images
-		"heif" | "heifs" => "image/heif,image/heif-sequence",
-		"heic" | "heics" => "image/heic,image/heic-sequence",
-		// AVIF images
-		"avif" | "avci" | "avcs" => "image/avif",
+		// HEIF images
+		"heif" => "image/heif",
+		// HEIF images sequence (animated)
+		"heifs" => "image/heif-sequence",
+		// HEIC images
+		"heic" | "hif" => "image/heic",
+		// HEIC images sequence (animated)
+		"heics" => "image/heic-sequence",
+		// AV1 in HEIF images
+		"avif" => "image/avif",
+		// AV1 in HEIF images sequence (DEPRECATED: https://github.com/AOMediaCodec/av1-avif/pull/86/files)
+		"avifs" => "image/avif-sequence",
+		// AVC in HEIF images
+		"avci" => "image/avci",
+		// AVC in HEIF images sequence (animated)
+		"avcs" => "image/avcs",
 		_ => "text/plain",
 	};
 

core/src/object/media/thumbnail/process.rs

@@ -1,7 +1,7 @@
 use crate::{api::CoreEvent, util::error::FileIOError};
 
 use sd_file_ext::extensions::{DocumentExtension, ImageExtension};
-use sd_images::{format_image, scale_dimensions};
+use sd_images::{format_image, scale_dimensions, ConvertableExtension};
 use sd_media_metadata::image::Orientation;
 use sd_prisma::prisma::location;
 
@@ -397,7 +397,7 @@ async fn generate_image_thumbnail(
 	let file_path = file_path.as_ref().to_path_buf();
 
 	let webp = spawn_blocking(move || -> Result<_, ThumbnailerError> {
-		let img = format_image(&file_path).map_err(|e| ThumbnailerError::SdImages {
+		let mut img = format_image(&file_path).map_err(|e| ThumbnailerError::SdImages {
 			path: file_path.clone().into_boxed_path(),
 			error: e,
 		})?;
@@ -406,17 +406,24 @@ async fn generate_image_thumbnail(
 		let (w_scaled, h_scaled) = scale_dimensions(w as f32, h as f32, TARGET_PX);
 
 		// Optionally, resize the existing photo and convert back into DynamicImage
-		let mut img = DynamicImage::ImageRgba8(imageops::resize(
-			&img,
-			w_scaled as u32,
-			h_scaled as u32,
-			imageops::FilterType::Triangle,
-		));
+		if w != w_scaled && h != h_scaled {
+			img = DynamicImage::ImageRgba8(imageops::resize(
+				&img,
+				w_scaled,
+				h_scaled,
+				imageops::FilterType::Triangle,
+			));
+		}
 
 		// this corrects the rotation/flip of the image based on the *available* exif data
-		// not all images have exif data, so we don't error
+		// not all images have exif data, so we don't error. we also don't rotate HEIF as that's against the spec
 		if let Some(orientation) = Orientation::from_path(&file_path) {
-			img = orientation.correct_thumbnail(img);
+			if ConvertableExtension::try_from(file_path.as_ref())
+				.expect("we already checked if the image was convertable")
+				.should_rotate()
+			{
+				img = orientation.correct_thumbnail(img);
+			}
 		}
 
 		// Create the WebP encoder for the above image

crates/file-ext/src/extensions.rs

@@ -32,6 +32,7 @@ extension_enum! {
 extension_category_enum! {
 	VideoExtension ALL_VIDEO_EXTENSIONS {
 		Avi = [0x52, 0x49, 0x46, 0x46, _, _, _, _, 0x41, 0x56, 0x49, 0x20],
+		Avifs = [],
 		Qt = [0x71, 0x74, 0x20, 0x20],
 		Mov = [0x66, 0x74, 0x79, 0x70, 0x71, 0x74, 0x20, 0x20] + 4,
 		Swf = [0x5A, 0x57, 0x53] | [0x46, 0x57, 0x53],

crates/images/src/consts.rs

@@ -1,4 +1,4 @@
-use std::fmt::Display;
+use std::{ffi::OsStr, fmt::Display, path::Path};
 
 /// The size of 1MiB in bytes
 const MIB: u64 = 1_048_576;
@@ -18,7 +18,9 @@ pub const GENERIC_EXTENSIONS: [&str; 17] = [
 pub const SVG_EXTENSIONS: [&str; 2] = ["svg", "svgz"];
 pub const PDF_EXTENSIONS: [&str; 1] = ["pdf"];
 #[cfg(feature = "heif")]
-pub const HEIF_EXTENSIONS: [&str; 7] = ["heif", "heifs", "heic", "heics", "avif", "avci", "avcs"];
+pub const HEIF_EXTENSIONS: [&str; 8] = [
+	"hif", "heif", "heifs", "heic", "heics", "avif", "avci", "avcs",
+];
 
 // Will be needed for validating HEIF images
 // #[cfg(feature = "heif")]
@@ -32,9 +34,10 @@ pub const SVG_TARGET_PX: f32 = 262_144_f32;
 
 /// The size that PDF pages are rendered at.
 ///
-/// This is 120 DPI at standard A4 printer paper size - the target aspect
+/// This is 96DPI at standard A4 printer paper size - the target aspect
 /// ratio and height are maintained.
-pub const PDF_RENDER_WIDTH: pdfium_render::prelude::Pixels = 992;
+pub const PDF_PORTRAIT_RENDER_WIDTH: pdfium_render::prelude::Pixels = 794;
+pub const PDF_LANDSCAPE_RENDER_WIDTH: pdfium_render::prelude::Pixels = 1123;
 
 #[cfg_attr(feature = "specta", derive(specta::Type))]
 #[cfg_attr(feature = "bincode", derive(bincode::Encode, bincode::Decode))]
@@ -57,6 +60,7 @@ pub enum ConvertableExtension {
 	Vst,
 	Tiff,
 	Tif,
+	Hif,
 	Heif,
 	Heifs,
 	Heic,
@@ -70,6 +74,20 @@ pub enum ConvertableExtension {
 	Webp,
 }
 
+impl ConvertableExtension {
+	#[must_use]
+	pub const fn should_rotate(self) -> bool {
+		!matches!(
+			self,
+			Self::Hif
+				| Self::Heif | Self::Heifs
+				| Self::Heic | Self::Heics
+				| Self::Avif | Self::Avci
+				| Self::Avcs
+		)
+	}
+}
+
 impl Display for ConvertableExtension {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		write!(f, "{self:?}")
@@ -98,6 +116,7 @@ impl TryFrom<String> for ConvertableExtension {
 			"vst" => Ok(Self::Vst),
 			"tiff" => Ok(Self::Tiff),
 			"tif" => Ok(Self::Tif),
+			"hif" => Ok(Self::Hif),
 			"heif" => Ok(Self::Heif),
 			"heifs" => Ok(Self::Heifs),
 			"heic" => Ok(Self::Heic),
@@ -114,6 +133,18 @@ impl TryFrom<String> for ConvertableExtension {
 	}
 }
 
+impl TryFrom<&Path> for ConvertableExtension {
+	type Error = crate::Error;
+
+	fn try_from(value: &Path) -> Result<Self, Self::Error> {
+		value
+			.extension()
+			.and_then(OsStr::to_str)
+			.map(str::to_string)
+			.map_or_else(|| Err(crate::Error::Unsupported), Self::try_from)
+	}
+}
+
 #[cfg(feature = "serde")]
 impl serde::Serialize for ConvertableExtension {
 	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

crates/images/src/lib.rs

@@ -45,12 +45,12 @@ pub trait ImageHandler {
 	where
 		Self: Sized,
 	{
-		self.validate_image(path)?;
+		self.validate_size(path)?;
 
 		fs::read(path).map_err(|e| Error::Io(e, path.to_path_buf().into_boxed_path()))
 	}
 
-	fn validate_image(&self, path: &Path) -> Result<()>
+	fn validate_size(&self, path: &Path) -> Result<()>
 	where
 		Self: Sized,
 	{
@@ -86,7 +86,7 @@ pub trait ImageHandler {
 	clippy::cast_sign_loss
 )]
 #[must_use]
-pub fn scale_dimensions(w: f32, h: f32, target_px: f32) -> (f32, f32) {
+pub fn scale_dimensions(w: f32, h: f32, target_px: f32) -> (u32, u32) {
 	let sf = (target_px / (w * h)).sqrt();
-	((w * sf).round(), (h * sf).round())
+	((w * sf).round() as u32, (h * sf).round() as u32)
 }

crates/images/src/pdf.rs

@@ -4,10 +4,13 @@ use std::{
 	path::{Path, PathBuf},
 };
 
-use crate::{consts::PDF_RENDER_WIDTH, ImageHandler, Result};
+use crate::{
+	consts::{PDF_LANDSCAPE_RENDER_WIDTH, PDF_PORTRAIT_RENDER_WIDTH},
+	ImageHandler, Result,
+};
 use image::DynamicImage;
 use once_cell::sync::Lazy;
-use pdfium_render::prelude::{PdfPageRenderRotation, PdfRenderConfig, Pdfium};
+use pdfium_render::prelude::{PdfRenderConfig, Pdfium};
 use tracing::error;
 
 // This path must be relative to the running binary
@@ -49,10 +52,16 @@ static PDFIUM_LIB: Lazy<String> = Lazy::new(|| {
 		})
 });
 
-static PDFIUM_RENDER_CONFIG: Lazy<PdfRenderConfig> = Lazy::new(|| {
+static PORTRAIT_CONFIG: Lazy<PdfRenderConfig> = Lazy::new(|| {
 	PdfRenderConfig::new()
-		.set_target_width(PDF_RENDER_WIDTH)
-		.rotate_if_landscape(PdfPageRenderRotation::Degrees90, true)
+		.set_target_width(PDF_PORTRAIT_RENDER_WIDTH)
+		.render_form_data(false)
+		.render_annotations(false)
+});
+
+static LANDSCAPE_CONFIG: Lazy<PdfRenderConfig> = Lazy::new(|| {
+	PdfRenderConfig::new()
+		.set_target_width(PDF_LANDSCAPE_RENDER_WIDTH)
 		.render_form_data(false)
 		.render_annotations(false)
 });
@@ -68,8 +77,13 @@ impl ImageHandler for PdfHandler {
 
 		let pdf = pdfium.load_pdf_from_file(path, None)?;
 		let first_page = pdf.pages().first()?;
+
 		let image = first_page
-			.render_with_config(&PDFIUM_RENDER_CONFIG)?
+			.render_with_config(if first_page.is_portrait() {
+				&PORTRAIT_CONFIG
+			} else {
+				&LANDSCAPE_CONFIG
+			})?
 			.as_image();
 
 		Ok(image)

crates/images/src/svg.rs

@@ -31,9 +31,9 @@ impl ImageHandler for SvgHandler {
 			scale_dimensions(rtree.size.width(), rtree.size.height(), SVG_TARGET_PX);
 
 		let size = if rtree.size.width() > rtree.size.height() {
-			rtree.size.to_int_size().scale_to_width(scaled_w as u32)
+			rtree.size.to_int_size().scale_to_width(scaled_w)
 		} else {
-			rtree.size.to_int_size().scale_to_height(scaled_h as u32)
+			rtree.size.to_int_size().scale_to_height(scaled_h)
 		}
 		.ok_or(Error::InvalidLength)?;
 

packages/client/src/core.ts

@@ -138,7 +138,7 @@ export type Composite = "Unknown" | "False" | "General" | "Live"
 
 export type ConvertImageArgs = { location_id: number; file_path_id: number; delete_src: boolean; desired_extension: ConvertableExtension; quality_percentage: number | null }
 
-export type ConvertableExtension = "bmp" | "dib" | "ff" | "gif" | "ico" | "jpg" | "jpeg" | "png" | "pnm" | "qoi" | "tga" | "icb" | "vda" | "vst" | "tiff" | "tif" | "heif" | "heifs" | "heic" | "heics" | "avif" | "avci" | "avcs" | "svg" | "svgz" | "pdf" | "webp"
+export type ConvertableExtension = "bmp" | "dib" | "ff" | "gif" | "ico" | "jpg" | "jpeg" | "png" | "pnm" | "qoi" | "tga" | "icb" | "vda" | "vst" | "tiff" | "tif" | "hif" | "heif" | "heifs" | "heic" | "heics" | "avif" | "avci" | "avcs" | "svg" | "svgz" | "pdf" | "webp"
 
 export type CreateEphemeralFolderArgs = { path: string; name: string | null }