[feature] Update attachment format, receive + send `focalPoint` prop + use it on the frontend (#4052)

* [feature] Update attachment format, receive + send `focalPoint` prop + use it on the frontend

* whoops

* boop

* restore function signature of ExtractAttachments

5 files changed, 145 insertions, 1 deletions

diff --git a/web/source/frontend/index.js b/web/source/frontend/index.js
index 47879b2e2..a1c2ca74b 100644
--- a/web/source/frontend/index.js
+++ b/web/source/frontend/index.js
@@ -29,6 +29,7 @@
 const Photoswipe = require("photoswipe/dist/umd/photoswipe.umd.min.js");
 const PhotoswipeLightbox = require("photoswipe/dist/umd/photoswipe-lightbox.umd.min.js");
 const PhotoswipeCaptionPlugin = require("photoswipe-dynamic-caption-plugin").default;
+const ObjectPosition = require("./photoswipe-object-position.js").default;
 const Plyr = require("plyr");
 const Prism = require("./prism.js");
 
@@ -61,6 +62,10 @@ new PhotoswipeCaptionPlugin(lightbox, {
 	}
 });
 
+// Enable object-position plugin for lightbox so that css
+// object-position property can be used on preview images.
+new ObjectPosition(lightbox);
+
 lightbox.addFilter('itemData', (item) => {
 	const el = item.element;
 	if (
diff --git a/web/source/frontend/photoswipe-object-position.js b/web/source/frontend/photoswipe-object-position.js
new file mode 100644
index 000000000..e67f9737d
--- /dev/null
+++ b/web/source/frontend/photoswipe-object-position.js
@@ -0,0 +1,119 @@
+/*
+	GoToSocial
+	Copyright (C) GoToSocial Authors admin@gotosocial.org
+	SPDX-License-Identifier: AGPL-3.0-or-later
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+	Code in this file adapted from:
+	https://github.com/vovayatsyuk/photoswipe-object-position (MIT License).
+*/
+
+function getCroppedBoundsOffset(position, imageSize, thumbSize, zoomLevel) {
+	const float = parseFloat(position); 
+	return position.indexOf('%') > 0
+		? (thumbSize - imageSize * zoomLevel) * float / 100
+		: float;
+}
+ 
+function getCroppedZoomPan(position, min, max) {
+	const float = parseFloat(position);
+	return position.indexOf('%') > 0 ? min + (max - min) * float / 100 : float;
+}
+
+function getThumbnail(el) {
+	return el.querySelector('img');
+}
+
+function getObjectPosition(el) {
+	return getComputedStyle(el).getPropertyValue('object-position').split(' ');
+}
+
+export default class ObjectPosition {
+	constructor(lightbox) {
+		/**
+		* Make pan adjustments if large image doesn't fit the viewport.
+		*
+		* Examples:
+		* 1. When thumb object-position is 50% 0 (top part is initially visible)
+		*    make sure you'll see the top part of the large image as well.
+		* 2. When thumb object-position is 50% 100% (bottom part is initially visible)
+		*    make sure you'll see the bottom part of the large image as well.
+		*/
+		lightbox.on('initialZoomPan', (event) => {
+			const slide = event.slide;
+			if (!slide.data.element) {
+				// No thumbnail
+				// image set.
+				return;
+			}
+			
+			const thumbnailImg = getThumbnail(slide.data.element);
+			if (!thumbnailImg) {
+				// No thumbnail
+				// image set.
+				return;
+			}
+
+			const [positionX, positionY] = getObjectPosition(thumbnailImg);
+
+			if (positionX !== '50%' && slide.pan.x < 0) {
+				slide.pan.x = getCroppedZoomPan(positionX, slide.bounds.min.x, slide.bounds.max.x);
+			}
+ 
+			if (positionY !== '50%' && slide.pan.y < 0) {
+				slide.pan.y = getCroppedZoomPan(positionY, slide.bounds.min.y, slide.bounds.max.y);
+			}
+		});
+
+		/**
+		* Fix opening animation when thumb object-position is not 50% 50%.
+		* https://github.com/dimsemenov/PhotoSwipe/pull/1868
+		*/
+		lightbox.addFilter('thumbBounds', (thumbBounds, itemData) => {
+			if (!itemData.element) {
+				// No thumbnail
+				// image set.
+				return;
+			}
+			
+			const thumbnailImg = getThumbnail(itemData.element);
+			if (!thumbnailImg) {
+				// No thumbnail
+				// image set.
+				return;
+			}
+
+			const thumbAreaRect = thumbnailImg.getBoundingClientRect();
+			const fillZoomLevel = thumbBounds.w / itemData.width;
+			const [positionX, positionY] = getObjectPosition(thumbnailImg);
+
+			if (positionX !== '50%') {
+				const offsetX = getCroppedBoundsOffset(positionX, itemData.width, thumbAreaRect.width, fillZoomLevel);
+				thumbBounds.x = thumbAreaRect.left + offsetX;
+				thumbBounds.innerRect.x = offsetX;
+			}
+
+			if (positionY !== '50%') {
+				const offsetY = getCroppedBoundsOffset(positionY, itemData.height, thumbAreaRect.height, fillZoomLevel);
+				thumbBounds.y = thumbAreaRect.top + offsetY;
+				thumbBounds.innerRect.y = offsetY;
+			}
+
+			return thumbBounds;
+		});
+	}
+}
diff --git a/web/source/frontend_prerender/index.js b/web/source/frontend_prerender/index.js
index 294c1ddb1..dd9420655 100644
--- a/web/source/frontend_prerender/index.js
+++ b/web/source/frontend_prerender/index.js
@@ -29,6 +29,11 @@ import { decode } from "blurhash";
 
 const reduceMotion = window.matchMedia('(prefers-reduced-motion: reduce)');
 
+// Adjust object-position of any image that has a focal point set.
+document.querySelectorAll("img[data-object-position]").forEach(img => {
+	img.style["object-position"] = img.dataset.objectPosition;
+});
+
 // Generate a blurhash canvas for each image for
 // each blurhash container and put it in the summary.
 Array.from(document.getElementsByClassName('blurhash-container')).forEach(blurhashContainer => {
@@ -36,6 +41,7 @@ Array.from(document.getElementsByClassName('blurhash-container')).forEach(blurha
 	const thumbHeight = blurhashContainer.dataset.blurhashHeight;
 	const thumbWidth = blurhashContainer.dataset.blurhashWidth;
 	const thumbAspect = blurhashContainer.dataset.blurhashAspect;	
+	const objectPosition = blurhashContainer.dataset.blurhashObjectPosition;
 	
 	/*
 		It's very expensive to draw big canvases
@@ -73,6 +79,12 @@ Array.from(document.getElementsByClassName('blurhash-container')).forEach(blurha
 	imageData.data.set(pixels);
 	ctx.putImageData(imageData, 0, 0);
 
+	// Set object-position css property on
+	// the canvas if it's set on the container.
+	if (objectPosition) {
+		canvas.style["object-position"] = objectPosition;
+	}
+
 	// Put the canvas inside the container.
 	blurhashContainer.appendChild(canvas);
 });
diff --git a/web/source/index.js b/web/source/index.js
index d66afe757..6a218cd08 100644
--- a/web/source/index.js
+++ b/web/source/index.js
@@ -60,7 +60,7 @@ skulk({
 			transform: [
 				["babelify", {
 					global: true,
-					ignore: [/node_modules\/(?!(photoswipe.*))/]
+					ignore: [/node_modules\/(?!(.*photoswipe.*))/]
 				}]
 			],
 		},
diff --git a/web/template/status_attachment.tmpl b/web/template/status_attachment.tmpl
index d04f11d5d..6f5041229 100644
--- a/web/template/status_attachment.tmpl
+++ b/web/template/status_attachment.tmpl
@@ -29,6 +29,10 @@
     height="{{- .Meta.Small.Height -}}"
     data-blurhash-hash="{{- .Blurhash -}}"
     data-sensitive="{{- .Sensitive -}}"
+    {{- if or (ne .Meta.Focus.X 0.0) (ne .Meta.Focus.Y 0.0) }}
+    data-object-position="{{ objectPosition .Meta.Focus.X .Meta.Focus.Y }}"
+    {{- else }}
+    {{- end }}
 />
 {{- else }}
 <img
@@ -69,6 +73,10 @@
                 data-blurhash-height="{{- .Item.Meta.Small.Height -}}"
                 data-blurhash-hash="{{- .Item.Blurhash -}}"
                 data-blurhash-aspect="{{- .Item.Meta.Small.Aspect -}}"
+                {{- if or (ne .Item.Meta.Focus.X 0.0) (ne .Item.Meta.Focus.Y 0.0) }}
+                data-blurhash-object-position="{{ objectPosition .Item.Meta.Focus.X .Item.Meta.Focus.Y }}"
+                {{- else }}
+                {{- end }}
             ></div>
             {{- end }}
         </summary>