[feature] Use blurhashes in frontend, tidy up gallery view a bit (#3948)

* [feature] Use blurhashes in frontend, tidy up gallery view a bit

* weeeeeeeeeeeeeeeee

* beep boop

8 files changed, 404 insertions, 75 deletions

diff --git a/web/source/blurhash/index.js b/web/source/blurhash/index.js
new file mode 100644
index 000000000..c964f69c4
--- /dev/null
+++ b/web/source/blurhash/index.js
@@ -0,0 +1,146 @@
+/*
+	GoToSocial
+	Copyright (C) GoToSocial Authors admin@gotosocial.org
+	SPDX-License-Identifier: AGPL-3.0-or-later
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { decode } from "blurhash";
+
+// Generate a blurhash canvas for each image for
+// each blurhash container and put it in the summary.
+Array.from(document.getElementsByClassName('blurhash-container')).forEach(blurhashContainer => {
+	const hash = blurhashContainer.dataset.blurhashHash;
+	const thumbHeight = blurhashContainer.dataset.blurhashHeight;
+	const thumbWidth = blurhashContainer.dataset.blurhashWidth;
+	const thumbAspect = blurhashContainer.dataset.blurhashAspect;	
+	
+	/*
+		It's very expensive to draw big canvases
+		with blurhashes, so use tiny ones, keeping
+		aspect ratio of the original thumbnail.
+	*/
+	var useWidth = 32;
+	var useHeight = 32;
+	switch (true) {
+		case thumbWidth > thumbHeight:
+			useHeight = Math.round(useWidth / thumbAspect);
+			break;
+		case thumbHeight > thumbWidth:
+			useWidth = Math.round(useHeight * thumbAspect);
+			break;
+	}
+
+	const pixels = decode(
+		hash,
+		useWidth,
+		useHeight,
+	);
+
+	// Create canvas of appropriate size.
+	const canvas = document.createElement("canvas");
+	canvas.width = useWidth;
+	canvas.height = useHeight;
+
+	// Draw the image data into the canvas.
+	const ctx = canvas.getContext("2d");
+	const imageData = ctx.createImageData(
+		useWidth,
+		useHeight,
+	);
+	imageData.data.set(pixels);
+	ctx.putImageData(imageData, 0, 0);
+
+	// Put the canvas inside the container.
+	blurhashContainer.appendChild(canvas);
+});
+
+// Add a smooth transition from blurhash
+// to image for each sensitive image.
+Array.from(document.getElementsByTagName('img')).forEach(img => {
+	if (!img.dataset.blurhashHash) {
+		// Has no blurhash,
+		// can't transition smoothly.
+		return;
+	}
+
+	if (img.dataset.sensitive !== "true") {
+		// Not sensitive, smooth
+		// transition doesn't matter.
+		return;
+	}
+
+	if (img.complete) {
+		// Image already loaded,
+		// don't stub it with anything.
+		return;
+	}
+	
+	const parentSlide = img.closest(".photoswipe-slide");
+	if (!parentSlide) {
+		// Parent slide was nil,
+		// can't do anything.
+		return;
+	}
+
+	const blurhashContainer = document.querySelector("div[data-blurhash-hash=\"" + img.dataset.blurhashHash + "\"]");
+	if (!blurhashContainer) {
+		// Blurhash div was nil,
+		// can't do anything.
+		return;
+	}
+
+	const canvas = blurhashContainer.children[0];
+	if (!canvas) {
+		// Canvas was nil,
+		// can't do anything.
+		return;
+	} 
+
+	// "Replace" the hidden img with a canvas
+	// that will show initially when it's clicked.
+	const clone = canvas.cloneNode(true);
+	clone.getContext("2d").drawImage(canvas, 0, 0);
+	parentSlide.prepend(clone);
+	img.className = img.className + " hidden";
+
+	// Add a listener so that when the spoiler
+	// is opened, loading of the image begins.
+	const parentSummary = img.closest(".media-spoiler");
+	parentSummary.addEventListener("toggle", (_) => {
+		if (parentSummary.hasAttribute("open") && !img.complete) {
+			img.loading = "eager";
+		}
+	});
+
+	// Add a callback that triggers
+	// when image loading is complete.
+	img.addEventListener("load", () => {
+		// Show the image now that it's loaded.
+		img.className = img.className.replace(" hidden", "");
+		
+		// Reset the lazy loading tag to its initial
+		// value. This doesn't matter too much since
+		// it's already loaded but it feels neater.
+		img.loading = "lazy";
+
+		// Remove the temporary blurhash
+		// canvas so only the image shows.
+		const canvas = parentSlide.getElementsByTagName("canvas")[0];
+		if (canvas) {
+			canvas.remove();
+		}
+	});
+});
diff --git a/web/source/css/_media-wrapper.css b/web/source/css/_media-wrapper.css
index 1c2ae1503..a567cb0fd 100644
--- a/web/source/css/_media-wrapper.css
+++ b/web/source/css/_media-wrapper.css
@@ -42,13 +42,26 @@
 		height: 100%;
 		width: 100%;
 
+		div.blurhash-container {
+			z-index: -1;
+			position: absolute;
+			height: 100%;
+			width: 100%;
+		}
+
+		canvas {
+			height: 100%;
+			width: 100%;
+			object-fit: cover;
+		}
+
 		&[open] summary {
 			height: auto;
 			width: auto;
 			margin: 1rem;
 			padding: 0;
 
-			.show, video, img {
+			.show {
 				display: none;
 			}
 
@@ -57,6 +70,10 @@
 				grid-column: 1 / span 3;
 				grid-row: 1 / span 2;
 			}
+
+			div.blurhash-container > canvas {
+				display: none;
+			}
 		}
 
 		summary {
@@ -65,7 +82,7 @@
 			width: 100%;
 			z-index: 3;
 			overflow: hidden;
-				
+
 			display: grid;
 			padding: 1rem;
 			grid-template-columns: 1fr auto 1fr;
@@ -107,24 +124,15 @@
 					align-self: center;
 				}
 			}
-
-			video, img {
-				z-index: -1;
-				position: absolute;
-				height: calc(100% + 1.2rem);
-				width: calc(100% + 1.2rem);
-				top: -0.6rem;
-				left: -0.6rem;
-				filter: blur(1.2rem);
-			}
 		}
 
 		video.plyr-video, .plyr {
 			position: absolute;
 			height: 100%;
 			width: 100%;
-			object-fit: contain;
+			object-fit: cover;
 			background: $gray1;
+			min-width: 100%;
 		}
 
 		.unknown-attachment {
@@ -161,6 +169,86 @@
 			}
 		}
 	}
+
+	@media screen and (max-width: 55rem) {
+		/*
+			Tablet-ish width, make "show sensitive"
+			and "eye" buttons smaller + more compact.
+		*/
+		& > details {
+			& > summary {
+				padding: 0.5rem;
+	
+				> div.show.sensitive.button {
+					font-size: smaller;
+					padding: 0.2rem;
+					line-height: 1.4rem;
+				}
+				
+				> span.eye.button {
+					font-size: smaller;
+					padding: 0 0.2rem 0 0.2rem;
+	
+					> .fa-fw {
+						line-height: 1.4rem;
+					}
+				}
+			}
+
+			&[open] > summary {
+				margin: 0.5rem;
+			}
+		}
+	}
+
+	@media screen and (max-width: 36rem) {
+		/*
+			Mobile-ish width, even more compact.
+		*/
+		& > details {
+			& > summary {
+				> div.show.sensitive.button {
+					font-size: small;
+					padding: 0.1rem;
+					line-height: 1.2rem;
+				}
+				
+				> span.eye.button {
+					font-size: small;
+					padding: 0 0.1rem 0 0.1rem;
+	
+					> .fa-fw {
+						line-height: 1.2rem;
+					}
+				}
+			}
+		}
+	}
+
+	@media screen and (max-width: 27rem) {
+		/*
+			Really really tiny, make the text
+			on show/hide sensitive even smaller.
+		*/
+		& > details > summary > div.show.sensitive.button {
+			font-size: x-small;
+		}
+	}
+}
+
+@media (scripting: none) {
+	.media-wrapper {
+		canvas.blurhash {
+			display: none
+		}
+
+		& > details:not([open]) {
+			background: linear-gradient(
+				var(--bg-accent),
+				var(--bg) 
+			);
+		}
+	}
 }
 
 .pswp__button--open-post-link {
@@ -187,21 +275,75 @@
 		position: initial;
 		padding: 0.1rem;
 		padding-top: 0.2rem;
+		gap: 0.15rem;
+
+		.plyr__controls__item {
+			/*
+				Override margins from plyr as
+				we're displaying in flex with a gap.
+			*/
+			margin-left: 0;
+			margin-right: 0;
+			&:first-child {
+				margin-right: 0;
+			}
+
+			/*
+				Try to split controls in at
+				least a somewhat sensible way.
+			*/
+			&.plyr__volume {
+				margin-left: auto;
+			}
+			&[data-plyr="restart"] {
+				margin-right: auto;
+			}
+		}
+
+		/*
+			Override the rule from plyr that
+			hides the total duration on thinner
+			screens, we have enough room.
+		*/
+		.plyr__time + .plyr__time {
+			display: initial;
+		}
 	}
 
 	.plyr__control {
 		box-shadow: none;
 	}
 
-	.plyr__control--overlaid {
-		top: calc(50% - 18px);
+	.plyr__poster {
+		background-size: cover;
+	}
+
+	/*
+		Hide plry controls when it's not inside
+		a lightbox, but use cursor pointer to
+		show the whole thing can be clicked.
+	*/
+	&.plyr--stopped, &.plyr--paused {
+		.plyr__controls {
+			display: none;
+		}
+
+		cursor: pointer;
 	}
 }
 
 .pswp__content {
 	padding: 2rem;
 
-	.plyr {
-		max-height: 100%;
+	/*
+		Render plyr controls as normal
+		when it's inside a lightbox.
+	*/
+	.plyr__control--overlaid {
+		top: calc(50% - 18px);
+	}
+	> .plyr--stopped .plyr__controls,
+	> .plyr--paused .plyr__controls {
+		display: flex;
 	}
 }
diff --git a/web/source/css/_status-media.css b/web/source/css/_status-media.css
index e8386c87a..761b969be 100644
--- a/web/source/css/_status-media.css
+++ b/web/source/css/_status-media.css
@@ -35,7 +35,12 @@
 		grid-row: span 2;
 	}
 
-	@media screen and (max-width: 42rem) {
+	/*
+		On really skinny screens allow
+		media wrapper to take up full
+		width instead of showing 2 columns.
+	*/
+	@media screen and (max-width: 23rem) {
 		.media-wrapper {
 			grid-column: span 2;
 			grid-row: span 2;
diff --git a/web/source/css/profile-gallery.css b/web/source/css/profile-gallery.css
index cb70eff22..ef18ed7b0 100644
--- a/web/source/css/profile-gallery.css
+++ b/web/source/css/profile-gallery.css
@@ -72,27 +72,16 @@
 			margin-top: 0.15rem;
 			margin-bottom: 0.15rem;
 		
+			/* Show 3 cols of media */
 			display: grid;
-			gap: 0.15rem;
-
-			/* Desktop-ish width, show 3 cols of media */
 			grid-template-columns: repeat(3, 1fr);
-		
-			@media screen and (max-width: 55rem) {
-				/* Tablet-ish width, switch to 2 cols */
-				grid-template-columns: repeat(2, 1fr);
-			}
-
-			@media screen and (max-width: 36rem) {
-				/* Mobile-ish width, switch to 1 col */
-				grid-template-columns: repeat(1, 1fr);
-			}
+			gap: 0.15rem;
 
 			.media-wrapper {
-				aspect-ratio: 4/3;
+				aspect-ratio: 1;
 				border: 0;
 				border-radius: 0;
-				background: $bg;
+				background: $status-bg;
 			}
 		}
 
diff --git a/web/source/frontend/index.js b/web/source/frontend/index.js
index d45420255..5a6224994 100644
--- a/web/source/frontend/index.js
+++ b/web/source/frontend/index.js
@@ -60,28 +60,39 @@ lightbox.addFilter('itemData', (item) => {
 		el._plyrContainer !== undefined
 	) {
 		const parentNode = el._plyrContainer.parentNode;
+		const loopingAuto = el.classList.contains("gifv");
 		return {
 			alt: el.getAttribute("alt"),
 			_video: {
 				open(c) {
 					c.appendChild(el._plyrContainer);
+					if (loopingAuto) {
+						// Start playing
+						// when opened.
+						el._player.play();
+					}
 				},
 				close() {
 					parentNode.appendChild(el._plyrContainer);
 				},
 				pause() {
 					el._player.pause();
+				},
+				play() {
+					el._player.play();
 				}
 			},
 			width: parseInt(el.dataset.pswpWidth),
 			height: parseInt(el.dataset.pswpHeight),
 			parentStatus: el.dataset.pswpParentStatus,
 			attachmentId: el.dataset.pswpAttachmentId,
+			loopingAuto: loopingAuto,
 		};
 	}
 	return item;
 });
 
+// Open video when user moves to its slide.
 lightbox.on("contentActivate", (e) => {
 	const { content } = e;
 	if (content.data._video != undefined) {
@@ -89,6 +100,8 @@ lightbox.on("contentActivate", (e) => {
 	}
 });
 
+// Pause + close video when user
+// moves away from its slide.
 lightbox.on("contentDeactivate", (e) => {
 	const { content } = e;
 	if (content.data._video != undefined) {
@@ -97,12 +110,27 @@ lightbox.on("contentDeactivate", (e) => {
 	}
 });
 
-lightbox.on("close", function () {
+// Pause video when lightbox is closed.
+lightbox.on("closingAnimationStart", function () {
 	if (lightbox.pswp.currSlide.data._video != undefined) {
 		lightbox.pswp.currSlide.data._video.close();
 	}
 });
+lightbox.on("close", function () {
+	if (lightbox.pswp.currSlide.data._video != undefined &&
+		!lightbox.pswp.currSlide.data.loopingAuto) {
+		lightbox.pswp.currSlide.data._video.pause();
+	}
+});
 
+// Open video when lightbox is opened.
+lightbox.on("openingAnimationEnd", function () {
+	if (lightbox.pswp.currSlide.data._video != undefined) {
+		lightbox.pswp.currSlide.data._video.play();
+	}
+});
+
+// Add "open this post" link to lightbox UI.
 lightbox.on('uiRegister', function() {
 	lightbox.pswp.ui.registerElement({
 		name: 'open-post-link',
@@ -164,59 +192,48 @@ dynamicSpoiler("media-spoiler", (spoiler) => {
 
 Array.from(document.getElementsByClassName("plyr-video")).forEach((video) => {
 	const loopingAuto = !reduceMotion.matches && video.classList.contains("gifv");
-	
-	if (loopingAuto) {
-		// If we're able to play this as a
-		// looping gifv, then do so, else fall
-		// back to user-controllable video player.
-		video.draggable = false;
-		video.autoplay = true;
-		video.loop = true;
-		video.classList.remove("photoswipe-slide");
-		video.classList.remove("plry-video");
-		video.load();
-		video.play();
-		return;
-	}
-	
 	let player = new Plyr(video, {
 		title: video.title,
 		settings: [],
-		controls: ['play-large', 'play', 'progress', 'current-time', 'volume', 'mute', 'fullscreen'],
-		disableContextMenu: false,
-		hideControls: false,
+		// Only show controls for video and audio,
+		// not looping soundless gifv. Don't show
+		// volume slider as it's unusable anyway
+		// when the video is inside a lightbox,
+		// mute toggle will have to be enough.
+		controls: loopingAuto
+			? []
+			: [
+				'play-large',   // The large play button in the center
+				'restart',      // Restart playback
+				'rewind',       // Rewind by the seek time (default 10 seconds)
+				'play',         // Play/pause playback
+				'fast-forward', // Fast forward by the seek time (default 10 seconds)
+				'current-time', // The current time of playback
+				'duration',     // The full duration of the media
+				'mute',         // Toggle mute
+				'fullscreen',   // Toggle fullscreen
+			],
 		tooltips: { controls: true, seek: true },
 		iconUrl: "/assets/plyr.svg",
 		invertTime: false,
+		hideControls: false,
 		listeners: {
-			fullscreen: () => {
-				// Check if the photoswipe lightbox is
-				// open with this as the current slide.
-				const alreadyInLightbox = (
-					lightbox.pswp !== undefined &&
-					video.dataset.pswpAttachmentId === lightbox.pswp.currSlide.data.attachmentId
-				);
-				
-				if (alreadyInLightbox) {
-					// If this video is already open as the
-					// current photoswipe slide, the fullscreen
-					// button toggles proper fullscreen.
-					player.fullscreen.toggle();
-				} else {
-					// Otherwise the fullscreen button opens
-					// the video as current photoswipe slide.
-					//
-					// (Don't pause the video while it's
-					// being transitioned to a slide.)
-					if (player.playing) {
-						setTimeout(() => player.play(), 1);
-					}
+			play: (_) => {
+				if (!inLightbox(video)) {
+					// If the video isn't open in the lightbox
+					// as the current photoswipe slide, clicking
+					// on it to play it opens it in the lightbox.
 					lightbox.loadAndOpen(parseInt(video.dataset.pswpIndex), {
 						gallery: video.closest(".photoswipe-gallery")
 					});
+				} else if (!loopingAuto) {
+					// If the video *is* open in the lightbox,
+					// and it's not a looping gifv, clicking
+					// play just plays or pauses the video.
+					player.togglePlay();
 				}
 				return false;
-			}
+			},
 		}
 	});
 
@@ -225,6 +242,21 @@ Array.from(document.getElementsByClassName("plyr-video")).forEach((video) => {
 	video._plyrContainer = player.elements.container;
 });
 
+// Return true if the photoswipe lightbox is
+// open with this element as the current slide.
+function inLightbox(element) {
+	if (lightbox.pswp === undefined) {
+		return false;
+	}
+
+	if (lightbox.pswp.currSlide === undefined) {
+		return false;
+	}
+
+	return element.dataset.pswpAttachmentId ===
+		lightbox.pswp.currSlide.data.attachmentId;
+}
+
 Array.from(document.getElementsByTagName('time')).forEach(timeTag => {
 	const datetime = timeTag.getAttribute('datetime');
 	const currentText = timeTag.textContent.trim();
diff --git a/web/source/index.js b/web/source/index.js
index 5cee28046..c47e9c5bb 100644
--- a/web/source/index.js
+++ b/web/source/index.js
@@ -64,6 +64,15 @@ skulk({
 				}]
 			],
 		},
+		blurhash: {
+			entryFile: "blurhash",
+			outputFile: "blurhash.js",
+			preset: ["js"],
+			prodCfg: prodCfg,
+			transform: [
+				["babelify", { global: true }]
+			],
+		},
 		settings: {
 			entryFile: "settings",
 			outputFile: "settings.js",
diff --git a/web/source/package.json b/web/source/package.json
index 7b11d7eb2..6a29eeeac 100644
--- a/web/source/package.json
+++ b/web/source/package.json
@@ -13,6 +13,7 @@
   "dependencies": {
     "@reduxjs/toolkit": "^1.8.6",
     "ariakit": "^2.0.0-next.41",
+    "blurhash": "^2.0.5",
     "get-by-dot": "^1.0.2",
     "html-to-text": "^9.0.5",
     "is-valid-domain": "^0.1.6",
diff --git a/web/source/yarn.lock b/web/source/yarn.lock
index f3f3a5752..8354d00c1 100644
--- a/web/source/yarn.lock
+++ b/web/source/yarn.lock
@@ -2398,6 +2398,11 @@ bluebird@^3.7.1, bluebird@^3.7.2:
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
   integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
 
+blurhash@^2.0.5:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/blurhash/-/blurhash-2.0.5.tgz#efde729fc14a2f03571a6aa91b49cba80d1abe4b"
+  integrity sha512-cRygWd7kGBQO3VEhPiTgq4Wc43ctsM+o46urrmPOiuAe+07fzlSB9OJVdpgDL0jPqXUVQ9ht7aq7kxOeJHRK+w==
+
 bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.11.9:
   version "4.12.0"
   resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.12.0.tgz#775b3f278efbb9718eec7361f483fb36fbbfea88"