[feature/frogend] (Mastodon) domain block CSV import (#1390)

* checkbox-list styling with taller <p> element

* CSV import/export, UI/UX improvements to import-export interface

* minor styling tweaks

* csv export, clean up export type branching

* abstract domain block entry validation

* foundation for PSL check + suggestions

* Squashed commit of the following:

commit e3655ba4fbea1d55738b2f9e407d3378af26afe6
Author: f0x <f0x@cthu.lu>
Date:   Tue Jan 31 15:19:10 2023 +0100

    let debug depend on env (prod/debug) again

commit 79c792b832a2b59e472dcdff646bad6d71b42cc9
Author: f0x <f0x@cthu.lu>
Date:   Tue Jan 31 00:34:01 2023 +0100

    update checklist components

commit 4367960fe4be4e3978077af06e63a729d64e32fb
Author: f0x <f0x@cthu.lu>
Date:   Mon Jan 30 23:46:20 2023 +0100

    checklist performance improvements

commit 204a4c02d16ffad189a6e8a6001d5bf4ff95fc4e
Author: f0x <f0x@cthu.lu>
Date:   Mon Jan 30 20:05:34 2023 +0100

    checklist field: use reducer for state

* remove debug logging

* show and use domain block suggestion

* restructure import/export buttons

* updating suggestions

* suggestion overview

* restructure check-list behavior, domain import/export

4 files changed, 298 insertions, 141 deletions

diff --git a/web/source/settings/lib/domain-block.js b/web/source/settings/lib/domain-block.js
new file mode 100644
index 000000000..d13c029d2
--- /dev/null
+++ b/web/source/settings/lib/domain-block.js
@@ -0,0 +1,50 @@
+/*
+	GoToSocial
+	Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+"use strict";
+
+const isValidDomain = require("is-valid-domain");
+const psl = require("psl");
+
+function isValidDomainBlock(domain) {
+	return isValidDomain(domain, {
+		/* 
+			Wildcard prefix *. can be stripped since it's equivalent to not having it,
+			but wildcard anywhere else in the domain is not handled by the backend so it's invalid.
+		*/
+		wildcard: false,
+		allowUnicode: true
+	});
+}
+
+/* 
+	Still can't think of a better function name for this,
+	but we're checking a domain against the Public Suffix List <https://publicsuffix.org/>
+	to see if we should suggest removing subdomain(s) since they're likely owned/ran by the same party
+	social.example.com -> suggests example.com
+*/
+function hasBetterScope(domain) {
+	const lookup = psl.get(domain);
+	if (lookup && lookup != domain) {
+		return lookup;
+	} else {
+		return false;
+	}
+}
+
+module.exports = { isValidDomainBlock, hasBetterScope };
\ No newline at end of file
diff --git a/web/source/settings/lib/form/check-list.jsx b/web/source/settings/lib/form/check-list.jsx
index c1233273d..b19e17a29 100644
--- a/web/source/settings/lib/form/check-list.jsx
+++ b/web/source/settings/lib/form/check-list.jsx
@@ -20,128 +20,163 @@
 
 const React = require("react");
 const syncpipe = require("syncpipe");
-
-function createState(entries, uniqueKey, oldState, defaultValue) {
-	return syncpipe(entries, [
-		(_) => _.map((entry) => {
-			let key = entry[uniqueKey];
-			return [
-				key,
-				{
-					...entry,
-					key,
-					checked: oldState[key]?.checked ?? entry.checked ?? defaultValue
+const { createSlice } = require("@reduxjs/toolkit");
+const { enableMapSet } = require("immer");
+
+enableMapSet(); // for use in reducers
+
+const { reducer, actions } = createSlice({
+	name: "checklist",
+	initialState: {}, // not handled by slice itself
+	reducers: {
+		updateAll: (state, { payload: checked }) => {
+			const selectedEntries = new Set();
+			return {
+				entries: syncpipe(state.entries, [
+					(_) => Object.values(_),
+					(_) => _.map((entry) => {
+						if (checked) {
+							selectedEntries.add(entry.key);
+						}
+						return [entry.key, {
+							...entry,
+							checked
+						}];
+					}),
+					(_) => Object.fromEntries(_)
+				]),
+				selectedEntries
+			};
+		},
+		update: (state, { payload: { key, value } }) => {
+			if (value.checked !== undefined) {
+				if (value.checked === true) {
+					state.selectedEntries.add(key);
+				} else {
+					state.selectedEntries.delete(key);
+				}
+			}
+
+			state.entries[key] = {
+				...state.entries[key],
+				...value
+			};
+		},
+		updateMultiple: (state, { payload }) => {
+			payload.forEach(([key, value]) => {
+				if (value.checked !== undefined) {
+					if (value.checked === true) {
+						state.selectedEntries.add(key);
+					} else {
+						state.selectedEntries.delete(key);
+					}
 				}
-			];
-		}),
-		(_) => Object.fromEntries(_)
-	]);
-}
 
-function updateAllState(state, newValue) {
-	return syncpipe(state, [
-		(_) => Object.values(_),
-		(_) => _.map((entry) => [entry.key, {
-			...entry,
-			checked: newValue
-		}]),
-		(_) => Object.fromEntries(_)
-	]);
-}
+				state.entries[key] = {
+					...state.entries[key],
+					...value
+				};
+			});
+		}
+	}
+});
 
-function updateState(state, key, newValue) {
+function initialState({ entries, uniqueKey, defaultValue }) {
+	const selectedEntries = new Set();
 	return {
-		...state,
-		[key]: {
-			...state[key],
-			...newValue
-		}
+		entries: syncpipe(entries, [
+			(_) => _.map((entry) => {
+				let key = entry[uniqueKey];
+				let checked = entry.checked ?? defaultValue;
+
+				if (checked) {
+					selectedEntries.add(key);
+				} else {
+					selectedEntries.delete(key);
+				}
+
+				return [
+					key,
+					{
+						...entry,
+						key,
+						checked
+					}
+				];
+			}),
+			(_) => Object.fromEntries(_)
+		]),
+		selectedEntries
 	};
 }
 
 module.exports = function useCheckListInput({ name }, { entries, uniqueKey = "key", defaultValue = false }) {
-	const [state, setState] = React.useState({});
+	const [state, dispatch] = React.useReducer(reducer, null,
+		() => initialState({ entries, uniqueKey, defaultValue }) // initial state
+	);
 
-	const [someSelected, setSomeSelected] = React.useState(false);
-	const [toggleAllState, setToggleAllState] = React.useState(0);
 	const toggleAllRef = React.useRef(null);
 
 	React.useEffect(() => {
-		/* 
-			entries changed, update state,
-			re-using old state if available for key
-		*/
-		setState(createState(entries, uniqueKey, state, defaultValue));
-
-		/* eslint-disable-next-line react-hooks/exhaustive-deps */
-	}, [entries]);
-
-	React.useEffect(() => {
-		/* Updates (un)check all checkbox, based on shortcode checkboxes
-			 Can be 0 (not checked), 1 (checked) or 2 (indeterminate)
-		 */
-		if (toggleAllRef.current == null) {
-			return;
+		if (toggleAllRef.current != null) {
+			let some = state.selectedEntries.size > 0;
+			let all = false;
+			if (some) {
+				all = state.selectedEntries.size == Object.values(state.entries).length;
+			}
+			toggleAllRef.current.checked = all;
+			toggleAllRef.current.indeterminate = some && !all;
 		}
-
-		let values = Object.values(state);
-		/* one or more boxes are checked */
-		let some = values.some((v) => v.checked);
-
-		let all = false;
-		if (some) {
-			/* there's not at least one unchecked box */
-			all = !values.some((v) => v.checked == false);
+		// only needs to update when state.selectedEntries changes, not state.entries
+		// eslint-disable-next-line react-hooks/exhaustive-deps
+	}, [state.selectedEntries]);
+
+	const reset = React.useCallback(
+		() => dispatch(actions.updateAll(defaultValue)),
+		[defaultValue]
+	);
+
+	const onChange = React.useCallback(
+		(key, value) => dispatch(actions.update({ key, value })),
+		[]
+	);
+
+	const updateMultiple = React.useCallback(
+		(entries) => dispatch(actions.updateMultiple(entries)),
+		[]
+	);
+
+	return React.useMemo(() => {
+		function toggleAll(e) {
+			let checked = e.target.checked;
+			if (e.target.indeterminate) {
+				checked = false;
+			}
+			dispatch(actions.updateAll(checked));
 		}
 
-		setSomeSelected(some);
-
-		if (some && !all) {
-			setToggleAllState(2);
-			toggleAllRef.current.indeterminate = true;
-		} else {
-			setToggleAllState(all ? 1 : 0);
-			toggleAllRef.current.indeterminate = false;
+		function selectedValues() {
+			return Array.from((state.selectedEntries)).map((key) => ({
+				...state.entries[key] // returned as new object, because reducer state is immutable
+			}));
 		}
-	}, [state, toggleAllRef]);
-
-	function toggleAll(e) {
-		let selectAll = e.target.checked;
 
-		if (toggleAllState == 2) { // indeterminate
-			selectAll = false;
-		}
-
-		setState(updateAllState(state, selectAll));
-		setToggleAllState(selectAll);
-	}
-
-	function reset() {
-		setState(updateAllState(state, defaultValue));
-	}
-
-	function selectedValues() {
-		return syncpipe(state, [
-			(_) => Object.values(_),
-			(_) => _.filter((entry) => entry.checked)
-		]);
-	}
-
-	return Object.assign([
-		state,
-		reset,
-		{ name }
-	], {
-		name,
-		value: state,
-		onChange: (key, newValue) => setState(updateState(state, key, newValue)),
-		selectedValues,
-		reset,
-		someSelected,
-		toggleAll: {
-			ref: toggleAllRef,
-			value: toggleAllState,
-			onChange: toggleAll
-		}
-	});
+		return Object.assign([
+			state,
+			reset,
+			{ name }
+		], {
+			name,
+			value: state.entries,
+			onChange,
+			selectedValues,
+			reset,
+			someSelected: state.someChecked,
+			updateMultiple,
+			toggleAll: {
+				ref: toggleAllRef,
+				onChange: toggleAll
+			}
+		});
+	}, [state, reset, name, onChange, updateMultiple]);
 };
\ No newline at end of file
diff --git a/web/source/settings/lib/form/text.jsx b/web/source/settings/lib/form/text.jsx
index 70e61657c..d9a9ab28c 100644
--- a/web/source/settings/lib/form/text.jsx
+++ b/web/source/settings/lib/form/text.jsx
@@ -20,14 +20,30 @@
 
 const React = require("react");
 
-module.exports = function useTextInput({ name, Name }, { validator, defaultValue = "", dontReset = false } = {}) {
+module.exports = function useTextInput({ name, Name }, {
+	defaultValue = "",
+	dontReset = false,
+	validator,
+	showValidation = true,
+	initValidation
+} = {}) {
+
 	const [text, setText] = React.useState(defaultValue);
-	const [valid, setValid] = React.useState(true);
 	const textRef = React.useRef(null);
 
+	const [validation, setValidation] = React.useState(initValidation ?? "");
+	const [_isValidating, startValidation] = React.useTransition();
+	let valid = validation == "";
+
 	function onChange(e) {
 		let input = e.target.value;
 		setText(input);
+
+		if (validator) {
+			startValidation(() => {
+				setValidation(validator(input));
+			});
+		}
 	}
 
 	function reset() {
@@ -38,11 +54,13 @@ module.exports = function useTextInput({ name, Name }, { validator, defaultValue
 
 	React.useEffect(() => {
 		if (validator && textRef.current) {
-			let res = validator(text);
-			setValid(res == "");
-			textRef.current.setCustomValidity(res);
+			if (showValidation) {
+				textRef.current.setCustomValidity(validation);
+			} else {
+				textRef.current.setCustomValidity("");
+			}
 		}
-	}, [text, textRef, validator]);
+	}, [validation, validator, showValidation]);
 
 	// Array / Object hybrid, for easier access in different contexts
 	return Object.assign([
@@ -62,6 +80,7 @@ module.exports = function useTextInput({ name, Name }, { validator, defaultValue
 		ref: textRef,
 		setter: setText,
 		valid,
+		validate: () => setValidation(validator(text)),
 		hasChanged: () => text != defaultValue
 	});
 };
\ No newline at end of file
diff --git a/web/source/settings/lib/query/admin/import-export.js b/web/source/settings/lib/query/admin/import-export.js
index 94e462bd2..a4a8b65e3 100644
--- a/web/source/settings/lib/query/admin/import-export.js
+++ b/web/source/settings/lib/query/admin/import-export.js
@@ -19,8 +19,11 @@
 "use strict";
 
 const Promise = require("bluebird");
-const isValidDomain = require("is-valid-domain");
 const fileDownload = require("js-file-download");
+const csv = require("papaparse");
+const { nanoid } = require("nanoid");
+
+const { isValidDomainBlock, hasBetterScope } = require("../../domain-block");
 
 const {
 	replaceCacheOnMutation,
@@ -31,6 +34,23 @@ const {
 function parseDomainList(list) {
 	if (list[0] == "[") {
 		return JSON.parse(list);
+	} else if (list.startsWith("#domain")) { // Mastodon CSV
+		const { data, errors } = csv.parse(list, {
+			header: true,
+			transformHeader: (header) => header.slice(1), // removes starting '#'
+			skipEmptyLines: true,
+			dynamicTyping: true
+		});
+
+		if (errors.length > 0) {
+			let error = "";
+			errors.forEach((err) => {
+				error += `${err.message} (line ${err.row})`;
+			});
+			throw error;
+		}
+
+		return data;
 	} else {
 		return list.split("\n").map((line) => {
 			let domain = line.trim();
@@ -51,7 +71,15 @@ function parseDomainList(list) {
 
 function validateDomainList(list) {
 	list.forEach((entry) => {
-		entry.valid = (entry.valid !== false) && isValidDomain(entry.domain, { wildcard: true, allowUnicode: true });
+		if (entry.domain.startsWith("*.")) {
+			// domain block always includes all subdomains, wildcard is meaningless here
+			entry.domain = entry.domain.slice(2);
+		}
+
+		entry.valid = (entry.valid !== false) && isValidDomainBlock(entry.domain);
+		if (entry.valid) {
+			entry.suggest = hasBetterScope(entry.domain);
+		}
 		entry.checked = entry.valid;
 	});
 
@@ -83,6 +111,9 @@ module.exports = (build) => ({
 			}).then((deduped) => {
 				return validateDomainList(deduped);
 			}).then((data) => {
+				data.forEach((entry) => {
+					entry.key = nanoid(); // unique id that stays stable even if domain gets modified by user
+				});
 				return { data };
 			}).catch((e) => {
 				return { error: e.toString() };
@@ -91,27 +122,53 @@ module.exports = (build) => ({
 	}),
 	exportDomainList: build.mutation({
 		queryFn: (formData, api, _extraOpts, baseQuery) => {
+			let process;
+
+			if (formData.exportType == "json") {
+				process = {
+					transformEntry: (entry) => ({
+						domain: entry.domain,
+						public_comment: entry.public_comment,
+						obfuscate: entry.obfuscate
+					}),
+					stringify: (list) => JSON.stringify(list),
+					extension: ".json",
+					mime: "application/json"
+				};
+			} else if (formData.exportType == "csv") {
+				process = {
+					transformEntry: (entry) => [
+						entry.domain,
+						"suspend", // severity
+						false, // reject_media
+						false, // reject_reports
+						entry.public_comment,
+						entry.obfuscate ?? false
+					],
+					stringify: (list) => csv.unparse({
+						fields: "#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate".split(","),
+						data: list
+					}),
+					extension: ".csv",
+					mime: "text/csv"
+				};
+			} else {
+				process = {
+					transformEntry: (entry) => entry.domain,
+					stringify: (list) => list.join("\n"),
+					extension: ".txt",
+					mime: "text/plain"
+				};
+			}
+
 			return Promise.try(() => {
 				return baseQuery({
 					url: `/api/v1/admin/domain_blocks`
 				});
 			}).then(unwrapRes).then((blockedInstances) => {
-				return blockedInstances.map((entry) => {
-					if (formData.exportType == "json") {
-						return {
-							domain: entry.domain,
-							public_comment: entry.public_comment
-						};
-					} else {
-						return entry.domain;
-					}
-				});
+				return blockedInstances.map(process.transformEntry);
 			}).then((exportList) => {
-				if (formData.exportType == "json") {
-					return JSON.stringify(exportList);
-				} else {
-					return exportList.join("\n");
-				}
+				return process.stringify(exportList);
 			}).then((exportAsString) => {
 				if (formData.action == "export") {
 					return {
@@ -120,7 +177,6 @@ module.exports = (build) => ({
 				} else if (formData.action == "export-file") {
 					let domain = new URL(api.getState().oauth.instance).host;
 					let date = new Date();
-					let mime;
 
 					let filename = [
 						domain,
@@ -130,15 +186,11 @@ module.exports = (build) => ({
 						date.getDate().toString().padStart(2, "0"),
 					].join("-");
 
-					if (formData.exportType == "json") {
-						filename += ".json";
-						mime = "application/json";
-					} else {
-						filename += ".txt";
-						mime = "text/plain";
-					}
-
-					fileDownload(exportAsString, filename, mime);
+					fileDownload(
+						exportAsString,
+						filename + process.extension,
+						process.mime
+					);
 				}
 				return { data: null };
 			}).catch((e) => {
@@ -171,6 +223,7 @@ module.exports = (build) => ({
 	})
 });
 
+const internalKeys = new Set("key,suggest,valid,checked".split(","));
 function entryProcessor(formData) {
 	let funcs = [];
 
@@ -204,7 +257,7 @@ function entryProcessor(formData) {
 		entry.obfuscate = formData.obfuscate;
 
 		Object.entries(entry).forEach(([key, val]) => {
-			if (val == undefined) {
+			if (internalKeys.has(key) || val == undefined) {
 				delete entry[key];
 			}
 		});