json savefiles (#1211)

* json savefiles

Co-authored-by: ZephyrTFA <[email protected]>

* json

* Resolve comments

* kill meeee

* COMMA

* fuck 2

* Update code/modules/client/migrations/reconcile_v44.dm

* Update code/modules/client/preferences/_preference.dm

* Hard to save a savefile when you don't know who's savefile it is.

* Pretend to be a client better.

* allergies are murdering my throat I feel like wet concrete

* Fixes character swap loading

This was clobbering savefiles with garbage.

* piss and shit

* Mock clients never save on prod
Fix Randomize Human Appearance

---------

Co-authored-by: ZephyrTFA <[email protected]>
Co-authored-by: Kapu1178 <[email protected]>

Author: 3 people

code/datums/json_savefile.dm

@@ -0,0 +1,122 @@
+/**
+ * A savefile implementation that handles all data using json.
+ * Also saves it using JSON too, fancy.
+ * If you pass in a null path, it simply acts as a memory tree instead, and cannot be saved.
+ */
+/datum/json_savefile
+	var/path = ""
+	VAR_PRIVATE/list/tree
+	/// If this is set to true, calling set_entry or remove_entry will automatically call save(), this does not catch modifying a sub-tree, nor do I know how to do that
+	var/auto_save = FALSE
+	/// Cooldown that tracks the time between attempts to download the savefile.
+	COOLDOWN_DECLARE(download_cooldown)
+
+GENERAL_PROTECT_DATUM(/datum/json_savefile)
+
+/datum/json_savefile/New(path)
+	src.path = path
+	tree = list()
+	if(path && fexists(path))
+		load()
+
+/**
+ * Gets an entry from the json tree, with an optional default value.
+ * If no key is specified it throws the entire tree at you instead
+ */
+/datum/json_savefile/proc/get_entry(key, default_value)
+	if(!key)
+		return tree
+	return (key in tree) ? tree[key] : default_value
+
+/// Sets an entry in the tree to the given value
+/datum/json_savefile/proc/set_entry(key, value)
+	tree[key] = value
+	if(auto_save)
+		save()
+
+/// Removes the given key from the tree
+/datum/json_savefile/proc/remove_entry(key)
+	if(key)
+		tree -= key
+	if(auto_save)
+		save()
+
+/// Wipes the entire tree
+/datum/json_savefile/proc/wipe()
+	tree?.Cut()
+
+/datum/json_savefile/proc/load()
+	if(!path || !fexists(path))
+		return FALSE
+	try
+		tree = json_decode(rustg_file_read(path))
+		return TRUE
+	catch(var/exception/err)
+		stack_trace("failed to load json savefile at '[path]': [err]")
+		return FALSE
+
+/datum/json_savefile/proc/save()
+	if(path)
+		rustg_file_write(json_encode(tree, JSON_PRETTY_PRINT), path)
+
+/datum/json_savefile/serialize_list(list/options, list/semvers)
+	SHOULD_CALL_PARENT(FALSE)
+	//SET_SERIALIZATION_SEMVER(semvers, "1.0.0")
+	return tree.Copy()
+
+/// Traverses the entire dir tree of the given savefile and dynamically assembles the tree from it
+/datum/json_savefile/proc/import_byond_savefile(savefile/savefile)
+	tree.Cut()
+	var/list/dirs_to_go = list("/" = tree)
+	while(length(dirs_to_go))
+		var/dir = dirs_to_go[1]
+		var/list/region = dirs_to_go[dir]
+		dirs_to_go.Cut(1, 2)
+		savefile.cd = dir
+		for(var/entry in savefile.dir)
+			var/entry_value
+			savefile.cd = "[dir]/[entry]"
+			//eof refers to the path you are cd'ed into, not the savefile as a whole. being false right after cding into an entry means this entry has no buffer, which only happens with nested save file directories
+			if (savefile.eof)
+				region[entry] = list()
+				dirs_to_go["[dir]/[entry]"] = region[entry]
+				continue
+			READ_FILE(savefile, entry_value) //we are cd'ed to the entry, so we don't need to specify a path to read from
+			region[entry] = entry_value
+
+/* no. not touching this right now.
+/// Proc that handles generating a JSON file (prettified if 515 and over!) of a user's preferences and showing it to them.
+/// Requester is passed in to the ftp() and tgui_alert() procs, and account_name is just used to generate the filename.
+/// We don't _need_ to pass in account_name since this is reliant on the json_savefile datum already knowing what we correspond to, but it's here to help people keep track of their stuff.
+/datum/json_savefile/proc/export_json_to_client(mob/requester, account_name)
+	if(!istype(requester) || !path)
+		return
+
+	if(!json_export_checks(requester))
+		return
+
+	COOLDOWN_START(src, download_cooldown, (CONFIG_GET(number/seconds_cooldown_for_preferences_export) * (1 SECONDS)))
+	var/file_name = "[account_name ? "[account_name]_" : ""]preferences_[time2text(world.timeofday, "MMM_DD_YYYY_hh-mm-ss")].json"
+	var/temporary_file_storage = "data/preferences_export_working_directory/[file_name]"
+
+	if(!text2file(json_encode(tree, JSON_PRETTY_PRINT), temporary_file_storage))
+		tgui_alert(requester, "Failed to export preferences to JSON! You might need to try again later.", "Export Preferences JSON")
+		return
+
+	var/exportable_json = file(temporary_file_storage)
+
+	DIRECT_OUTPUT(requester, ftp(exportable_json, file_name))
+	fdel(temporary_file_storage)
+
+/// Proc that just handles all of the checks for exporting a preferences file, returns TRUE if all checks are passed, FALSE otherwise.
+/// Just done like this to make the code in the export_json_to_client() proc a bit cleaner.
+/datum/json_savefile/proc/json_export_checks(mob/requester)
+	if(!COOLDOWN_FINISHED(src, download_cooldown))
+		tgui_alert(requester, "You must wait [DisplayTimeText(COOLDOWN_TIMELEFT(src, download_cooldown))] before exporting your preferences again!", "Export Preferences JSON")
+		return FALSE
+
+	if(tgui_alert(requester, "Are you sure you want to export your preferences as a JSON file? This will save to a file on your computer.", "Export Preferences JSON", list("Cancel", "Yes")) == "Yes")
+		return TRUE
+
+	return FALSE
+*/

code/datums/mocking/client.dm

@@ -5,3 +5,49 @@
 
 	/// The view of the client, similar to /client/var/view.
 	var/view = "15x15"
+
+	/// View data of the client, similar to /client/var/view_size.
+	var/datum/view_data/view_size
+
+	/// Objects on the screen of the client
+	var/list/screen = list()
+
+	/// The mob the client controls
+	var/mob/mob
+
+	/// The ckey for this mock interface
+	var/ckey = "mockclient"
+
+	/// The key for this mock interface
+	var/key = "mockclient"
+
+	/// client prefs
+	var/fps
+	var/hotkeys
+	var/tgui_say
+	var/typing_indicators
+
+/datum/client_interface/New()
+	..()
+	var/static/mock_client_uid = 0
+	mock_client_uid++
+
+	src.key = "[key]_[mock_client_uid]"
+	ckey = ckey(key)
+
+#ifdef UNIT_TESTS // otherwise this shit can leak into production servers which is drather bad
+	GLOB.directory[ckey] = src
+#endif
+
+/datum/client_interface/Destroy(force)
+	GLOB.directory -= ckey
+	return ..()
+
+/datum/client_interface/proc/IsByondMember()
+	return FALSE
+
+/datum/client_interface/proc/set_macros()
+	return
+
+/datum/client_interface/proc/update_ambience_pref()
+	return

code/game/data_huds.dm

@@ -257,7 +257,7 @@ Security HUDs! Basic mode shows only the job.
 
 /mob/living/carbon/human/proc/sec_hud_set_security_status()
 	var/perpname = get_face_name(get_id_name(""))
-	if(perpname && SSdatacore)
+	if(perpname && SSdatacore.initialized)
 		var/datum/data/record/security/R = SSdatacore.get_record_by_name(name, DATACORE_RECORDS_SECURITY)
 		if(R)
 			var/new_state

code/modules/client/client_procs.dm

@@ -288,6 +288,7 @@ GLOBAL_LIST_INIT(blacklisted_builds, list(
 	prefs = GLOB.preferences_datums[ckey]
 	if(prefs)
 		prefs.parent = src
+		prefs.load_savefile() // just to make sure we have the latest data
 		prefs.apply_all_client_preferences()
 	else
 		prefs = new /datum/preferences(src)

code/modules/client/migrations/convert_to_json_savefile.dm

@@ -0,0 +1,10 @@
+/datum/preferences/proc/try_savefile_type_migration()
+	load_path(parent.ckey, "preferences.sav") // old save file
+	var/old_path = path
+	load_path(parent.ckey)
+	if(!fexists(old_path))
+		return
+	var/datum/json_savefile/json_savefile = new(path)
+	json_savefile.import_byond_savefile(new /savefile(old_path))
+	json_savefile.save()
+	return TRUE

code/modules/client/migrations/migrate_med_sec_fancy_job_titles.dm

@@ -0,0 +1,17 @@
+/datum/preferences/proc/migrate_med_sec_fancy_job_titles()
+	var/list/migrate_jobs = list(
+		"Head of Security" = JOB_SECURITY_MARSHAL,
+		"Detective" = JOB_DETECTIVE,
+		"Medical Doctor" = JOB_ACOLYTE,
+		"Curator" = JOB_ARCHIVIST,
+		"Cargo Technician" = JOB_DECKHAND,
+	)
+
+	var/list/job_prefs = read_preference(/datum/preference/blob/job_priority)
+	for(var/job in job_prefs)
+		if(job in migrate_jobs)
+			var/old_value = job_prefs[job]
+			job_prefs -= job
+			job_prefs[migrate_jobs[job]] = old_value
+	var/datum/preference/blob/job_priority/actual_datum = GLOB.preference_entries[/datum/preference/blob/job_priority]
+	write_preference(/datum/preference/blob/job_priority, actual_datum.serialize(job_prefs))

code/modules/client/migrations/reconcile_v44.dm

@@ -0,0 +1,22 @@
+/datum/preferences/proc/reconcile_v44(current_version)
+	//Due to unsafe edits to the v44 updater, I need to make sure that both states are safely reconciled..
+
+	if(current_version < 44)
+		//If we called the v44 handler in this same session, we have nothing to resolve.
+		return
+
+	var/list/migrate_jobs = list(
+		// If my logic is correct, this state should never occur, but just to ensure all states are stable.
+		"Medical Doctor" = JOB_ACOLYTE,
+		// Pre-Aethering-II Doctor.
+		"General Practitioner" = JOB_ACOLYTE
+	)
+	var/list/job_prefs = read_preference(/datum/preference/blob/job_priority)
+	for(var/job in job_prefs)
+		if(job in migrate_jobs)
+			var/old_value = job_prefs[job]
+			job_prefs -= job
+			job_prefs[migrate_jobs[job]] = old_value
+	var/datum/preference/blob/job_priority/actual_datum = GLOB.preference_entries[/datum/preference/blob/job_priority]
+	write_preference(/datum/preference/blob/job_priority, actual_datum.serialize(job_prefs))
+

code/modules/client/preferences.dm

@@ -4,16 +4,22 @@ GLOBAL_LIST_EMPTY(preferences_datums)
 	var/client/parent
 	//doohickeys for savefiles
 	var/path
-	var/default_slot = 1 //Holder so it doesn't default to slot 1, rather the last one used
-	var/max_save_slots = 10
-
-	//non-preference stuff
-	var/muted = 0
+	/// Whether or not we allow saving/loading. Used for guests, if they're enabled
+	var/load_and_save = TRUE
+	/// Ensures that we always load the last used save, QOL
+	var/default_slot = 1
+	/// The maximum number of slots we're allowed to contain
+	var/max_save_slots = 3
+
+	/// Bitflags for communications that are muted
+	var/muted = NONE
+	/// Last IP that this client has connected from
 	var/last_ip
+	/// Last CID that this client has connected from
 	var/last_id
 
-	//game-preferences
-	var/lastchangelog = "" //Saved changlog filesize to detect if there was a change
+	/// Cached changelog size, to detect new changelogs since last join
+	var/lastchangelog = ""
 
 	/// Custom keybindings. Map of keybind names to keyboard inputs.
 	/// For example, by default would have "swap_hands" -> list("X")
@@ -24,7 +30,7 @@ GLOBAL_LIST_EMPTY(preferences_datums)
 	var/list/key_bindings_by_key = list()
 
 	var/toggles = TOGGLES_DEFAULT
-	var/db_flags
+	var/db_flags = NONE
 	var/chat_toggles = TOGGLES_DEFAULT_CHAT
 	var/ghost_form = "ghost"
 
@@ -52,11 +58,11 @@ GLOBAL_LIST_EMPTY(preferences_datums)
 	/// A list of instantiated middleware
 	var/list/datum/preference_middleware/middleware = list()
 
-	/// The savefile relating to core preferences, PREFERENCE_PLAYER
-	var/savefile/game_savefile
+	/// The json savefile for this datum
+	var/datum/json_savefile/savefile
 
 	/// The savefile relating to character preferences, PREFERENCE_CHARACTER
-	var/savefile/character_savefile
+	var/list/character_data
 
 	/// A list of keys that have been updated since the last save.
 	var/list/recently_updated_keys = list()
@@ -89,19 +95,29 @@ GLOBAL_LIST_EMPTY(preferences_datums)
 	return ..()
 
 /datum/preferences/New(client/C)
+	if(!C)
+		CRASH("Attempted to create preferences without a client or mock.")
 	parent = C
 
 	for (var/middleware_type in subtypesof(/datum/preference_middleware))
 		middleware += new middleware_type(src)
 
 	selected_category = locate(/datum/preference_group/category/general) in GLOB.all_pref_groups
 
-	if(istype(C))
-		if(!is_guest_key(C.key))
-			load_path(C.ckey)
-			unlock_content = !!C.IsByondMember()
-			if(unlock_content)
-				max_save_slots = 15
+	if(istype(C) || istype(C, /datum/client_interface))
+#ifdef UNIT_TESTS
+		load_and_save = !is_guest_key(parent.key) //Treat them as fully real.
+#else
+		load_and_save = !is_guest_key(parent.key) && !istype(C, /datum/client_interface) // Never save mock clients to disk on prod.
+#endif
+		load_path(parent.ckey)
+		if(load_and_save && !fexists(path))
+			try_savefile_type_migration()
+		unlock_content = !!C.IsByondMember() //TG made this a preference level var, I can do that if you want.
+		if(unlock_content)
+			max_save_slots = 15
+
+	load_savefile()
 
 	// give them default keybinds and update their movement keys
 	key_bindings = deep_copy_list(GLOB.default_hotkeys)
@@ -486,17 +502,15 @@ INITIALIZE_IMMEDIATE(/atom/movable/screen/subscreen)
 /datum/preferences/proc/create_character_profiles()
 	var/list/profiles = list()
 
-	var/savefile/savefile = new(path)
 	for (var/index in 1 to max_save_slots)
 		// It won't be updated in the savefile yet, so just read the name directly
 		if (index == default_slot)
 			profiles += read_preference(/datum/preference/name/real_name)
 			continue
 
-		savefile.cd = "/character[index]"
-
-		var/name
-		READ_FILE(savefile["real_name"], name)
+		var/tree_key = "character[index]"
+		var/save_data = savefile.get_entry(tree_key)
+		var/name = save_data?["real_name"]
 
 		if (isnull(name))
 			profiles += null