feat: archive servers.json files

etc/archives/archive-recent.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+self="$(basename "$0")"
+
+files_daily=$((24*60/45))
+# TODO: Increase these for compression ratio
+# Values are low for test run
+recent_file_limit=$((files_daily * 2))
+recent_file_archive_size=$((files_daily))
+
+archives_dir="cache-lt/archive/servers"
+base_file_name="$archives_dir/recent/servers.json"
+num_recent_files=$(/bin/ls -1 $base_file_name* 2>/dev/null | wc -l)
+if [ "$num_recent_files" -ge "$recent_file_limit" ]; then
+	>&2 echo "$self: recent file limit reached, compressing"
+	/bin/ls -1tr $base_file_name* |
+	head -n "$recent_file_archive_size" |
+	xargs tar cf "$archives_dir/servers.tar" --remove-files
+fi

etc/archives/logrotate.conf

@@ -0,0 +1,22 @@
+"output/servers.json" {
+	rotate -1
+	olddir ../cache-lt/archive/servers/recent
+	createolddir 700
+	nocompress
+	dateext
+	dateformat -%Y-%m-%d-%H:%M:%S
+	copy
+	prerotate
+		etc/archives/archive-recent.sh
+	endscript
+}
+
+"cache-lt/archive/servers/servers.tar" {
+	rotate -1
+	missingok
+	compress
+	compresscmd /usr/bin/zstd
+	compressext .zst
+	compressoptions -T0 --long -19
+	uncompresscmd /usr/bin/unzstd
+}

etc/archives/logrotate.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+cd "$(dirname "$0")" || exit
+while ! [ -f ".phpenv.php" ]; do cd ..; done
+logrotate -f -s etc/archives/logrotate.status etc/archives/logrotate.conf

etc/archives/logrotate.status

@@ -0,0 +1,3 @@
+logrotate state -- version 2
+"output/servers.json" 2024-1-21-13:32:41
+"cache-lt/archive/servers/servers.tar" 2024-1-21-13:0:0

etc/systemd/sessioncommunities.service

@@ -5,4 +5,4 @@ Description=Scrape every known source for Session Communities and put the data t
 [Service]
 WorkingDirectory=/home/session/sessioncommunities.online
 User=session
-ExecStart=/usr/bin/php php/update-all.php
+ExecStart=/usr/bin/sh -c "/usr/bin/php php/update-all.php --verbose --archive --no-color 2>&1 | /usr/bin/tee sessioncommunities.log"

php/fetch-servers.php

@@ -26,7 +26,8 @@
 	 * 6. De-dupe servers based on pubkey
 	 */
 	function main() {
-		global $CACHE_ROOT, $ROOMS_FILE, $TAGS_FILE, $KNOWN_SERVERS, $KNOWN_PUBKEYS, $DO_DRY_RUN;
+		global $PROJECT_ROOT, $CACHE_ROOT, $ROOMS_FILE, $TAGS_FILE,
+			$KNOWN_SERVERS, $KNOWN_PUBKEYS, $DO_DRY_RUN, $DO_ARCHIVE_FILES;
 
 		// Create default directories..
 		file_exists($CACHE_ROOT) or mkdir($CACHE_ROOT, 0700);
@@ -68,6 +69,9 @@
 		if (!$DO_DRY_RUN) {
 			file_put_contents($ROOMS_FILE, json_encode($servers));
 			file_put_contents($TAGS_FILE, CommunityTag::serializeClassData());
+			if ($DO_ARCHIVE_FILES) {
+				passthru("$PROJECT_ROOT/etc/archives/logrotate.sh", $result_code);
+			}
 		}
 	}
 

php/utils/getopt.php

@@ -6,7 +6,7 @@
 	include_once 'utils/logging.php';
 
 	// Read the -v|--verbose option increasing logging verbosity to debug.
-	$options = getopt("vn", ["verbose", "fast", "no-color", "dry-run"]);
+	$options = getopt("vn", ["verbose", "fast", "no-color", "dry-run", "archive"]);
 	if (isset($options["v"]) or isset($options["verbose"])) {
 		$LOGGING_VERBOSITY = LoggingVerbosity::Debug;
 	}
@@ -19,6 +19,8 @@
 		LoggingVerbosity::$showColor = false;
 	}
 
+	$DO_ARCHIVE_FILES = isset($options["archive"]);
+
 	// set timeout for file_get_contents()
 	ini_set('default_socket_timeout', 6); // in seconds, default is 60