From bfa61acf31dbefd79156158bb2b6f2d3d1dabab2 Mon Sep 17 00:00:00 2001 From: gravel Date: Sun, 21 Jan 2024 13:33:42 +0000 Subject: [PATCH] feat: archive servers.json files --- etc/archives/archive-recent.sh | 18 ++++++++++++++++++ etc/archives/logrotate.conf | 22 ++++++++++++++++++++++ etc/archives/logrotate.sh | 4 ++++ etc/archives/logrotate.status | 3 +++ etc/systemd/sessioncommunities.service | 2 +- php/fetch-servers.php | 6 +++++- php/utils/getopt.php | 4 +++- 7 files changed, 56 insertions(+), 3 deletions(-) create mode 100755 etc/archives/archive-recent.sh create mode 100644 etc/archives/logrotate.conf create mode 100755 etc/archives/logrotate.sh create mode 100644 etc/archives/logrotate.status diff --git a/etc/archives/archive-recent.sh b/etc/archives/archive-recent.sh new file mode 100755 index 0000000..ba5ed8a --- /dev/null +++ b/etc/archives/archive-recent.sh @@ -0,0 +1,18 @@ +#!/bin/sh +self="$(basename "$0")" + +files_daily=$((24*60/45)) +# TODO: Increase these for compression ratio +# Values are low for test run +recent_file_limit=$((files_daily * 2)) +recent_file_archive_size=$((files_daily)) + +archives_dir="cache-lt/archive/servers" +base_file_name="$archives_dir/recent/servers.json" +num_recent_files=$(/bin/ls -1 $base_file_name* 2>/dev/null | wc -l) +if [ "$num_recent_files" -ge "$recent_file_limit" ]; then + >&2 echo "$self: recent file limit reached, compressing" + /bin/ls -1tr $base_file_name* | + head -n "$recent_file_archive_size" | + xargs tar cf "$archives_dir/servers.tar" --remove-files +fi diff --git a/etc/archives/logrotate.conf b/etc/archives/logrotate.conf new file mode 100644 index 0000000..b752f9b --- /dev/null +++ b/etc/archives/logrotate.conf @@ -0,0 +1,22 @@ +"output/servers.json" { + rotate -1 + olddir ../cache-lt/archive/servers/recent + createolddir 700 + nocompress + dateext + dateformat -%Y-%m-%d-%H:%M:%S + copy + prerotate + etc/archives/archive-recent.sh + endscript +} + +"cache-lt/archive/servers/servers.tar" { + rotate -1 + missingok + compress + compresscmd /usr/bin/zstd + compressext .zst + compressoptions -T0 --long -19 + uncompresscmd /usr/bin/unzstd +} diff --git a/etc/archives/logrotate.sh b/etc/archives/logrotate.sh new file mode 100755 index 0000000..459e330 --- /dev/null +++ b/etc/archives/logrotate.sh @@ -0,0 +1,4 @@ +#!/bin/sh +cd "$(dirname "$0")" || exit +while ! [ -f ".phpenv.php" ]; do cd ..; done +logrotate -f -s etc/archives/logrotate.status etc/archives/logrotate.conf diff --git a/etc/archives/logrotate.status b/etc/archives/logrotate.status new file mode 100644 index 0000000..dd7ea21 --- /dev/null +++ b/etc/archives/logrotate.status @@ -0,0 +1,3 @@ +logrotate state -- version 2 +"output/servers.json" 2024-1-21-13:32:41 +"cache-lt/archive/servers/servers.tar" 2024-1-21-13:0:0 diff --git a/etc/systemd/sessioncommunities.service b/etc/systemd/sessioncommunities.service index a546c15..f728eb1 100644 --- a/etc/systemd/sessioncommunities.service +++ b/etc/systemd/sessioncommunities.service @@ -5,4 +5,4 @@ Description=Scrape every known source for Session Communities and put the data t [Service] WorkingDirectory=/home/session/sessioncommunities.online User=session -ExecStart=/usr/bin/php php/update-all.php +ExecStart=/usr/bin/sh -c "/usr/bin/php php/update-all.php --verbose --archive --no-color 2>&1 | /usr/bin/tee sessioncommunities.log" diff --git a/php/fetch-servers.php b/php/fetch-servers.php index da04197..6777758 100644 --- a/php/fetch-servers.php +++ b/php/fetch-servers.php @@ -26,7 +26,8 @@ * 6. De-dupe servers based on pubkey */ function main() { - global $CACHE_ROOT, $ROOMS_FILE, $TAGS_FILE, $KNOWN_SERVERS, $KNOWN_PUBKEYS, $DO_DRY_RUN; + global $PROJECT_ROOT, $CACHE_ROOT, $ROOMS_FILE, $TAGS_FILE, + $KNOWN_SERVERS, $KNOWN_PUBKEYS, $DO_DRY_RUN, $DO_ARCHIVE_FILES; // Create default directories.. file_exists($CACHE_ROOT) or mkdir($CACHE_ROOT, 0700); @@ -68,6 +69,9 @@ if (!$DO_DRY_RUN) { file_put_contents($ROOMS_FILE, json_encode($servers)); file_put_contents($TAGS_FILE, CommunityTag::serializeClassData()); + if ($DO_ARCHIVE_FILES) { + passthru("$PROJECT_ROOT/etc/archives/logrotate.sh", $result_code); + } } } diff --git a/php/utils/getopt.php b/php/utils/getopt.php index edf4648..73e0cac 100644 --- a/php/utils/getopt.php +++ b/php/utils/getopt.php @@ -6,7 +6,7 @@ include_once 'utils/logging.php'; // Read the -v|--verbose option increasing logging verbosity to debug. - $options = getopt("vn", ["verbose", "fast", "no-color", "dry-run"]); + $options = getopt("vn", ["verbose", "fast", "no-color", "dry-run", "archive"]); if (isset($options["v"]) or isset($options["verbose"])) { $LOGGING_VERBOSITY = LoggingVerbosity::Debug; } @@ -19,6 +19,8 @@ LoggingVerbosity::$showColor = false; } + $DO_ARCHIVE_FILES = isset($options["archive"]); + // set timeout for file_get_contents() ini_set('default_socket_timeout', 6); // in seconds, default is 60