mirror of
https://github.com/PC-Admin/matrix-moderation-tool.git
synced 2024-12-19 07:00:27 -05:00
139 lines
9.3 KiB
Python
139 lines
9.3 KiB
Python
|
|
import subprocess
|
|
import csv
|
|
import time
|
|
import os
|
|
import requests
|
|
import datetime
|
|
import hardcoded_variables
|
|
|
|
def delete_block_media():
|
|
# Take media_id from user
|
|
media_id = input("\nEnter the media_id of the media you would like to delete and block on your server. (Example: For this media https://matrix.perthchat.org/_matrix/media/r0/download/matrix.org/eDmjusOjnHyFPOYGxlrOsULJ the media_id is 'eDmjusOjnHyFPOYGxlrOsULJ'): ")
|
|
remote_server = input("\nEnter the remote servers URL without the 'https://' (Example: matrix.org): ")
|
|
# find filesystem_id from database
|
|
command_collect_filesystem_id = "ssh " + hardcoded_variables.homeserver_url + """ "/matrix/postgres/bin/cli-non-interactive --dbname=synapse -t -c 'SELECT DISTINCT filesystem_id FROM remote_media_cache WHERE media_id = '\\''""" + media_id + """'\\'" | xargs"""
|
|
print(command_collect_filesystem_id)
|
|
process_collect_filesystem_id = subprocess.run([command_collect_filesystem_id], shell=True, stdout=subprocess.PIPE, universal_newlines=True)
|
|
filesystem_id = process_collect_filesystem_id.stdout
|
|
print(process_collect_filesystem_id.stdout)
|
|
# list the target files on disk
|
|
command_collect_thumbnails = "ssh " + hardcoded_variables.homeserver_url + ' "find /matrix/synapse/storage/media-store/remote_thumbnail/' + remote_server + '/' + filesystem_id[:2] + "/" + filesystem_id[2:4] + "/" + filesystem_id[4:].rstrip() + """ -type f -printf '%p\\n'\""""
|
|
print(command_collect_thumbnails)
|
|
process_collect_thumbnails = subprocess.run([command_collect_thumbnails], shell=True, stdout=subprocess.PIPE, universal_newlines=True)
|
|
remote_thumbnails_list = process_collect_thumbnails.stdout
|
|
print(remote_thumbnails_list)
|
|
command_content_location = "ssh " + hardcoded_variables.homeserver_url + ' "ls /matrix/synapse/storage/media-store/remote_content/' + remote_server + '/' + filesystem_id[:2] + "/" + filesystem_id[2:4] + "/" + filesystem_id[4:].rstrip() + '"'
|
|
print(command_content_location)
|
|
process_content_location = subprocess.run([command_content_location], shell=True, stdout=subprocess.PIPE, universal_newlines=True)
|
|
remote_content_location = process_content_location.stdout
|
|
print(remote_content_location)
|
|
# Zero the target files on disk then chattr +i them
|
|
for line in remote_thumbnails_list.split('\n'):
|
|
if line:
|
|
command_zero_thumbnails = 'ssh ' + hardcoded_variables.homeserver_url + ' "true > ' + line + '"'
|
|
print(command_zero_thumbnails)
|
|
process_zero_thumbnails = subprocess.run(command_zero_thumbnails, shell=True)
|
|
print(process_zero_thumbnails.stdout)
|
|
command_make_thumbnail_immutable = 'ssh ' + hardcoded_variables.homeserver_url + ' "chattr +i ' + line + '"'
|
|
print(command_make_thumbnail_immutable)
|
|
process_make_thumbnail_immutable = subprocess.run(command_make_thumbnail_immutable, shell=True)
|
|
print(process_make_thumbnail_immutable.stdout)
|
|
command_zero_media = 'ssh ' + hardcoded_variables.homeserver_url + ' "true > ' + remote_content_location.rstrip() + '"'
|
|
print(command_zero_media)
|
|
process_remove_media = subprocess.run(command_zero_media, shell=True)
|
|
print(process_remove_media.stdout)
|
|
command_make_content_immutable = 'ssh ' + hardcoded_variables.homeserver_url + ' "chattr +i ' + remote_content_location.rstrip() + '"'
|
|
print(command_make_content_immutable)
|
|
process_make_content_immutable = subprocess.run(command_make_content_immutable, shell=True)
|
|
print(process_make_content_immutable.stdout)
|
|
|
|
# Example, first use the media_id to find the filesystem_id:
|
|
# $ ssh matrix.perthchat.org "/matrix/postgres/bin/cli-non-interactive --dbname=synapse -t -c 'SELECT DISTINCT filesystem_id FROM remote_media_cache WHERE media_id = '\''eDmjusOjnHyFPOYGxlrOsULJ'\'" | xargs
|
|
# ehckzWWeUkDhhPfNFkcfCFNv
|
|
|
|
# Then use that filesystem_id to locate the remote file and all it's thumbnails:
|
|
# $ ssh matrix.perthchat.org "find /matrix/synapse/storage/media-store/remote_thumbnail/matrix.org/eh/ck/zWWeUkDhhPfNFkcfCFNv -type f -printf '%p\n'"
|
|
#/matrix/synapse/storage/media-store/remote_thumbnail/matrix.org/eh/ck/zWWeUkDhhPfNFkcfCFNv/32-32-image-jpeg-crop
|
|
#/matrix/synapse/storage/media-store/remote_thumbnail/matrix.org/eh/ck/zWWeUkDhhPfNFkcfCFNv/640-480-image-jpeg-scale
|
|
# ...
|
|
# $ ssh matrix.perthchat.org "ls /matrix/synapse/storage/media-store/remote_content/matrix.org/eh/ck/zWWeUkDhhPfNFkcfCFNv"
|
|
# /matrix/synapse/storage/media-store/remote_content/matrix.org/eh/ck/zWWeUkDhhPfNFkcfCFNv
|
|
|
|
# Then zero each file and make it immutable:
|
|
# $ ssh matrix.perthchat.org "true > /matrix/synapse/storage/media-store/remote_thumbnail/matrix.org/eh/ck/zWWeUkDhhPfNFkcfCFNv/32-32-image-jpeg-crop"
|
|
# $ ssh matrix.perthchat.org "chattr +i /matrix/synapse/storage/media-store/remote_thumbnail/matrix.org/eh/ck/zWWeUkDhhPfNFkcfCFNv/32-32-image-jpeg-crop"
|
|
# $ ssh matrix.perthchat.org "true > /matrix/synapse/storage/media-store/remote_thumbnail/matrix.org/eh/ck/zWWeUkDhhPfNFkcfCFNv/640-480-image-jpeg-scale"
|
|
# $ ssh matrix.perthchat.org "chattr +i /matrix/synapse/storage/media-store/remote_thumbnail/matrix.org/eh/ck/zWWeUkDhhPfNFkcfCFNv/640-480-image-jpeg-scale"
|
|
# ...
|
|
# $ ssh matrix.perthchat.org "true > /matrix/synapse/storage/media-store/remote_content/matrix.org/eh/ck/zWWeUkDhhPfNFkcfCFNv"
|
|
# $ ssh matrix.perthchat.org "chattr +i /matrix/synapse/storage/media-store/remote_content/matrix.org/eh/ck/zWWeUkDhhPfNFkcfCFNv"
|
|
|
|
def purge_remote_media_repo():
|
|
purge_from = int(input("\nEnter the number of days to purge from: "))
|
|
purge_too = int(input("\nEnter the number of days to purge too: "))
|
|
|
|
while purge_from >= purge_too:
|
|
# Calculate the epoch timestamp for 'purge_from' days ago
|
|
epoch_time = int((datetime.datetime.now() - datetime.timedelta(days=purge_from)).timestamp())
|
|
# Convert to milliseconds (as per your original code)
|
|
epoch_time_millis = epoch_time * 1000
|
|
|
|
# Make the request
|
|
headers = {"Authorization": "Bearer " + hardcoded_variables.access_token}
|
|
url = f"https://{hardcoded_variables.homeserver_url}/_synapse/admin/v1/purge_media_cache"
|
|
params = {"before_ts": epoch_time_millis}
|
|
response = requests.post(url, headers=headers, params=params)
|
|
|
|
print(response.text)
|
|
|
|
purge_from -= 1
|
|
time.sleep(2)
|
|
|
|
# This loop is quite slow, our server was having disk issues.
|
|
print("Done! :)")
|
|
|
|
# Example:
|
|
# $ date --date '149 days ago' +%s
|
|
# 1589442217
|
|
# $ curl -X POST --header "Authorization: Bearer ACCESS_TOKEN" 'https://matrix.perthchat.org/_synapse/admin/v1/purge_media_cache?before_ts=1589439628000'
|
|
|
|
def prepare_database_copy_of_multiple_rooms():
|
|
print("Preparing database copying of events from multiple rooms selected\n")
|
|
print("This command needs to be run on the target server as root, it will setup postgres commands to download the join-leave events and all-events from a list of rooms.\n\nIt mounts a ramdisk beforehand at /matrix/postgres/data/ramdisk\n\nThis function is only compatible with Spantaleevs Matrix deploy script: https://github.com/spantaleev/matrix-docker-ansible-deploy\n")
|
|
database_copy_list_location = input("Please enter the path of the file containing a newline seperated list of room ids: ")
|
|
with open(database_copy_list_location, newline='') as f:
|
|
reader = csv.reader(f)
|
|
data = list(reader)
|
|
|
|
make_ramdisk_command = "mkdir /matrix/postgres/data/ramdisk; mount -t ramfs -o size=512m ramfs /matrix/postgres/data/ramdisk; chown -R matrix:matrix /matrix/postgres/data/ramdisk"
|
|
make_ramdisk_command_process = subprocess.run([make_ramdisk_command], shell=True, stdout=subprocess.PIPE, universal_newlines=True)
|
|
print(make_ramdisk_command_process.stdout)
|
|
|
|
x = 0
|
|
while x <= (len(data) - 1):
|
|
print(data[x][0])
|
|
roomid_trimmed = data[x][0]
|
|
roomid_trimmed = roomid_trimmed.replace('!', '')
|
|
roomid_trimmed = roomid_trimmed.replace(':', '-')
|
|
os.mkdir("/matrix/postgres/data/ramdisk/" + roomid_trimmed)
|
|
touch_command = "touch /matrix/postgres/data/ramdisk/" + roomid_trimmed + "/dump_room_data.sql"
|
|
touch_command_process = subprocess.run([touch_command], shell=True, stdout=subprocess.PIPE, universal_newlines=True)
|
|
print(touch_command_process.stdout)
|
|
sql_file_contents = "\set ROOMID '" + data[x][0] + "'\nCOPY (SELECT * FROM current_state_events JOIN room_memberships ON room_memberships.event_id = current_state_events.event_id WHERE current_state_events.room_id = :'ROOMID') TO '/var/lib/postgresql/data/ramdisk/" + roomid_trimmed + "/user_join-leave.csv' WITH CSV HEADER;\nCOPY (SELECT * FROM event_json WHERE room_id=:'ROOMID') TO '/var/lib/postgresql/data/ramdisk/" + roomid_trimmed + "/room_events.csv' WITH CSV HEADER;"
|
|
print(sql_file_contents)
|
|
sql_file_location = "/matrix/postgres/data/ramdisk/" + roomid_trimmed + "/dump_room_data.sql"
|
|
sql_file = open(sql_file_location,"w+")
|
|
sql_file.write(sql_file_contents)
|
|
sql_file.close()
|
|
|
|
x += 1
|
|
#print(x)
|
|
time.sleep(1)
|
|
|
|
chown_command = "chown -R matrix:matrix /matrix/postgres/data/ramdisk; docker restart matrix-postgres"
|
|
chown_command_process = subprocess.run([chown_command], shell=True, stdout=subprocess.PIPE, universal_newlines=True)
|
|
print(chown_command_process.stdout)
|
|
|
|
print("\nThe sql query files have been generated, as postgres user in container run:\n# docker exec -it matrix-postgres /bin/bash\nbash-5.0$ export PGPASSWORD=your-db-password\nbash-5.0$ for f in /var/lib/postgresql/data/ramdisk/*/dump_room_data.sql; do psql --host=127.0.0.1 --port=5432 --username=synapse -w -f $f; done\n\nAfter copying the data to a cloud location law enforcement can access, clean up the ramdisk like so:\n# rm -r /matrix/postgres/data/ramdisk/*\n# umount /matrix/postgres/data/ramdisk")
|