From 591d97ac4055f410cebf9766d6788cc72a961188 Mon Sep 17 00:00:00 2001 From: susanodd Date: Tue, 26 Nov 2024 15:21:58 +0100 Subject: [PATCH 01/24] #1397: Command to delete excess backup videos and reorder versions --- signbank/video/admin.py | 46 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 3c1888f33..0f574c770 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -89,12 +89,57 @@ def matching_file_exists(videofile, key): return queryset.all() +@admin.action(description="Delete backup videos") +def remove_backups(modeladmin, request, queryset): + import os + glosses_in_queryset = [obj.gloss for obj in queryset if obj.version > 0] + distinct_glosses = list(set(glosses_in_queryset)) + lookup_backup_files = dict() + for gloss in distinct_glosses: + lookup_backup_files[gloss] = GlossVideo.objects.filter(gloss=gloss, version__gt=0).order_by('version') + for obj in queryset: + # unlink all the files + if obj.version == 0: + # skip version 0 video + continue + video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) + if os.path.exists(video_file_full_path): + try: + # os.unlink(obj.videofile.path) + print('unlink ', video_file_full_path) + except (OSError, PermissionError): + print('could not delete video file: ', video_file_full_path) + remaining_backup_files = dict() + for gloss, videos in lookup_backup_files.items(): + cleaned_up_videos = [] + for video in videos: + print(video) + video_file_full_path = os.path.join(WRITABLE_FOLDER, str(video)) + if os.path.exists(video_file_full_path): + # file was not deleted in previous step + cleaned_up_videos.append(video) + continue + try: + # video.delete() + print('delete video') + except (OSError, PermissionError): + cleaned_up_videos.append(video) + remaining_backup_files[gloss] = cleaned_up_videos + print(remaining_backup_files) + for gloss, videos in remaining_backup_files.items(): + for inx, video in enumerate(videos, 1): + print('gloss video version ', inx) + # video.version = inx + # video.save() + + class GlossVideoAdmin(admin.ModelAdmin): list_display = ['id', 'gloss', 'video_file', 'perspective', 'NME', 'file_timestamp', 'file_group', 'permissions', 'file_size', 'version'] list_filter = (GlossVideoDatasetFilter, GlossVideoFileSystemGroupFilter, GlossVideoExistenceFilter) search_fields = ['^gloss__annotationidglosstranslation__text'] + actions = [remove_backups] def video_file(self, obj=None): # this will display the full path in the list view @@ -181,6 +226,7 @@ def has_delete_permission(self, request, obj=None): return False + class GlossVideoHistoryDatasetFilter(admin.SimpleListFilter): title = _('Dataset') From c9608be2a49d9d98237d9c893b150e85856902c3 Mon Sep 17 00:00:00 2001 From: susanodd Date: Wed, 27 Nov 2024 13:02:58 +0100 Subject: [PATCH 02/24] #1397, #1398: Added renaming of backup files to those that remain after selection was deleted, version numbers adjusted for new set. Change extension of backup file if it has the wrong video extension for its format. --- signbank/video/admin.py | 120 ++++++++++++++++++++++++++++++---------- 1 file changed, 91 insertions(+), 29 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 0f574c770..4feb6b079 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -1,3 +1,5 @@ +import os.path + from django.contrib import admin from django import forms from django.db import models @@ -5,9 +7,12 @@ from signbank.dictionary.models import Dataset, AnnotatedGloss from django.contrib.auth.models import User from signbank.settings.base import * -from signbank.settings.server_specific import WRITABLE_FOLDER, FILESYSTEM_SIGNBANK_GROUPS +from signbank.settings.server_specific import WRITABLE_FOLDER, FILESYSTEM_SIGNBANK_GROUPS, DEBUG_VIDEOS from django.utils.translation import override, gettext_lazy as _ from django.db.models import Q, Count, CharField, TextField, Value as V +from signbank.tools import get_two_letter_dir +# from signbank.video.convertvideo import probe_format +import subprocess class GlossVideoDatasetFilter(admin.SimpleListFilter): @@ -89,46 +94,104 @@ def matching_file_exists(videofile, key): return queryset.all() -@admin.action(description="Delete backup videos") +def rename_extension_video(glossvideo): + if glossvideo.version == 0: + return + + video_file_full_path = os.path.join(WRITABLE_FOLDER, str(glossvideo.videofile)) + if not os.path.exists(video_file_full_path): + return + + # the video is a backup video that exists on the file system + base_filename = os.path.basename(video_file_full_path) + + idgloss = glossvideo.gloss.idgloss + two_letter_dir = get_two_letter_dir(idgloss) + dataset_dir = glossvideo.gloss.lemma.dataset.acronym + desired_filename_without_extension = idgloss + '-' + str(glossvideo.gloss.id) + + filetype_output = subprocess.run(["file", video_file_full_path], stdout=subprocess.PIPE) + filetype = str(filetype_output.stdout) + if 'MOV' in filetype: + desired_video_extension = '.mov' + elif 'M4V' in filetype: + desired_video_extension = 'm4v' + elif 'MP4' in filetype: + desired_video_extension = '.mp4' + elif 'Matroska' in filetype: + desired_video_extension = '.webm' + else: + if DEBUG_VIDEOS: + print('video:admin:remove_backups:rename_extension_videos:file:UNKNOWN ', filetype) + desired_video_extension = '.mp4' + + desired_extension = desired_video_extension + '.bak' + str(glossvideo.id) + + desired_filename = desired_filename_without_extension + desired_extension + + if base_filename == desired_filename: + if DEBUG_VIDEOS: + print('video:admin:remove_backups:rename_extension_videos:basename: OKAY', base_filename) + return + + current_relative_path = str(glossvideo.videofile) + + source = os.path.join(WRITABLE_FOLDER, current_relative_path) + destination = os.path.join(WRITABLE_FOLDER, GLOSS_VIDEO_DIRECTORY, + dataset_dir, two_letter_dir, desired_filename) + print('video:admin:remove_backups:rename_extension_videos:rename: ', source, destination) + + # os.rename(source, destination) + # glossvideo.videofile.name = desired_filename + # glossvideo.save() + + +@admin.action(description="Delete backup videos for glosses") def remove_backups(modeladmin, request, queryset): import os - glosses_in_queryset = [obj.gloss for obj in queryset if obj.version > 0] + # retrieve glosses of selected GlossVideo objects for later step + glosses_in_queryset = [obj.gloss for obj in queryset] distinct_glosses = list(set(glosses_in_queryset)) - lookup_backup_files = dict() - for gloss in distinct_glosses: - lookup_backup_files[gloss] = GlossVideo.objects.filter(gloss=gloss, version__gt=0).order_by('version') for obj in queryset: # unlink all the files + relative_path = str(obj.videofile) if obj.version == 0: - # skip version 0 video + # skip version 0 video, the user may have selected these in the queryset + if DEBUG_VIDEOS: + print('video:admin:remove_backups:ignore: ', relative_path) continue video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) if os.path.exists(video_file_full_path): + # remove the video file so the GlossVideo object can be deleteds + # this is in addition to the signal pre_delete of a GlossVideo object, which may not delete the files try: # os.unlink(obj.videofile.path) - print('unlink ', video_file_full_path) + if DEBUG_VIDEOS: + print('video:admin:remove_backups:unlink: ', relative_path) + # os.remove(video_file_full_path) + if DEBUG_VIDEOS: + print('video:admin:remove_backups:remove: ', video_file_full_path) except (OSError, PermissionError): - print('could not delete video file: ', video_file_full_path) - remaining_backup_files = dict() - for gloss, videos in lookup_backup_files.items(): - cleaned_up_videos = [] - for video in videos: - print(video) - video_file_full_path = os.path.join(WRITABLE_FOLDER, str(video)) - if os.path.exists(video_file_full_path): - # file was not deleted in previous step - cleaned_up_videos.append(video) + if DEBUG_VIDEOS: + print('Exception video:admin:remove_backups: could not delete video file: ', video_file_full_path) continue - try: - # video.delete() - print('delete video') - except (OSError, PermissionError): - cleaned_up_videos.append(video) - remaining_backup_files[gloss] = cleaned_up_videos - print(remaining_backup_files) - for gloss, videos in remaining_backup_files.items(): + # only backup videos are deleted here + if DEBUG_VIDEOS: + print('video:admin:remove_backups:delete: ', relative_path) + # obj.delete() + # construct data structure for glosses and remaining backup videos that were not selected + lookup_backup_files = dict() + for gloss in distinct_glosses: + lookup_backup_files[gloss] = GlossVideo.objects.filter(gloss=gloss, version__gt=0).order_by('version', 'id') + for gloss, videos in lookup_backup_files.items(): + # enumerate over the backup videos and give them new version numbers for inx, video in enumerate(videos, 1): - print('gloss video version ', inx) + if DEBUG_VIDEOS: + original_version = video.version + print('video:admin:remove_backups:reversion ', original_version, inx, str(video.videofile)) + # if the file has an old bak-format, its name is fixed here + rename_extension_video(video) + # then the version of the gloss video object is updated since objects may have been deleted # video.version = inx # video.save() @@ -138,7 +201,7 @@ class GlossVideoAdmin(admin.ModelAdmin): list_display = ['id', 'gloss', 'video_file', 'perspective', 'NME', 'file_timestamp', 'file_group', 'permissions', 'file_size', 'version'] list_filter = (GlossVideoDatasetFilter, GlossVideoFileSystemGroupFilter, GlossVideoExistenceFilter) - search_fields = ['^gloss__annotationidglosstranslation__text'] + search_fields = ['^gloss__annotationidglosstranslation__text', '^gloss__lemma__lemmaidglosstranslation__text'] actions = [remove_backups] def video_file(self, obj=None): @@ -226,7 +289,6 @@ def has_delete_permission(self, request, obj=None): return False - class GlossVideoHistoryDatasetFilter(admin.SimpleListFilter): title = _('Dataset') From aa2ec41132ecc296167d2ff7787ec71f6ad02e8d Mon Sep 17 00:00:00 2001 From: susanodd Date: Thu, 28 Nov 2024 16:37:24 +0100 Subject: [PATCH 03/24] #1397, #1398: Split into two commands, refined code as per review --- signbank/video/admin.py | 110 ++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 4feb6b079..db5f7d951 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -4,7 +4,7 @@ from django import forms from django.db import models from signbank.video.models import GlossVideo, GlossVideoHistory, AnnotatedVideo, ExampleVideoHistory -from signbank.dictionary.models import Dataset, AnnotatedGloss +from signbank.dictionary.models import Dataset, AnnotatedGloss, Gloss from django.contrib.auth.models import User from signbank.settings.base import * from signbank.settings.server_specific import WRITABLE_FOLDER, FILESYSTEM_SIGNBANK_GROUPS, DEBUG_VIDEOS @@ -94,72 +94,72 @@ def matching_file_exists(videofile, key): return queryset.all() -def rename_extension_video(glossvideo): - if glossvideo.version == 0: - return - - video_file_full_path = os.path.join(WRITABLE_FOLDER, str(glossvideo.videofile)) - if not os.path.exists(video_file_full_path): - return - - # the video is a backup video that exists on the file system - base_filename = os.path.basename(video_file_full_path) - - idgloss = glossvideo.gloss.idgloss - two_letter_dir = get_two_letter_dir(idgloss) - dataset_dir = glossvideo.gloss.lemma.dataset.acronym - desired_filename_without_extension = idgloss + '-' + str(glossvideo.gloss.id) - - filetype_output = subprocess.run(["file", video_file_full_path], stdout=subprocess.PIPE) - filetype = str(filetype_output.stdout) - if 'MOV' in filetype: - desired_video_extension = '.mov' - elif 'M4V' in filetype: - desired_video_extension = 'm4v' - elif 'MP4' in filetype: - desired_video_extension = '.mp4' - elif 'Matroska' in filetype: - desired_video_extension = '.webm' - else: - if DEBUG_VIDEOS: - print('video:admin:remove_backups:rename_extension_videos:file:UNKNOWN ', filetype) - desired_video_extension = '.mp4' +@admin.action(description="Rename extension of backup videos for glosses") +def rename_extension_videos(modeladmin, request, queryset): + import os + # retrieve glosses of selected GlossVideo objects for later step + distinct_glosses = Gloss.objects.filter(glossvideo__in=queryset).distinct() - desired_extension = desired_video_extension + '.bak' + str(glossvideo.id) + lookup_backup_files = dict() + for gloss in distinct_glosses: + for glossvideo in GlossVideo.objects.filter(gloss=gloss, version__gt=0).order_by('version', 'id'): - desired_filename = desired_filename_without_extension + desired_extension + video_file_full_path = os.path.join(WRITABLE_FOLDER, str(glossvideo.videofile)) + if not os.path.exists(video_file_full_path): + continue - if base_filename == desired_filename: - if DEBUG_VIDEOS: - print('video:admin:remove_backups:rename_extension_videos:basename: OKAY', base_filename) - return + # the video is a backup video that exists on the file system + base_filename = os.path.basename(video_file_full_path) + + idgloss = gloss.idgloss + two_letter_dir = get_two_letter_dir(idgloss) + dataset_dir = gloss.lemma.dataset.acronym + desired_filename_without_extension = idgloss + '-' + str(gloss.id) + + filetype_output = subprocess.run(["file", video_file_full_path], stdout=subprocess.PIPE) + filetype = str(filetype_output.stdout) + if 'MOV' in filetype: + desired_video_extension = '.mov' + elif 'M4V' in filetype: + desired_video_extension = 'm4v' + elif 'MP4' in filetype: + desired_video_extension = '.mp4' + elif 'Matroska' in filetype: + desired_video_extension = '.webm' + else: + if DEBUG_VIDEOS: + print('video:admin:remove_backups:rename_extension_videos:file:UNKNOWN ', filetype) + desired_video_extension = '.mp4' + + desired_extension = desired_video_extension + '.bak' + str(glossvideo.id) - current_relative_path = str(glossvideo.videofile) + desired_filename = desired_filename_without_extension + desired_extension - source = os.path.join(WRITABLE_FOLDER, current_relative_path) - destination = os.path.join(WRITABLE_FOLDER, GLOSS_VIDEO_DIRECTORY, - dataset_dir, two_letter_dir, desired_filename) - print('video:admin:remove_backups:rename_extension_videos:rename: ', source, destination) + if base_filename == desired_filename: + if DEBUG_VIDEOS: + print('video:admin:remove_backups:rename_extension_videos:basename: OKAY', base_filename) + continue - # os.rename(source, destination) - # glossvideo.videofile.name = desired_filename - # glossvideo.save() + current_relative_path = str(glossvideo.videofile) + + source = os.path.join(WRITABLE_FOLDER, current_relative_path) + destination = os.path.join(WRITABLE_FOLDER, GLOSS_VIDEO_DIRECTORY, + dataset_dir, two_letter_dir, desired_filename) + print('video:admin:remove_backups:rename_extension_videos:rename: ', source, destination) + + # os.rename(source, destination) + # glossvideo.videofile.name = desired_filename + # glossvideo.save() @admin.action(description="Delete backup videos for glosses") def remove_backups(modeladmin, request, queryset): import os # retrieve glosses of selected GlossVideo objects for later step - glosses_in_queryset = [obj.gloss for obj in queryset] - distinct_glosses = list(set(glosses_in_queryset)) - for obj in queryset: + distinct_glosses = Gloss.objects.filter(glossvideo__in=queryset).distinct() + for obj in queryset.filter(version__gt=0): # unlink all the files relative_path = str(obj.videofile) - if obj.version == 0: - # skip version 0 video, the user may have selected these in the queryset - if DEBUG_VIDEOS: - print('video:admin:remove_backups:ignore: ', relative_path) - continue video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) if os.path.exists(video_file_full_path): # remove the video file so the GlossVideo object can be deleteds @@ -190,7 +190,7 @@ def remove_backups(modeladmin, request, queryset): original_version = video.version print('video:admin:remove_backups:reversion ', original_version, inx, str(video.videofile)) # if the file has an old bak-format, its name is fixed here - rename_extension_video(video) + # rename_extension_video(video) # then the version of the gloss video object is updated since objects may have been deleted # video.version = inx # video.save() @@ -202,7 +202,7 @@ class GlossVideoAdmin(admin.ModelAdmin): list_filter = (GlossVideoDatasetFilter, GlossVideoFileSystemGroupFilter, GlossVideoExistenceFilter) search_fields = ['^gloss__annotationidglosstranslation__text', '^gloss__lemma__lemmaidglosstranslation__text'] - actions = [remove_backups] + actions = [remove_backups, rename_extension_videos] def video_file(self, obj=None): # this will display the full path in the list view From cae023a19af5ad915a910d200507d937b20cfa6e Mon Sep 17 00:00:00 2001 From: susanodd Date: Fri, 29 Nov 2024 13:54:13 +0100 Subject: [PATCH 04/24] #1397, #1398: Added filter on video file type mp4 true false change file extension command, match file type --- signbank/video/admin.py | 85 +++++++++++++++++++++++----------- signbank/video/convertvideo.py | 24 ++++++++++ 2 files changed, 81 insertions(+), 28 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index db5f7d951..8fa826ed7 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -11,7 +11,7 @@ from django.utils.translation import override, gettext_lazy as _ from django.db.models import Q, Count, CharField, TextField, Value as V from signbank.tools import get_two_letter_dir -# from signbank.video.convertvideo import probe_format +from signbank.video.convertvideo import video_file_type_extension import subprocess @@ -94,15 +94,46 @@ def matching_file_exists(videofile, key): return queryset.all() -@admin.action(description="Rename extension of backup videos for glosses") +class GlossVideoFileTypeFilter(admin.SimpleListFilter): + + title = _('MP4 File') + parameter_name = 'file_type' + + def lookups(self, request, model_admin): + file_type = tuple((b, b) for b in ('True', 'False')) + return file_type + + def queryset(self, request, queryset): + + def matching_file_type(videofile, key): + if not key: + return False + from pathlib import Path + video_file_full_path = Path(WRITABLE_FOLDER, videofile) + if video_file_full_path.exists(): + file_extension = video_file_type_extension(video_file_full_path) + return key == str((file_extension == '.mp4')) + else: + return key == 'False' + + queryset_res = queryset.values('id', 'videofile') + results = [qv['id'] for qv in queryset_res + if matching_file_type(qv['videofile'], self.value())] + + if self.value(): + return queryset.filter(id__in=results) + else: + return queryset.all() + + +@admin.action(description="Rename extension to match video type") def rename_extension_videos(modeladmin, request, queryset): import os # retrieve glosses of selected GlossVideo objects for later step distinct_glosses = Gloss.objects.filter(glossvideo__in=queryset).distinct() - lookup_backup_files = dict() for gloss in distinct_glosses: - for glossvideo in GlossVideo.objects.filter(gloss=gloss, version__gt=0).order_by('version', 'id'): + for glossvideo in GlossVideo.objects.filter(gloss=gloss).order_by('version', 'id'): video_file_full_path = os.path.join(WRITABLE_FOLDER, str(glossvideo.videofile)) if not os.path.exists(video_file_full_path): @@ -116,28 +147,17 @@ def rename_extension_videos(modeladmin, request, queryset): dataset_dir = gloss.lemma.dataset.acronym desired_filename_without_extension = idgloss + '-' + str(gloss.id) - filetype_output = subprocess.run(["file", video_file_full_path], stdout=subprocess.PIPE) - filetype = str(filetype_output.stdout) - if 'MOV' in filetype: - desired_video_extension = '.mov' - elif 'M4V' in filetype: - desired_video_extension = 'm4v' - elif 'MP4' in filetype: - desired_video_extension = '.mp4' - elif 'Matroska' in filetype: - desired_video_extension = '.webm' - else: - if DEBUG_VIDEOS: - print('video:admin:remove_backups:rename_extension_videos:file:UNKNOWN ', filetype) - desired_video_extension = '.mp4' + # use the file system command 'file' to determine the extension for the type of video file + desired_video_extension = video_file_type_extension(video_file_full_path) - desired_extension = desired_video_extension + '.bak' + str(glossvideo.id) + if glossvideo.version > 0: + desired_extension = desired_video_extension + '.bak' + str(glossvideo.id) + else: + desired_extension = desired_video_extension desired_filename = desired_filename_without_extension + desired_extension if base_filename == desired_filename: - if DEBUG_VIDEOS: - print('video:admin:remove_backups:rename_extension_videos:basename: OKAY', base_filename) continue current_relative_path = str(glossvideo.videofile) @@ -152,7 +172,7 @@ def rename_extension_videos(modeladmin, request, queryset): # glossvideo.save() -@admin.action(description="Delete backup videos for glosses") +@admin.action(description="Delete selected backup videos and renumber remaining backups") def remove_backups(modeladmin, request, queryset): import os # retrieve glosses of selected GlossVideo objects for later step @@ -166,18 +186,16 @@ def remove_backups(modeladmin, request, queryset): # this is in addition to the signal pre_delete of a GlossVideo object, which may not delete the files try: # os.unlink(obj.videofile.path) - if DEBUG_VIDEOS: - print('video:admin:remove_backups:unlink: ', relative_path) # os.remove(video_file_full_path) if DEBUG_VIDEOS: - print('video:admin:remove_backups:remove: ', video_file_full_path) + print('video:admin:remove_backups:remove file: ', video_file_full_path) except (OSError, PermissionError): if DEBUG_VIDEOS: print('Exception video:admin:remove_backups: could not delete video file: ', video_file_full_path) continue # only backup videos are deleted here if DEBUG_VIDEOS: - print('video:admin:remove_backups:delete: ', relative_path) + print('video:admin:remove_backups:delete object: ', relative_path) # obj.delete() # construct data structure for glosses and remaining backup videos that were not selected lookup_backup_files = dict() @@ -198,8 +216,8 @@ def remove_backups(modeladmin, request, queryset): class GlossVideoAdmin(admin.ModelAdmin): - list_display = ['id', 'gloss', 'video_file', 'perspective', 'NME', 'file_timestamp', 'file_group', 'permissions', 'file_size', 'version'] - list_filter = (GlossVideoDatasetFilter, GlossVideoFileSystemGroupFilter, GlossVideoExistenceFilter) + list_display = ['id', 'gloss', 'video_file', 'perspective', 'NME', 'file_timestamp', 'file_group', 'permissions', 'file_size', 'video_type', 'version'] + list_filter = (GlossVideoDatasetFilter, GlossVideoFileSystemGroupFilter, GlossVideoExistenceFilter, GlossVideoFileTypeFilter) search_fields = ['^gloss__annotationidglosstranslation__text', '^gloss__lemma__lemmaidglosstranslation__text'] actions = [remove_backups, rename_extension_videos] @@ -275,6 +293,17 @@ def permissions(self, obj=None): else: return "" + def video_type(self, obj=None): + # if the file exists, this will display its timestamp in the list view + if obj is None: + return "" + import os + video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) + if os.path.exists(video_file_full_path): + return video_file_type_extension(video_file_full_path) + else: + return "" + def get_list_display_links(self, request, list_display): # do not allow the user to view individual revisions in list self.list_display_links = (None, ) diff --git a/signbank/video/convertvideo.py b/signbank/video/convertvideo.py index 1386835d4..4456a6bab 100755 --- a/signbank/video/convertvideo.py +++ b/signbank/video/convertvideo.py @@ -18,6 +18,8 @@ import ffmpeg from subprocess import Popen, PIPE import re +import subprocess +from signbank.settings.server_specific import DEBUG_VIDEOS def parse_ffmpeg_output(text): @@ -209,6 +211,28 @@ def make_thumbnail_video(sourcefile, targetfile): os.remove(temp_target) +def video_file_type_extension(video_file_full_path): + filetype_output = subprocess.run(["file", video_file_full_path], stdout=subprocess.PIPE) + filetype = str(filetype_output.stdout) + if 'MOV' in filetype: + desired_video_extension = '.mov' + elif 'M4V' in filetype: + desired_video_extension = '.m4v' + elif 'MP4' in filetype: + desired_video_extension = '.mp4' + elif 'Matroska' in filetype: + desired_video_extension = '.webm' + elif 'MKV' in filetype: + desired_video_extension = '.mkv' + elif 'MPEG-2' in filetype: + desired_video_extension = '.m2v' + else: + if DEBUG_VIDEOS: + print('video:admin:convertvideo:video_file_type_extension:file:UNKNOWN ', filetype) + desired_video_extension = '.unknown' + return desired_video_extension + + def convert_video(sourcefile, targetfile, force=False): """convert a video to h264 format if force=True, do the conversion even if the video is already From f59a97ffef1821e7f43658cc661aecd1ee13663c Mon Sep 17 00:00:00 2001 From: susanodd Date: Mon, 2 Dec 2024 06:42:23 +0100 Subject: [PATCH 05/24] #1390: Import get two char dir from tools instead of defining. --- signbank/dataset_checks.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/signbank/dataset_checks.py b/signbank/dataset_checks.py index d7630c42a..58664eeba 100644 --- a/signbank/dataset_checks.py +++ b/signbank/dataset_checks.py @@ -2,15 +2,7 @@ from signbank.dictionary.models import * from signbank.video.models import GlossVideo, GlossVideoNME, GlossVideoPerspective - - -def get_two_letter_dir(idgloss): - foldername = idgloss[:2] - - if len(foldername) == 1: - foldername += '-' - - return foldername +from signbank.tools import get_two_letter_dir def gloss_annotations_check(dataset): From c080da72857c50ddcda0b291a1bcfcf7b8bb4622 Mon Sep 17 00:00:00 2001 From: susanodd Date: Tue, 7 Jan 2025 13:25:07 +0100 Subject: [PATCH 06/24] #1398: Added video admin filters for NME, Perspective videos plus filter for wrongly named files that do not match the type of object. --- signbank/video/admin.py | 143 +++++++++++++++++++++------------ signbank/video/convertvideo.py | 19 ++++- 2 files changed, 109 insertions(+), 53 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 8fa826ed7..27da49adf 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -94,6 +94,90 @@ def matching_file_exists(videofile, key): return queryset.all() +class GlossVideoFilenameFilter(admin.SimpleListFilter): + + title = _('Filename Correct') + parameter_name = 'filename_correct' + + def lookups(self, request, model_admin): + file_exists = tuple((b, b) for b in ('True', 'False')) + return file_exists + + def queryset(self, request, queryset): + import re + + def filename_matches_nme(filename): + filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) + try: + if m := re.search(r".+-(\d+)_(nme_\d+|nme_\d+_left|nme_\d+_right)$", filename_without_extension): + return 'True' + return 'False' + except (IndexError, ValueError): + return 'False' + + def filename_matches_perspective(filename): + filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) + try: + if m := re.search(r".+-(\d+)_(left|right|nme_\d+_left|nme_\d+_right)$", filename_without_extension): + return 'True' + return 'False' + except (IndexError, ValueError): + return 'False' + + def matching_filename(videofile, nmevideo, perspective, key): + if not key: + return False + from pathlib import Path + video_file_full_path = Path(WRITABLE_FOLDER, videofile) + if nmevideo: + filename_correct = filename_matches_nme(video_file_full_path) + return key == filename_correct + elif perspective: + filename_correct = filename_matches_perspective(video_file_full_path) + return key == filename_correct + else: + return key == 'False' + + queryset_res = queryset.values('id', 'videofile', 'glossvideonme', 'glossvideoperspective') + results = [qv['id'] for qv in queryset_res + if matching_filename(qv['videofile'], qv['glossvideonme'], qv['glossvideoperspective'], self.value())] + + if self.value(): + return queryset.filter(id__in=results) + else: + return queryset.all() + + +class GlossVideoNMEFilter(admin.SimpleListFilter): + + title = _('NME Video') + parameter_name = 'nme_videos' + + def lookups(self, request, model_admin): + nme_video = tuple((b, b) for b in ('True', 'False')) + return nme_video + + def queryset(self, request, queryset): + if self.value(): + return queryset.filter(glossvideonme__isnull=False) + return queryset.all() + + +class GlossVideoPerspectiveFilter(admin.SimpleListFilter): + + title = _('Perspective Video') + parameter_name = 'perspective_videos' + + def lookups(self, request, model_admin): + perspective_video = tuple((b, b) for b in ('True', 'False')) + return perspective_video + + def queryset(self, request, queryset): + if self.value(): + return queryset.filter(glossvideoperspective__isnull=False) + return queryset.all() + + class GlossVideoFileTypeFilter(admin.SimpleListFilter): title = _('MP4 File') @@ -133,11 +217,9 @@ def rename_extension_videos(modeladmin, request, queryset): distinct_glosses = Gloss.objects.filter(glossvideo__in=queryset).distinct() for gloss in distinct_glosses: - for glossvideo in GlossVideo.objects.filter(gloss=gloss).order_by('version', 'id'): + for glossvideo in GlossVideo.objects.filter(gloss=gloss, glossvideonme=None, glossvideoperspective=None).order_by('version', 'id'): video_file_full_path = os.path.join(WRITABLE_FOLDER, str(glossvideo.videofile)) - if not os.path.exists(video_file_full_path): - continue # the video is a backup video that exists on the file system base_filename = os.path.basename(video_file_full_path) @@ -149,14 +231,12 @@ def rename_extension_videos(modeladmin, request, queryset): # use the file system command 'file' to determine the extension for the type of video file desired_video_extension = video_file_type_extension(video_file_full_path) - if glossvideo.version > 0: desired_extension = desired_video_extension + '.bak' + str(glossvideo.id) else: desired_extension = desired_video_extension desired_filename = desired_filename_without_extension + desired_extension - if base_filename == desired_filename: continue @@ -165,62 +245,23 @@ def rename_extension_videos(modeladmin, request, queryset): source = os.path.join(WRITABLE_FOLDER, current_relative_path) destination = os.path.join(WRITABLE_FOLDER, GLOSS_VIDEO_DIRECTORY, dataset_dir, two_letter_dir, desired_filename) - print('video:admin:remove_backups:rename_extension_videos:rename: ', source, destination) + print('video:admin:rename_extension_videos:rename: ', source, destination) - # os.rename(source, destination) + if os.path.exists(video_file_full_path): + # os.rename(source, destination) + print('rename fake') # glossvideo.videofile.name = desired_filename # glossvideo.save() -@admin.action(description="Delete selected backup videos and renumber remaining backups") -def remove_backups(modeladmin, request, queryset): - import os - # retrieve glosses of selected GlossVideo objects for later step - distinct_glosses = Gloss.objects.filter(glossvideo__in=queryset).distinct() - for obj in queryset.filter(version__gt=0): - # unlink all the files - relative_path = str(obj.videofile) - video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) - if os.path.exists(video_file_full_path): - # remove the video file so the GlossVideo object can be deleteds - # this is in addition to the signal pre_delete of a GlossVideo object, which may not delete the files - try: - # os.unlink(obj.videofile.path) - # os.remove(video_file_full_path) - if DEBUG_VIDEOS: - print('video:admin:remove_backups:remove file: ', video_file_full_path) - except (OSError, PermissionError): - if DEBUG_VIDEOS: - print('Exception video:admin:remove_backups: could not delete video file: ', video_file_full_path) - continue - # only backup videos are deleted here - if DEBUG_VIDEOS: - print('video:admin:remove_backups:delete object: ', relative_path) - # obj.delete() - # construct data structure for glosses and remaining backup videos that were not selected - lookup_backup_files = dict() - for gloss in distinct_glosses: - lookup_backup_files[gloss] = GlossVideo.objects.filter(gloss=gloss, version__gt=0).order_by('version', 'id') - for gloss, videos in lookup_backup_files.items(): - # enumerate over the backup videos and give them new version numbers - for inx, video in enumerate(videos, 1): - if DEBUG_VIDEOS: - original_version = video.version - print('video:admin:remove_backups:reversion ', original_version, inx, str(video.videofile)) - # if the file has an old bak-format, its name is fixed here - # rename_extension_video(video) - # then the version of the gloss video object is updated since objects may have been deleted - # video.version = inx - # video.save() - - class GlossVideoAdmin(admin.ModelAdmin): list_display = ['id', 'gloss', 'video_file', 'perspective', 'NME', 'file_timestamp', 'file_group', 'permissions', 'file_size', 'video_type', 'version'] - list_filter = (GlossVideoDatasetFilter, GlossVideoFileSystemGroupFilter, GlossVideoExistenceFilter, GlossVideoFileTypeFilter) + list_filter = (GlossVideoDatasetFilter, GlossVideoFileSystemGroupFilter, GlossVideoExistenceFilter, + GlossVideoFileTypeFilter, GlossVideoNMEFilter, GlossVideoPerspectiveFilter, GlossVideoFilenameFilter) search_fields = ['^gloss__annotationidglosstranslation__text', '^gloss__lemma__lemmaidglosstranslation__text'] - actions = [remove_backups, rename_extension_videos] + actions = [rename_extension_videos] def video_file(self, obj=None): # this will display the full path in the list view diff --git a/signbank/video/convertvideo.py b/signbank/video/convertvideo.py index 234ed3c42..a37f26f32 100755 --- a/signbank/video/convertvideo.py +++ b/signbank/video/convertvideo.py @@ -211,9 +211,24 @@ def make_thumbnail_video(sourcefile, targetfile): os.remove(temp_target) +ACCEPTABLE_VIDEO_EXTENSIONS = ['.mp4', '.mov', '.webm', '.m4v', '.mkv', '.m2v'] + + +def get_video_extension_from_stored_filenpath(video_file_full_path): + file_path, file_extension = os.path.splitext(video_file_full_path) + if '.bak' in file_extension: + # this is a backup file, remove the extension again + file_path, file_extension = os.path.splitext(file_path) + if file_extension not in ACCEPTABLE_VIDEO_EXTENSIONS: + # some other extension is present in the filename + file_extension = '.mp4' + return file_extension + + def video_file_type_extension(video_file_full_path): if not os.path.exists(video_file_full_path): - return ".mp4" + return get_video_extension_from_stored_filenpath(video_file_full_path) + filetype_output = subprocess.run(["file", video_file_full_path], stdout=subprocess.PIPE) filetype = str(filetype_output.stdout) if 'MOV' in filetype: @@ -231,7 +246,7 @@ def video_file_type_extension(video_file_full_path): else: if DEBUG_VIDEOS: print('video:admin:convertvideo:video_file_type_extension:file:UNKNOWN ', filetype) - desired_video_extension = '.mp4' + desired_video_extension = get_video_extension_from_stored_filenpath(video_file_full_path) return desired_video_extension From ae588755e166e978ce38dee9c6b047e09939f6a3 Mon Sep 17 00:00:00 2001 From: susanodd Date: Wed, 8 Jan 2025 11:56:23 +0100 Subject: [PATCH 07/24] #1398: Expanded filters for video admin. Renaming files implemented. --- signbank/video/admin.py | 68 ++++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 27da49adf..50ae23085 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -105,7 +105,6 @@ def lookups(self, request, model_admin): def queryset(self, request, queryset): import re - def filename_matches_nme(filename): filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) try: @@ -124,23 +123,47 @@ def filename_matches_perspective(filename): except (IndexError, ValueError): return 'False' - def matching_filename(videofile, nmevideo, perspective, key): + def filename_matches_video(filename): + filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) + try: + if m := re.search(r".+-(\d+)$", filename_without_extension): + return 'True' + return 'False' + except (IndexError, ValueError): + return 'False' + + def filename_matches_backup_video(filename): + filename_with_extension = os.path.basename(filename) + try: + if m := re.search(r".+-(\d+)\.(mp4|m4v|mov|webm|mkv|m2v)\.(bak\d+)$", filename_with_extension): + return 'True' + return 'False' + except (IndexError, ValueError): + return 'False' + + def matching_filename(videofile, nmevideo, perspective, version, key): if not key: return False from pathlib import Path video_file_full_path = Path(WRITABLE_FOLDER, videofile) if nmevideo: - filename_correct = filename_matches_nme(video_file_full_path) - return key == filename_correct + filename_is_correct = filename_matches_nme(video_file_full_path) + return key == filename_is_correct elif perspective: - filename_correct = filename_matches_perspective(video_file_full_path) - return key == filename_correct + filename_is_correct = filename_matches_perspective(video_file_full_path) + return key == filename_is_correct + elif version > 0: + filename_is_correct = filename_matches_backup_video(video_file_full_path) + return key == filename_is_correct else: - return key == 'False' + filename_is_correct = filename_matches_video(video_file_full_path) + return key == filename_is_correct - queryset_res = queryset.values('id', 'videofile', 'glossvideonme', 'glossvideoperspective') + queryset_res = queryset.values('id', 'videofile', 'glossvideonme', 'glossvideoperspective', 'version') results = [qv['id'] for qv in queryset_res - if matching_filename(qv['videofile'], qv['glossvideonme'], qv['glossvideoperspective'], self.value())] + if matching_filename(qv['videofile'], + qv['glossvideonme'], + qv['glossvideoperspective'], qv['version'], self.value())] if self.value(): return queryset.filter(id__in=results) @@ -159,7 +182,10 @@ def lookups(self, request, model_admin): def queryset(self, request, queryset): if self.value(): - return queryset.filter(glossvideonme__isnull=False) + if self.value() == 'True': + return queryset.filter(glossvideonme__isnull=False) + else: + return queryset.filter(glossvideonme__isnull=True) return queryset.all() @@ -174,7 +200,10 @@ def lookups(self, request, model_admin): def queryset(self, request, queryset): if self.value(): - return queryset.filter(glossvideoperspective__isnull=False) + if self.value() == 'True': + return queryset.filter(glossvideoperspective__isnull=False) + else: + return queryset.filter(glossvideoperspective__isnull=True) return queryset.all() @@ -210,7 +239,7 @@ def matching_file_type(videofile, key): return queryset.all() -@admin.action(description="Rename extension to match video type") +@admin.action(description="Rename files to match video type") def rename_extension_videos(modeladmin, request, queryset): import os # retrieve glosses of selected GlossVideo objects for later step @@ -245,13 +274,16 @@ def rename_extension_videos(modeladmin, request, queryset): source = os.path.join(WRITABLE_FOLDER, current_relative_path) destination = os.path.join(WRITABLE_FOLDER, GLOSS_VIDEO_DIRECTORY, dataset_dir, two_letter_dir, desired_filename) - print('video:admin:rename_extension_videos:rename: ', source, destination) - + desired_relative_path = os.path.join(GLOSS_VIDEO_DIRECTORY, + dataset_dir, two_letter_dir, desired_filename) + if DEBUG_VIDEOS: + print('video:admin:rename_extension_videos:os.rename: ', source, destination) if os.path.exists(video_file_full_path): - # os.rename(source, destination) - print('rename fake') - # glossvideo.videofile.name = desired_filename - # glossvideo.save() + os.rename(source, destination) + if DEBUG_VIDEOS: + print('video:admin:rename_extension_videos:videofile.name = ', desired_relative_path) + glossvideo.videofile.name = desired_relative_path + glossvideo.save() class GlossVideoAdmin(admin.ModelAdmin): From e8e15691da8a5989ec8377a7a557e404aa5239f9 Mon Sep 17 00:00:00 2001 From: susanodd Date: Wed, 8 Jan 2025 13:31:52 +0100 Subject: [PATCH 08/24] #1398: Video admin filter for backups, commands renumber, remove --- signbank/video/admin.py | 75 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 50ae23085..3458e72cd 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -239,7 +239,25 @@ def matching_file_type(videofile, key): return queryset.all() -@admin.action(description="Rename files to match video type") +class GlossVideoBackupFilter(admin.SimpleListFilter): + + title = _('Backup Video') + parameter_name = 'backup_videos' + + def lookups(self, request, model_admin): + nme_video = tuple((b, b) for b in ('True', 'False')) + return nme_video + + def queryset(self, request, queryset): + if self.value(): + if self.value() == 'True': + return queryset.filter(version__gt=0) + else: + return queryset.filter(version=0) + return queryset.all() + + +@admin.action(description="Rename video files to match type") def rename_extension_videos(modeladmin, request, queryset): import os # retrieve glosses of selected GlossVideo objects for later step @@ -281,19 +299,68 @@ def rename_extension_videos(modeladmin, request, queryset): if os.path.exists(video_file_full_path): os.rename(source, destination) if DEBUG_VIDEOS: - print('video:admin:rename_extension_videos:videofile.name = ', desired_relative_path) + print('video:admin:rename_extension_videos:videofile.name := ', desired_relative_path) glossvideo.videofile.name = desired_relative_path glossvideo.save() +@admin.action(description="Remove selected backups") +def remove_backups(modeladmin, request, queryset): + import os + for obj in queryset.filter(glossvideonme=None, glossvideoperspective=None, version__gt=0): + # unlink all the files + relative_path = str(obj.videofile) + video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) + if os.path.exists(video_file_full_path): + # remove the video file so the GlossVideo object can be deleted + # this is in addition to the signal pre_delete of a GlossVideo object, which may not delete the files + try: + os.unlink(obj.videofile.path) + os.remove(video_file_full_path) + if DEBUG_VIDEOS: + print('video:admin:remove_backups:os.remove: ', video_file_full_path) + except (OSError, PermissionError): + if DEBUG_VIDEOS: + print('Exception video:admin:remove_backups: could not delete video file: ', video_file_full_path) + continue + # only backup videos are deleted here + if DEBUG_VIDEOS: + print('video:admin:remove_backups:delete object: ', relative_path) + obj.delete() + + +@admin.action(description="Renumber selected backups") +def renumber_backups(modeladmin, request, queryset): + import os + # retrieve glosses of selected GlossVideo objects for later step + distinct_glosses = Gloss.objects.filter(glossvideo__in=queryset).distinct() + # construct data structure for glosses and backup videos including those that are not selected + lookup_backup_files = dict() + for gloss in distinct_glosses: + lookup_backup_files[gloss] = GlossVideo.objects.filter(gloss=gloss, + glossvideonme=None, + glossvideoperspective=None, + version__gt=0).order_by('version', 'id') + for gloss, videos in lookup_backup_files.items(): + # enumerate over the backup videos and give them new version numbers + for inx, video in enumerate(videos, 1): + if DEBUG_VIDEOS: + original_version = video.version + print('video:admin:renumber_backups: ', original_version, inx, str(video.videofile)) + # the version of the gloss video object is updated since objects may have been deleted + video.version = inx + video.save() + + class GlossVideoAdmin(admin.ModelAdmin): list_display = ['id', 'gloss', 'video_file', 'perspective', 'NME', 'file_timestamp', 'file_group', 'permissions', 'file_size', 'video_type', 'version'] list_filter = (GlossVideoDatasetFilter, GlossVideoFileSystemGroupFilter, GlossVideoExistenceFilter, - GlossVideoFileTypeFilter, GlossVideoNMEFilter, GlossVideoPerspectiveFilter, GlossVideoFilenameFilter) + GlossVideoFileTypeFilter, GlossVideoNMEFilter, GlossVideoPerspectiveFilter, + GlossVideoFilenameFilter, GlossVideoBackupFilter) search_fields = ['^gloss__annotationidglosstranslation__text', '^gloss__lemma__lemmaidglosstranslation__text'] - actions = [rename_extension_videos] + actions = [rename_extension_videos, remove_backups, renumber_backups] def video_file(self, obj=None): # this will display the full path in the list view From 08e620f75c6793a7c7b030e05537ff71fd6ac1a5 Mon Sep 17 00:00:00 2001 From: susanodd Date: Mon, 13 Jan 2025 11:45:40 +0100 Subject: [PATCH 09/24] #1398: Admin videos moved functions out of method to allow reuse Added operation Erase name from oncorrect NME/Perspective object in order to allow deletion Accomodate possibility of empty name field in video models signals and delete operations. --- signbank/video/admin.py | 126 ++++++++++++++++++++------------- signbank/video/convertvideo.py | 6 ++ signbank/video/models.py | 27 +++++-- 3 files changed, 105 insertions(+), 54 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 3458e72cd..add92e771 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -13,6 +13,7 @@ from signbank.tools import get_two_letter_dir from signbank.video.convertvideo import video_file_type_extension import subprocess +import re class GlossVideoDatasetFilter(admin.SimpleListFilter): @@ -77,6 +78,8 @@ def queryset(self, request, queryset): def matching_file_exists(videofile, key): if not key: return False + if 'glossvideo' not in videofile: + return False from pathlib import Path video_file_full_path = Path(WRITABLE_FOLDER, videofile) if video_file_full_path.exists(): @@ -87,13 +90,52 @@ def matching_file_exists(videofile, key): queryset_res = queryset.values('id', 'videofile') results = [qv['id'] for qv in queryset_res if matching_file_exists(qv['videofile'], self.value())] - if self.value(): return queryset.filter(id__in=results) else: return queryset.all() +def filename_matches_nme(filename): + filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) + try: + if m := re.search(r".+-(\d+)_(nme_\d+|nme_\d+_left|nme_\d+_right)$", filename_without_extension): + return True + return False + except (IndexError, ValueError): + return False + + +def filename_matches_perspective(filename): + filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) + try: + if m := re.search(r".+-(\d+)_(left|right|nme_\d+_left|nme_\d+_right)$", filename_without_extension): + return True + return False + except (IndexError, ValueError): + return False + + +def filename_matches_video(filename): + filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) + try: + if m := re.search(r".+-(\d+)$", filename_without_extension): + return True + return False + except (IndexError, ValueError): + return False + + +def filename_matches_backup_video(filename): + filename_with_extension = os.path.basename(filename) + try: + if m := re.search(r".+-(\d+)\.(mp4|m4v|mov|webm|mkv|m2v)\.(bak\d+)$", filename_with_extension): + return True + return False + except (IndexError, ValueError): + return False + + class GlossVideoFilenameFilter(admin.SimpleListFilter): title = _('Filename Correct') @@ -104,42 +146,6 @@ def lookups(self, request, model_admin): return file_exists def queryset(self, request, queryset): - import re - def filename_matches_nme(filename): - filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) - try: - if m := re.search(r".+-(\d+)_(nme_\d+|nme_\d+_left|nme_\d+_right)$", filename_without_extension): - return 'True' - return 'False' - except (IndexError, ValueError): - return 'False' - - def filename_matches_perspective(filename): - filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) - try: - if m := re.search(r".+-(\d+)_(left|right|nme_\d+_left|nme_\d+_right)$", filename_without_extension): - return 'True' - return 'False' - except (IndexError, ValueError): - return 'False' - - def filename_matches_video(filename): - filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) - try: - if m := re.search(r".+-(\d+)$", filename_without_extension): - return 'True' - return 'False' - except (IndexError, ValueError): - return 'False' - - def filename_matches_backup_video(filename): - filename_with_extension = os.path.basename(filename) - try: - if m := re.search(r".+-(\d+)\.(mp4|m4v|mov|webm|mkv|m2v)\.(bak\d+)$", filename_with_extension): - return 'True' - return 'False' - except (IndexError, ValueError): - return 'False' def matching_filename(videofile, nmevideo, perspective, version, key): if not key: @@ -148,16 +154,16 @@ def matching_filename(videofile, nmevideo, perspective, version, key): video_file_full_path = Path(WRITABLE_FOLDER, videofile) if nmevideo: filename_is_correct = filename_matches_nme(video_file_full_path) - return key == filename_is_correct + return key == str(filename_is_correct) elif perspective: filename_is_correct = filename_matches_perspective(video_file_full_path) - return key == filename_is_correct + return key == str(filename_is_correct) elif version > 0: filename_is_correct = filename_matches_backup_video(video_file_full_path) - return key == filename_is_correct + return key == str(filename_is_correct) else: filename_is_correct = filename_matches_video(video_file_full_path) - return key == filename_is_correct + return key == str(filename_is_correct) queryset_res = queryset.values('id', 'videofile', 'glossvideonme', 'glossvideoperspective', 'version') results = [qv['id'] for qv in queryset_res @@ -352,6 +358,30 @@ def renumber_backups(modeladmin, request, queryset): video.save() +@admin.action(description="Set incorrect NME/Perspective filenames to empty string") +def unlink_files(modeladmin, request, queryset): + # allow to erase the filename from an object if it has the wrong format and it is for a subclass video + # this is a patch for repairing doubly linked files + import os + for obj in queryset: + if not hasattr(obj, 'glossvideonme') and not hasattr(obj, 'glossvideoperspective'): + # this is not a subclass video that was selected by the user + continue + + from pathlib import Path + video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) + if hasattr(obj, 'glossvideonme') and filename_matches_nme(video_file_full_path): + continue + if hasattr(obj, 'glossvideoperspective') and filename_matches_perspective(video_file_full_path): + continue + + # erase the file path if it has the wrong format + if DEBUG_VIDEOS: + print('unlink_files: erase incorrect path from object: ', obj, video_file_full_path) + obj.videofile.name = "" + obj.save() + + class GlossVideoAdmin(admin.ModelAdmin): list_display = ['id', 'gloss', 'video_file', 'perspective', 'NME', 'file_timestamp', 'file_group', 'permissions', 'file_size', 'video_type', 'version'] @@ -360,11 +390,11 @@ class GlossVideoAdmin(admin.ModelAdmin): GlossVideoFilenameFilter, GlossVideoBackupFilter) search_fields = ['^gloss__annotationidglosstranslation__text', '^gloss__lemma__lemmaidglosstranslation__text'] - actions = [rename_extension_videos, remove_backups, renumber_backups] + actions = [rename_extension_videos, remove_backups, renumber_backups, unlink_files] def video_file(self, obj=None): # this will display the full path in the list view - if obj is None: + if obj is None or not str(obj.videofile): return "" import os video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) @@ -383,7 +413,7 @@ def NME(self, obj=None): def file_timestamp(self, obj=None): # if the file exists, this will display its timestamp in the list view - if obj is None: + if obj is None or not str(obj.videofile): return "" import os import datetime @@ -395,7 +425,7 @@ def file_timestamp(self, obj=None): def file_group(self, obj=None): # this will display a group in the list view - if obj is None: + if obj is None or not str(obj.videofile): return "" else: from pathlib import Path @@ -408,7 +438,7 @@ def file_group(self, obj=None): def file_size(self, obj=None): # this will display a group in the list view - if obj is None: + if obj is None or not str(obj.videofile): return "" else: from pathlib import Path @@ -421,7 +451,7 @@ def file_size(self, obj=None): def permissions(self, obj=None): # this will display a group in the list view - if obj is None: + if obj is None or not str(obj.videofile): return "" else: from pathlib import Path @@ -435,7 +465,7 @@ def permissions(self, obj=None): def video_type(self, obj=None): # if the file exists, this will display its timestamp in the list view - if obj is None: + if obj is None or not str(obj.videofile): return "" import os video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) diff --git a/signbank/video/convertvideo.py b/signbank/video/convertvideo.py index a37f26f32..9f8e80915 100755 --- a/signbank/video/convertvideo.py +++ b/signbank/video/convertvideo.py @@ -226,6 +226,12 @@ def get_video_extension_from_stored_filenpath(video_file_full_path): def video_file_type_extension(video_file_full_path): + + if not video_file_full_path: + return '' + if 'glossvideo' not in video_file_full_path: + return '' + if not os.path.exists(video_file_full_path): return get_video_extension_from_stored_filenpath(video_file_full_path) diff --git a/signbank/video/models.py b/signbank/video/models.py index 2447e1c4e..16b35fcab 100755 --- a/signbank/video/models.py +++ b/signbank/video/models.py @@ -693,6 +693,8 @@ def small_video(self, use_name=False): """Return the URL of the small version for this video :param use_name: whether videofile.name should be used instead of videofile.path """ + if not self.videofile: + return None small_video_path = add_small_appendix(self.videofile.path) if os.path.exists(small_video_path): if use_name: @@ -751,6 +753,9 @@ def delete_files(self): if settings.DEBUG_VIDEOS: print('delete_files GlossVideo: ', str(self.videofile)) + if not self.videofile.name: + return + small_video_path = self.small_video() try: os.unlink(self.videofile.path) @@ -817,8 +822,6 @@ def __str__(self): # this coercion to a string type sometimes causes special characters in the filename to be a problem # code has been introduced elsewhere to make sure paths are the correct encoding glossvideoname = self.videofile.name - if settings.DEBUG_VIDEOS: - print('__str__ GlossVideo: ', self.videofile.name) return glossvideoname def is_glossvideonme(self): @@ -958,13 +961,23 @@ def move_video(self, move_files_on_disk=True): def delete_files(self): """Delete the files associated with this object""" + old_path = str(self.videofile) + if not old_path: + return + file_system_path = os.path.join(settings.WRITABLE_FOLDER, old_path) if settings.DEBUG_VIDEOS: - print('delete_files GlossVideoNME: ', str(self.videofile)) + print('delete_files GlossVideoNME: ', file_system_path) + if not os.path.exists(file_system_path): + # Video file not found on server + # on the production server this is a problem + msg = "GlossVideoNME video file not found: " + file_system_path + print(msg) + return try: - os.unlink(self.videofile.path) + os.unlink(file_system_path) except (OSError, PermissionError): if settings.DEBUG_VIDEOS: - print('delete_files exception GlossVideo OSError, PermissionError: ', str(self.videofile)) + print('delete_files exception GlossVideoNME OSError, PermissionError: ', file_system_path) pass def reversion(self, revert=False): @@ -1024,6 +1037,8 @@ def move_video(self, move_files_on_disk=True): def delete_files(self): """Delete the files associated with this object""" old_path = str(self.videofile) + if not old_path: + return file_system_path = os.path.join(settings.WRITABLE_FOLDER, old_path) if settings.DEBUG_VIDEOS: print('perspective video delete files: ', file_system_path) @@ -1035,7 +1050,7 @@ def delete_files(self): return try: os.unlink(file_system_path) - except OSError: + except (PermissionError, OSError): msg = "Perspective video file could not be deleted: " + file_system_path print(msg) From 3624bd71c14d3096290b3d4c13d0ae81074eb109 Mon Sep 17 00:00:00 2001 From: susanodd Date: Mon, 27 Jan 2025 08:03:12 +0100 Subject: [PATCH 10/24] #1398: Added checks for empty videofile path to package --- signbank/dictionary/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/signbank/dictionary/views.py b/signbank/dictionary/views.py index eb4259c61..5972f5251 100644 --- a/signbank/dictionary/views.py +++ b/signbank/dictionary/views.py @@ -2219,13 +2219,13 @@ def package(request): video_urls = {os.path.splitext(os.path.basename(gv.videofile.name))[0]: reverse('dictionary:protected_media', args=[gv.small_video(use_name=True) or gv.videofile.name]) for gv in GlossVideo.objects.filter(gloss__in=available_glosses, glossvideonme=None, glossvideoperspective=None, version=0) - if os.path.exists(str(gv.videofile.path)) + if gv.videofile and gv.videofile.name and os.path.exists(str(gv.videofile.path)) and os.path.getmtime(str(gv.videofile.path)) > since_timestamp} image_urls = {os.path.splitext(os.path.basename(gv.videofile.name))[0]: reverse('dictionary:protected_media', args=[gv.poster_file()]) for gv in GlossVideo.objects.filter(gloss__in=available_glosses, glossvideonme=None, glossvideoperspective=None, version=0) - if os.path.exists(str(gv.videofile.path)) - and os.path.getmtime(str(gv.videofile.path)) > since_timestamp} + if gv.videofile and gv.videofile.name and os.path.exists(str(gv.videofile.path)) + and os.path.getmtime(str(gv.videofile.path)) > since_timestamp} interface_language_code = get_interface_language_api(request, request.user) From 722cbc89b2f86084c73ac60bd22d8097750ff697 Mon Sep 17 00:00:00 2001 From: susanodd Date: Mon, 17 Feb 2025 10:27:40 +0100 Subject: [PATCH 11/24] 1398: Per review, moved all imports to top. --- signbank/video/admin.py | 38 ++++++++------------------------------ 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index add92e771..57ec1d2d4 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -1,19 +1,18 @@ -import os.path from django.contrib import admin -from django import forms -from django.db import models from signbank.video.models import GlossVideo, GlossVideoHistory, AnnotatedVideo, ExampleVideoHistory from signbank.dictionary.models import Dataset, AnnotatedGloss, Gloss from django.contrib.auth.models import User from signbank.settings.base import * from signbank.settings.server_specific import WRITABLE_FOLDER, FILESYSTEM_SIGNBANK_GROUPS, DEBUG_VIDEOS -from django.utils.translation import override, gettext_lazy as _ -from django.db.models import Q, Count, CharField, TextField, Value as V +from django.utils.translation import gettext_lazy as _ from signbank.tools import get_two_letter_dir from signbank.video.convertvideo import video_file_type_extension -import subprocess import re +from pathlib import Path +import os +import stat +import datetime as DT class GlossVideoDatasetFilter(admin.SimpleListFilter): @@ -47,7 +46,6 @@ def queryset(self, request, queryset): def matching_file_group(videofile, key): if not key: return False - from pathlib import Path video_file_full_path = Path(WRITABLE_FOLDER, videofile) if video_file_full_path.exists(): return video_file_full_path.group() == key @@ -80,7 +78,6 @@ def matching_file_exists(videofile, key): return False if 'glossvideo' not in videofile: return False - from pathlib import Path video_file_full_path = Path(WRITABLE_FOLDER, videofile) if video_file_full_path.exists(): return key == 'True' @@ -150,7 +147,6 @@ def queryset(self, request, queryset): def matching_filename(videofile, nmevideo, perspective, version, key): if not key: return False - from pathlib import Path video_file_full_path = Path(WRITABLE_FOLDER, videofile) if nmevideo: filename_is_correct = filename_matches_nme(video_file_full_path) @@ -227,7 +223,6 @@ def queryset(self, request, queryset): def matching_file_type(videofile, key): if not key: return False - from pathlib import Path video_file_full_path = Path(WRITABLE_FOLDER, videofile) if video_file_full_path.exists(): file_extension = video_file_type_extension(video_file_full_path) @@ -265,7 +260,6 @@ def queryset(self, request, queryset): @admin.action(description="Rename video files to match type") def rename_extension_videos(modeladmin, request, queryset): - import os # retrieve glosses of selected GlossVideo objects for later step distinct_glosses = Gloss.objects.filter(glossvideo__in=queryset).distinct() @@ -312,7 +306,6 @@ def rename_extension_videos(modeladmin, request, queryset): @admin.action(description="Remove selected backups") def remove_backups(modeladmin, request, queryset): - import os for obj in queryset.filter(glossvideonme=None, glossvideoperspective=None, version__gt=0): # unlink all the files relative_path = str(obj.videofile) @@ -337,7 +330,6 @@ def remove_backups(modeladmin, request, queryset): @admin.action(description="Renumber selected backups") def renumber_backups(modeladmin, request, queryset): - import os # retrieve glosses of selected GlossVideo objects for later step distinct_glosses = Gloss.objects.filter(glossvideo__in=queryset).distinct() # construct data structure for glosses and backup videos including those that are not selected @@ -362,13 +354,11 @@ def renumber_backups(modeladmin, request, queryset): def unlink_files(modeladmin, request, queryset): # allow to erase the filename from an object if it has the wrong format and it is for a subclass video # this is a patch for repairing doubly linked files - import os for obj in queryset: if not hasattr(obj, 'glossvideonme') and not hasattr(obj, 'glossvideoperspective'): # this is not a subclass video that was selected by the user continue - from pathlib import Path video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) if hasattr(obj, 'glossvideonme') and filename_matches_nme(video_file_full_path): continue @@ -396,7 +386,6 @@ def video_file(self, obj=None): # this will display the full path in the list view if obj is None or not str(obj.videofile): return "" - import os video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) return video_file_full_path @@ -415,11 +404,9 @@ def file_timestamp(self, obj=None): # if the file exists, this will display its timestamp in the list view if obj is None or not str(obj.videofile): return "" - import os - import datetime video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) if os.path.exists(video_file_full_path): - return datetime.datetime.fromtimestamp(os.path.getctime(video_file_full_path)) + return DT.datetime.fromtimestamp(os.path.getctime(video_file_full_path)) else: return "" @@ -428,7 +415,6 @@ def file_group(self, obj=None): if obj is None or not str(obj.videofile): return "" else: - from pathlib import Path video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) if video_file_full_path.exists(): group = video_file_full_path.group() @@ -441,7 +427,6 @@ def file_size(self, obj=None): if obj is None or not str(obj.videofile): return "" else: - from pathlib import Path video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) if video_file_full_path.exists(): size = str(video_file_full_path.stat().st_size) @@ -454,8 +439,6 @@ def permissions(self, obj=None): if obj is None or not str(obj.videofile): return "" else: - from pathlib import Path - import stat video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) if video_file_full_path.exists(): stats = stat.filemode(video_file_full_path.stat().st_mode) @@ -467,7 +450,6 @@ def video_type(self, obj=None): # if the file exists, this will display its timestamp in the list view if obj is None or not str(obj.videofile): return "" - import os video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) if os.path.exists(video_file_full_path): return video_file_type_extension(video_file_full_path) @@ -585,11 +567,9 @@ def timestamp(self, obj=None): # if the file exists, this will display its timestamp in the list view if obj is None: return "" - import os - import datetime video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) if os.path.exists(video_file_full_path): - return datetime.datetime.fromtimestamp(os.path.getctime(video_file_full_path)) + return DT.datetime.fromtimestamp(os.path.getctime(video_file_full_path)) else: return "" @@ -602,11 +582,9 @@ def eaf_timestamp(self, obj=None): # if the file exists, this will display its timestamp in the list view if obj is None: return "" - import os - import datetime eaf_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.eaffile)) if os.path.exists(eaf_file_full_path): - return datetime.datetime.fromtimestamp(os.path.getctime(eaf_file_full_path)) + return DT.datetime.fromtimestamp(os.path.getctime(eaf_file_full_path)) else: return "" From b8bffb11dfd2e72b395dd25ff35e25b18837ff63 Mon Sep 17 00:00:00 2001 From: susanodd Date: Mon, 17 Feb 2025 11:56:26 +0100 Subject: [PATCH 12/24] #1398: Per review simplied function code. Modifed caller to check not None since it's not a Boolean anymore. --- signbank/video/admin.py | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 57ec1d2d4..493c09db1 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -95,42 +95,22 @@ def matching_file_exists(videofile, key): def filename_matches_nme(filename): filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) - try: - if m := re.search(r".+-(\d+)_(nme_\d+|nme_\d+_left|nme_\d+_right)$", filename_without_extension): - return True - return False - except (IndexError, ValueError): - return False + return re.search(r".+-(\d+)_(nme_\d+|nme_\d+_left|nme_\d+_right)$", filename_without_extension) def filename_matches_perspective(filename): filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) - try: - if m := re.search(r".+-(\d+)_(left|right|nme_\d+_left|nme_\d+_right)$", filename_without_extension): - return True - return False - except (IndexError, ValueError): - return False + return re.search(r".+-(\d+)_(left|right|nme_\d+_left|nme_\d+_right)$", filename_without_extension) def filename_matches_video(filename): filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) - try: - if m := re.search(r".+-(\d+)$", filename_without_extension): - return True - return False - except (IndexError, ValueError): - return False + return re.search(r".+-(\d+)$", filename_without_extension) def filename_matches_backup_video(filename): filename_with_extension = os.path.basename(filename) - try: - if m := re.search(r".+-(\d+)\.(mp4|m4v|mov|webm|mkv|m2v)\.(bak\d+)$", filename_with_extension): - return True - return False - except (IndexError, ValueError): - return False + return re.search(r".+-(\d+)\.(mp4|m4v|mov|webm|mkv|m2v)\.(bak\d+)$", filename_with_extension) class GlossVideoFilenameFilter(admin.SimpleListFilter): @@ -149,16 +129,16 @@ def matching_filename(videofile, nmevideo, perspective, version, key): return False video_file_full_path = Path(WRITABLE_FOLDER, videofile) if nmevideo: - filename_is_correct = filename_matches_nme(video_file_full_path) + filename_is_correct = filename_matches_nme(video_file_full_path) is not None return key == str(filename_is_correct) elif perspective: - filename_is_correct = filename_matches_perspective(video_file_full_path) + filename_is_correct = filename_matches_perspective(video_file_full_path) is not None return key == str(filename_is_correct) elif version > 0: - filename_is_correct = filename_matches_backup_video(video_file_full_path) + filename_is_correct = filename_matches_backup_video(video_file_full_path) is not None return key == str(filename_is_correct) else: - filename_is_correct = filename_matches_video(video_file_full_path) + filename_is_correct = filename_matches_video(video_file_full_path) is not None return key == str(filename_is_correct) queryset_res = queryset.values('id', 'videofile', 'glossvideonme', 'glossvideoperspective', 'version') From 0f2a0e46a947087fad240d734782fda23856ce9c Mon Sep 17 00:00:00 2001 From: susanodd Date: Mon, 17 Feb 2025 12:30:45 +0100 Subject: [PATCH 13/24] #1398: Per review: Added docstring documentation to Filters of GlossVideo Amin. --- signbank/video/admin.py | 61 ++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 493c09db1..736b82f16 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -16,7 +16,12 @@ class GlossVideoDatasetFilter(admin.SimpleListFilter): - + """ + Filter the GlossVideo objects on the Dataset of the gloss + The values of lookups show in the right-hand column of the admin under a heading "Dataset" + Called from GlossVideoAdmin + :model: GlossVideoAdmin + """ title = _('Dataset') parameter_name = 'videos_per_dataset' @@ -33,7 +38,12 @@ def queryset(self, request, queryset): class GlossVideoFileSystemGroupFilter(admin.SimpleListFilter): - + """ + Filter the GlossVideo objects on the file system group of the video file + The values of lookups show in the right-hand column of the admin under a heading "File System Group" + Called from GlossVideoAdmin + :model: GlossVideoAdmin + """ title = _('File System Group') parameter_name = 'filesystem_group' @@ -63,7 +73,12 @@ def matching_file_group(videofile, key): class GlossVideoExistenceFilter(admin.SimpleListFilter): - + """ + Filter the GlossVideo objects on whether the the video file exists + The values of lookups show in the right-hand column of the admin under a heading "File Exists" + Called from GlossVideoAdmin + :model: GlossVideoAdmin + """ title = _('File Exists') parameter_name = 'file_exists' @@ -114,7 +129,12 @@ def filename_matches_backup_video(filename): class GlossVideoFilenameFilter(admin.SimpleListFilter): - + """ + Filter the GlossVideo objects on whether the filename is correct for the type of video + The values of lookups show in the right-hand column of the admin under a heading "Filename Correct" + Called from GlossVideoAdmin + :model: GlossVideoAdmin + """ title = _('Filename Correct') parameter_name = 'filename_correct' @@ -154,7 +174,12 @@ def matching_filename(videofile, nmevideo, perspective, version, key): class GlossVideoNMEFilter(admin.SimpleListFilter): - + """ + Filter the GlossVideo objects on whether the video is an NME Video + The values of lookups show in the right-hand column of the admin under a heading "NME Video" + Called from GlossVideoAdmin + :model: GlossVideoAdmin + """ title = _('NME Video') parameter_name = 'nme_videos' @@ -172,7 +197,12 @@ def queryset(self, request, queryset): class GlossVideoPerspectiveFilter(admin.SimpleListFilter): - + """ + Filter the GlossVideo objects on whether the video is a Perspective Video + The values of lookups show in the right-hand column of the admin under a heading "Perspective Video" + Called from GlossVideoAdmin + :model: GlossVideoAdmin + """ title = _('Perspective Video') parameter_name = 'perspective_videos' @@ -190,7 +220,12 @@ def queryset(self, request, queryset): class GlossVideoFileTypeFilter(admin.SimpleListFilter): - + """ + Filter the GlossVideo objects on whether the video is an MP4 video + The values of lookups show in the right-hand column of the admin under a heading "MP4 File" + Called from GlossVideoAdmin + :model: GlossVideoAdmin + """ title = _('MP4 File') parameter_name = 'file_type' @@ -221,13 +256,18 @@ def matching_file_type(videofile, key): class GlossVideoBackupFilter(admin.SimpleListFilter): - + """ + Filter the GlossVideo objects on whether the video is a backup video + The values of lookups show in the right-hand column of the admin under a heading "Backup Video" + Called from GlossVideoAdmin + :model: GlossVideoAdmin + """ title = _('Backup Video') parameter_name = 'backup_videos' def lookups(self, request, model_admin): - nme_video = tuple((b, b) for b in ('True', 'False')) - return nme_video + is_backup = tuple((b, b) for b in ('True', 'False')) + return is_backup def queryset(self, request, queryset): if self.value(): @@ -512,6 +552,7 @@ def queryset(self, request, queryset): return queryset.filter(annotatedsentence_id__in=annotated_sentences_ids) return queryset.all() + class AnnotatedVideoAdmin(admin.ModelAdmin): actions = None list_display = ('dataset', 'annotated_sentence', 'video_file', 'timestamp', 'eaf_file', 'eaf_timestamp', 'url', 'source') From 2b983e9a83f653cde9e200ac3fe93abc14d33a99 Mon Sep 17 00:00:00 2001 From: susanodd Date: Tue, 18 Feb 2025 10:53:18 +0100 Subject: [PATCH 14/24] #1398: Per review: filename strings are f-strings. Doc-strings for commands --- signbank/video/admin.py | 115 +++++++++++++++++++++++++++++----------- 1 file changed, 83 insertions(+), 32 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 736b82f16..0b8b41dc2 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -280,26 +280,43 @@ def queryset(self, request, queryset): @admin.action(description="Rename video files to match type") def rename_extension_videos(modeladmin, request, queryset): - # retrieve glosses of selected GlossVideo objects for later step + """ + Command for the GlossVideo objects selected in queryset + The command appears in the admin pull-down list of commands for the selected gloss videos + The command determines which glosses are selected, then retrieves the normal video objects for those glosses + This allows the user to merely select one of the objects and hereby change them all instead of numerous selections + For those gloss video objects, it renames the file if the filename is not correct + This also applies to wrong video types in filenames, e.g., a webm video that has mp4 in its filename + This applies to backup videos as well as normal videos + Called from GlossVideoAdmin + :model: GlossVideoAdmin + """ + # retrieve glosses of selected GlossVideo objects distinct_glosses = Gloss.objects.filter(glossvideo__in=queryset).distinct() for gloss in distinct_glosses: for glossvideo in GlossVideo.objects.filter(gloss=gloss, glossvideonme=None, glossvideoperspective=None).order_by('version', 'id'): + current_relative_path = str(glossvideo.videofile) + if not current_relative_path: + # make sure the path is not empty + continue - video_file_full_path = os.path.join(WRITABLE_FOLDER, str(glossvideo.videofile)) + video_file_full_path = os.path.join(WRITABLE_FOLDER, current_relative_path) + + # retrieve the actual filename stored in the gloss video object + # and also compute the name it should have - # the video is a backup video that exists on the file system base_filename = os.path.basename(video_file_full_path) idgloss = gloss.idgloss two_letter_dir = get_two_letter_dir(idgloss) dataset_dir = gloss.lemma.dataset.acronym - desired_filename_without_extension = idgloss + '-' + str(gloss.id) + desired_filename_without_extension = f'{idgloss}-{gloss.id}' # use the file system command 'file' to determine the extension for the type of video file desired_video_extension = video_file_type_extension(video_file_full_path) if glossvideo.version > 0: - desired_extension = desired_video_extension + '.bak' + str(glossvideo.id) + desired_extension = f'{desired_video_extension}.bak{glossvideo.id}' else: desired_extension = desired_video_extension @@ -307,8 +324,8 @@ def rename_extension_videos(modeladmin, request, queryset): if base_filename == desired_filename: continue - current_relative_path = str(glossvideo.videofile) - + # if we get to here, the file has the wrong path + # the path needs to be fixed and the file renamed source = os.path.join(WRITABLE_FOLDER, current_relative_path) destination = os.path.join(WRITABLE_FOLDER, GLOSS_VIDEO_DIRECTORY, dataset_dir, two_letter_dir, desired_filename) @@ -326,23 +343,36 @@ def rename_extension_videos(modeladmin, request, queryset): @admin.action(description="Remove selected backups") def remove_backups(modeladmin, request, queryset): + """ + Command for the GlossVideo objects selected in queryset + The command appears in the admin pull-down list of commands for the selected gloss videos + The command removes the selected backup files + Other selected objects are ignored + This allows the user to keep a number of the backup files by not selecting everything + Called from GlossVideoAdmin + :model: GlossVideoAdmin + """ + # make sure the queryset only applies to backups for normal videos for obj in queryset.filter(glossvideonme=None, glossvideoperspective=None, version__gt=0): - # unlink all the files relative_path = str(obj.videofile) - video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) - if os.path.exists(video_file_full_path): - # remove the video file so the GlossVideo object can be deleted - # this is in addition to the signal pre_delete of a GlossVideo object, which may not delete the files - try: - os.unlink(obj.videofile.path) - os.remove(video_file_full_path) - if DEBUG_VIDEOS: - print('video:admin:remove_backups:os.remove: ', video_file_full_path) - except (OSError, PermissionError): - if DEBUG_VIDEOS: - print('Exception video:admin:remove_backups: could not delete video file: ', video_file_full_path) - continue + if not relative_path: + continue + video_file_full_path = os.path.join(WRITABLE_FOLDER, relative_path) + if not os.path.exists(video_file_full_path): + continue + # first remove the video file so the GlossVideo object can be deleted later + # this is done to avoid the signals on GlossVideo delete + try: + os.unlink(obj.videofile.path) + os.remove(video_file_full_path) + if DEBUG_VIDEOS: + print('video:admin:remove_backups:os.remove: ', video_file_full_path) + except (OSError, PermissionError): + if DEBUG_VIDEOS: + print('Exception video:admin:remove_backups: could not delete video file: ', video_file_full_path) + continue # only backup videos are deleted here + # the object does not point to anything anymore, so it can be deleted if DEBUG_VIDEOS: print('video:admin:remove_backups:delete object: ', relative_path) obj.delete() @@ -350,6 +380,16 @@ def remove_backups(modeladmin, request, queryset): @admin.action(description="Renumber selected backups") def renumber_backups(modeladmin, request, queryset): + """ + Command for the GlossVideo objects selected in queryset + The command appears in the admin pull-down list of commands for the selected gloss videos + The command renumbers the backup video objects for the GlossVideo queryset + The command determines which glosses are selected, then retrieves the backup video objects for those glosses + This allows the user to merely select one of the objects and hereby renumber them all + Because the backup objects are numbered by version, all of the objects must be renumbered + Called from GlossVideoAdmin + :model: GlossVideoAdmin + """ # retrieve glosses of selected GlossVideo objects for later step distinct_glosses = Gloss.objects.filter(glossvideo__in=queryset).distinct() # construct data structure for glosses and backup videos including those that are not selected @@ -361,33 +401,44 @@ def renumber_backups(modeladmin, request, queryset): version__gt=0).order_by('version', 'id') for gloss, videos in lookup_backup_files.items(): # enumerate over the backup videos and give them new version numbers + # the version of the gloss video object is updated since objects may have been deleted for inx, video in enumerate(videos, 1): - if DEBUG_VIDEOS: - original_version = video.version - print('video:admin:renumber_backups: ', original_version, inx, str(video.videofile)) - # the version of the gloss video object is updated since objects may have been deleted + if inx == video.version: + continue video.version = inx video.save() @admin.action(description="Set incorrect NME/Perspective filenames to empty string") def unlink_files(modeladmin, request, queryset): - # allow to erase the filename from an object if it has the wrong format and it is for a subclass video - # this is a patch for repairing doubly linked files + """ + Command for the GlossVideo objects selected in queryset + The command appears in the admin pull-down list of commands for the selected gloss videos + The command only applies to Perspective and NME videos, other selected videos are ignored. + Allow to erase the filename from an object if it has the wrong format + This is for the purpose of repairing doubly linked files where the subclass object points to the normal video + Once the filename has been cleared, the user can delete the object as normal with the Admin delete command + This prevents a normal video linked to by a subclass video from being deleted + Called from GlossVideoAdmin + :model: GlossVideoAdmin + """ for obj in queryset: if not hasattr(obj, 'glossvideonme') and not hasattr(obj, 'glossvideoperspective'): - # this is not a subclass video that was selected by the user + # the selected gloss video is not a subclass video continue - - video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) + relative_path = str(obj.videofile) + if not relative_path: + continue + video_file_full_path = Path(WRITABLE_FOLDER, relative_path) + # ignore the files that have the correct filename if hasattr(obj, 'glossvideonme') and filename_matches_nme(video_file_full_path): continue if hasattr(obj, 'glossvideoperspective') and filename_matches_perspective(video_file_full_path): continue - # erase the file path if it has the wrong format + # erase the file path since it has the wrong format if DEBUG_VIDEOS: - print('unlink_files: erase incorrect path from object: ', obj, video_file_full_path) + print('unlink_files: erase incorrect path from NME or Perspective object: ', obj, video_file_full_path) obj.videofile.name = "" obj.save() From e1aea910548c68afd7aac23b5e99f638b16dd66c Mon Sep 17 00:00:00 2001 From: susanodd Date: Tue, 18 Feb 2025 11:14:19 +0100 Subject: [PATCH 15/24] #1398: Per review, added exceptions caught (not this issue) --- signbank/video/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/signbank/video/models.py b/signbank/video/models.py index 013d1f8c1..11e2ed54c 100755 --- a/signbank/video/models.py +++ b/signbank/video/models.py @@ -355,7 +355,7 @@ def make_poster_image(self): from signbank.tools import generate_still_image try: generate_still_image(self) - except OSError: + except (OSError, PermissionError): import sys print('Error generating still image', sys.exc_info()) @@ -377,7 +377,7 @@ def delete_files(self): small_video_path = self.small_video() try: os.unlink(self.videofile.path) - except OSError: + except (OSError, PermissionError): pass def reversion(self, revert=False): @@ -503,7 +503,7 @@ def delete_files(self, only_eaf=False): if not only_eaf: video_path = os.path.join(settings.WRITABLE_FOLDER, settings.ANNOTATEDSENTENCE_VIDEO_DIRECTORY, self.annotatedsentence.get_dataset().acronym, str(self.annotatedsentence.id)) shutil.rmtree(video_path) - except OSError: + except (OSError, PermissionError): pass def get_eaffile_name(self): @@ -732,7 +732,7 @@ def make_poster_image(self): from signbank.tools import generate_still_image try: generate_still_image(self) - except OSError: + except (OSError, PermissionError): import sys print('Error generating still image', sys.exc_info()) From 51740274e138774189b53f9cc704e93ad8ae88a0 Mon Sep 17 00:00:00 2001 From: susanodd Date: Tue, 18 Feb 2025 12:22:32 +0100 Subject: [PATCH 16/24] #1398: Per review, guarded clauses (also fixed other places in file) --- signbank/video/admin.py | 58 +++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 0b8b41dc2..d581f8059 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -57,10 +57,9 @@ def matching_file_group(videofile, key): if not key: return False video_file_full_path = Path(WRITABLE_FOLDER, videofile) - if video_file_full_path.exists(): - return video_file_full_path.group() == key - else: + if not video_file_full_path.exists(): return False + return video_file_full_path.group() == key queryset_res = queryset.values('id', 'videofile') results = [qv['id'] for qv in queryset_res @@ -239,11 +238,10 @@ def matching_file_type(videofile, key): if not key: return False video_file_full_path = Path(WRITABLE_FOLDER, videofile) - if video_file_full_path.exists(): - file_extension = video_file_type_extension(video_file_full_path) - return key == str((file_extension == '.mp4')) - else: + if not video_file_full_path.exists(): return key == 'False' + file_extension = video_file_type_extension(video_file_full_path) + return key == str((file_extension == '.mp4')) queryset_res = queryset.values('id', 'videofile') results = [qv['id'] for qv in queryset_res @@ -476,56 +474,48 @@ def file_timestamp(self, obj=None): if obj is None or not str(obj.videofile): return "" video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) - if os.path.exists(video_file_full_path): - return DT.datetime.fromtimestamp(os.path.getctime(video_file_full_path)) - else: + if not os.path.exists(video_file_full_path): return "" + return DT.datetime.fromtimestamp(os.path.getctime(video_file_full_path)) def file_group(self, obj=None): # this will display a group in the list view if obj is None or not str(obj.videofile): return "" - else: - video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) - if video_file_full_path.exists(): - group = video_file_full_path.group() - return group - else: - return "" + video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) + if not video_file_full_path.exists(): + return "" + group = video_file_full_path.group() + return group def file_size(self, obj=None): # this will display a group in the list view if obj is None or not str(obj.videofile): return "" - else: - video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) - if video_file_full_path.exists(): - size = str(video_file_full_path.stat().st_size) - return size - else: - return "" + video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) + if not video_file_full_path.exists(): + return "" + size = str(video_file_full_path.stat().st_size) + return size def permissions(self, obj=None): # this will display a group in the list view if obj is None or not str(obj.videofile): return "" - else: - video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) - if video_file_full_path.exists(): - stats = stat.filemode(video_file_full_path.stat().st_mode) - return stats - else: - return "" + video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) + if not video_file_full_path.exists(): + return "" + stats = stat.filemode(video_file_full_path.stat().st_mode) + return stats def video_type(self, obj=None): # if the file exists, this will display its timestamp in the list view if obj is None or not str(obj.videofile): return "" video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) - if os.path.exists(video_file_full_path): - return video_file_type_extension(video_file_full_path) - else: + if not os.path.exists(video_file_full_path): return "" + return video_file_type_extension(video_file_full_path) def get_list_display_links(self, request, list_display): # do not allow the user to view individual revisions in list From 618d9cd6035c4ce8c800c8e5371a324c94c964ce Mon Sep 17 00:00:00 2001 From: susanodd Date: Wed, 19 Feb 2025 13:41:48 +0100 Subject: [PATCH 17/24] #1398: Per review: More code made guarded to remove indentation Changed typo in name of function Made delete_files return Boolean, print OS exception text in log Moved functions from admin.py to models.py so they can be shared by models Check on correct filename before deleting doubly linked file in normal delete --- signbank/video/admin.py | 26 ++--------- signbank/video/convertvideo.py | 10 ++-- signbank/video/models.py | 83 ++++++++++++++++++++++------------ 3 files changed, 64 insertions(+), 55 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index d581f8059..0eee9a528 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin -from signbank.video.models import GlossVideo, GlossVideoHistory, AnnotatedVideo, ExampleVideoHistory +from signbank.video.models import (GlossVideo, GlossVideoHistory, AnnotatedVideo, ExampleVideoHistory, + filename_matches_nme, filename_matches_perspective, filename_matches_video, filename_matches_backup_video) from signbank.dictionary.models import Dataset, AnnotatedGloss, Gloss from django.contrib.auth.models import User from signbank.settings.base import * @@ -107,26 +108,6 @@ def matching_file_exists(videofile, key): return queryset.all() -def filename_matches_nme(filename): - filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) - return re.search(r".+-(\d+)_(nme_\d+|nme_\d+_left|nme_\d+_right)$", filename_without_extension) - - -def filename_matches_perspective(filename): - filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) - return re.search(r".+-(\d+)_(left|right|nme_\d+_left|nme_\d+_right)$", filename_without_extension) - - -def filename_matches_video(filename): - filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) - return re.search(r".+-(\d+)$", filename_without_extension) - - -def filename_matches_backup_video(filename): - filename_with_extension = os.path.basename(filename) - return re.search(r".+-(\d+)\.(mp4|m4v|mov|webm|mkv|m2v)\.(bak\d+)$", filename_with_extension) - - class GlossVideoFilenameFilter(admin.SimpleListFilter): """ Filter the GlossVideo objects on whether the filename is correct for the type of video @@ -357,6 +338,9 @@ def remove_backups(modeladmin, request, queryset): continue video_file_full_path = os.path.join(WRITABLE_FOLDER, relative_path) if not os.path.exists(video_file_full_path): + if DEBUG_VIDEOS: + print('video:admin:remove_backups:delete object: ', relative_path) + obj.delete() continue # first remove the video file so the GlossVideo object can be deleted later # this is done to avoid the signals on GlossVideo delete diff --git a/signbank/video/convertvideo.py b/signbank/video/convertvideo.py index baf8bb461..8cdedae94 100755 --- a/signbank/video/convertvideo.py +++ b/signbank/video/convertvideo.py @@ -216,7 +216,7 @@ def make_thumbnail_video(sourcefile, targetfile): ACCEPTABLE_VIDEO_EXTENSIONS = ['.mp4', '.mov', '.webm', '.m4v', '.mkv', '.m2v'] -def get_video_extension_from_stored_filenpath(video_file_full_path): +def get_video_extension_from_stored_filepath(video_file_full_path): file_path, file_extension = os.path.splitext(video_file_full_path) if '.bak' in file_extension: # this is a backup file, remove the extension again @@ -229,13 +229,11 @@ def get_video_extension_from_stored_filenpath(video_file_full_path): def video_file_type_extension(video_file_full_path): - if not video_file_full_path: - return '' - if 'glossvideo' not in video_file_full_path: + if not video_file_full_path or 'glossvideo' not in video_file_full_path: return '' if not os.path.exists(video_file_full_path): - return get_video_extension_from_stored_filenpath(video_file_full_path) + return get_video_extension_from_stored_filepath(video_file_full_path) filetype_output = subprocess.run(["file", video_file_full_path], stdout=subprocess.PIPE) filetype = str(filetype_output.stdout) @@ -255,7 +253,7 @@ def video_file_type_extension(video_file_full_path): # no match found, print something to the log and just keep using mp4 if DEBUG_VIDEOS: print('video:admin:convertvideo:video_file_type_extension:file:UNKNOWN ', filetype) - desired_video_extension = get_video_extension_from_stored_filenpath(video_file_full_path) + desired_video_extension = get_video_extension_from_stored_filepath(video_file_full_path) return desired_video_extension diff --git a/signbank/video/models.py b/signbank/video/models.py index 11e2ed54c..4aa3c47bf 100755 --- a/signbank/video/models.py +++ b/signbank/video/models.py @@ -9,6 +9,8 @@ import time import stat import shutil +# from signbank.video.admin import filename_matches_nme, filename_matches_perspective + from signbank.video.convertvideo import extract_frame, convert_video, probe_format, make_thumbnail_video, generate_image_sequence, remove_stills @@ -28,6 +30,27 @@ from signbank.dictionary.models import Gloss, Language +def filename_matches_nme(filename): + filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) + return re.search(r".+-(\d+)_(nme_\d+|nme_\d+_left|nme_\d+_right)$", filename_without_extension) + + +def filename_matches_perspective(filename): + filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) + return re.search(r".+-(\d+)_(left|right|nme_\d+_left|nme_\d+_right)$", filename_without_extension) + + +def filename_matches_video(filename): + filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) + return re.search(r".+-(\d+)$", filename_without_extension) + + +def filename_matches_backup_video(filename): + filename_with_extension = os.path.basename(filename) + return re.search(r".+-(\d+)\.(mp4|m4v|mov|webm|mkv|m2v)\.(bak\d+)$", filename_with_extension) + + + class GlossVideoStorage(FileSystemStorage): """Implement our shadowing video storage system""" @@ -754,7 +777,7 @@ def delete_files(self): if settings.DEBUG_VIDEOS: print('delete_files GlossVideo: ', str(self.videofile)) - if not self.videofile.name: + if not self.videofile or not self.videofile.name: return small_video_path = self.small_video() @@ -945,47 +968,51 @@ def move_video(self, move_files_on_disk=True): :return: """ old_path = str(self.videofile) + if not move_files_on_disk or not old_path: + return new_path = get_video_file_path(self, old_path, nmevideo=True, perspective='', offset=self.offset, version=0) - if old_path != new_path: - if move_files_on_disk: - source = os.path.join(settings.WRITABLE_FOLDER, old_path) - destination = os.path.join(settings.WRITABLE_FOLDER, new_path) - if os.path.exists(source): - destination_dir = os.path.dirname(destination) - if not os.path.exists(destination_dir): - os.makedirs(destination_dir) - if os.path.isdir(destination_dir): - shutil.move(source, destination) + if old_path == new_path: + return + source = os.path.join(settings.WRITABLE_FOLDER, old_path) + destination = os.path.join(settings.WRITABLE_FOLDER, new_path) + if not os.path.exists(source): + return + destination_dir = os.path.dirname(destination) + if not os.path.exists(destination_dir): + os.makedirs(destination_dir) + if os.path.isdir(destination_dir): + shutil.move(source, destination) - self.videofile.name = new_path - self.save() + self.videofile.name = new_path + self.save() def delete_files(self): """Delete the files associated with this object""" old_path = str(self.videofile) if not old_path: - return + return True file_system_path = os.path.join(settings.WRITABLE_FOLDER, old_path) - if settings.DEBUG_VIDEOS: - print('delete_files GlossVideoNME: ', file_system_path) + if filename_matches_nme(file_system_path) is None: + # this points to the normal video file, just erase the name rather than delete file + self.videofile.name = "" + self.save() + return True if not os.path.exists(file_system_path): - # Video file not found on server - # on the production server this is a problem - msg = "GlossVideoNME video file not found: " + file_system_path - print(msg) - return + return True try: os.unlink(file_system_path) - except (OSError, PermissionError): - if settings.DEBUG_VIDEOS: - print('delete_files exception GlossVideoNME OSError, PermissionError: ', file_system_path) - pass + return True + except (OSError, PermissionError) as e: + print(e) + return False def reversion(self, revert=False): """Delete the video file of this object""" - print("DELETE NME VIDEO", self.videofile.name) - self.delete_files() - self.delete() + status = self.delete_files() + if not status: + print("DELETE NME VIDEO FAILED: ", self.videofile.name) + else: + self.delete() PERSPECTIVE_CHOICES = (('left', 'Left'), From 3492a1fd693f1271448ba56690e18987161f7ccf Mon Sep 17 00:00:00 2001 From: susanodd Date: Thu, 20 Feb 2025 09:51:58 +0100 Subject: [PATCH 18/24] #1398: Per review - made other delete_files also return Boolean for symmetry. Also takes care of #1502 by checking if filename is correct before deleting --- signbank/video/models.py | 44 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/signbank/video/models.py b/signbank/video/models.py index 4aa3c47bf..996444303 100755 --- a/signbank/video/models.py +++ b/signbank/video/models.py @@ -1042,6 +1042,8 @@ def move_video(self, move_files_on_disk=True): return # other code does this too. It's a dubious way to obtain the path old_path = str(self.videofile) + if not old_path: + return new_path = get_video_file_path(self, old_path, nmevideo=False, perspective=str(self.perspective)) if old_path == new_path: return @@ -1055,38 +1057,36 @@ def move_video(self, move_files_on_disk=True): if os.path.isdir(destination_dir): shutil.move(source, destination) - self.videofile.name = new_path - self.save() - else: - # on the production server this is a problem - msg = "Perspective video file not found: " + source - print(msg) + self.videofile.name = new_path + self.save() def delete_files(self): """Delete the files associated with this object""" old_path = str(self.videofile) if not old_path: - return + return True file_system_path = os.path.join(settings.WRITABLE_FOLDER, old_path) - if settings.DEBUG_VIDEOS: - print('perspective video delete files: ', file_system_path) + if filename_matches_perspective(file_system_path) is None: + # this points to the normal video file, just erase the name rather than delete file + self.videofile.name = "" + self.save() + return True if not os.path.exists(file_system_path): - # Video file not found on server - # on the production server this is a problem - msg = "Perspective video file not found: " + file_system_path - print(msg) - return + return True try: os.unlink(file_system_path) - except (PermissionError, OSError): - msg = "Perspective video file could not be deleted: " + file_system_path - print(msg) + return True + except (PermissionError, OSError) as e: + print(e) + return False def reversion(self, revert=False): """Delete the video file of this object""" - print("DELETE Perspective VIDEO", self.videofile.name) - self.delete_files() - self.delete() + status = self.delete_files() + if not status: + print("DELETE Perspective VIDEO FAILED: ", self.videofile.name) + else: + self.delete() def move_videos_for_filter(filter, move_files_on_disk: bool=False) -> None: @@ -1248,10 +1248,10 @@ def delete_files(sender, instance, **kwargs): print('delete_files settings.DELETE_FILES_ON_GLOSSVIDEO_DELETE: ', settings.DELETE_FILES_ON_GLOSSVIDEO_DELETE) if hasattr(instance, 'glossvideonme'): # before deleting a GlossVideoNME object, delete the files - instance.delete_files() + status = instance.delete_files() elif hasattr(instance, 'glossvideoperspective'): # before deleting a GlossVideoPerspective object, delete the files - instance.delete_files() + status = instance.delete_files() elif settings.DELETE_FILES_ON_GLOSSVIDEO_DELETE: # before a GlossVideo object, only delete the files if the setting is True # default.py has this set to false so primary gloss video files are (never) deleted From 97d4861bfd253f7fc37ce8b9ec75e005f825f297 Mon Sep 17 00:00:00 2001 From: susanodd Date: Fri, 21 Feb 2025 08:08:51 +0100 Subject: [PATCH 19/24] #1398: Per review: Rewrote video file extension helper function. Renamed, added patterns, added comments, mp4 still last resort value --- signbank/video/admin.py | 11 +++++++--- signbank/video/convertvideo.py | 40 ++++++++++++++++++++++++---------- signbank/video/models.py | 5 +---- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 0eee9a528..971839e59 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -257,7 +257,7 @@ def queryset(self, request, queryset): return queryset.all() -@admin.action(description="Rename video files to match type") +@admin.action(description="Rename normal video files to match type") def rename_extension_videos(modeladmin, request, queryset): """ Command for the GlossVideo objects selected in queryset @@ -282,6 +282,13 @@ def rename_extension_videos(modeladmin, request, queryset): video_file_full_path = os.path.join(WRITABLE_FOLDER, current_relative_path) + # use the file system command 'file' to determine the extension for the type of video file + desired_video_extension = video_file_type_extension(video_file_full_path) + if not desired_video_extension: + # if we get here, the file extension for the video type could not be determined + # either there is no file for this object or it has an unknown video type + continue + # retrieve the actual filename stored in the gloss video object # and also compute the name it should have @@ -292,8 +299,6 @@ def rename_extension_videos(modeladmin, request, queryset): dataset_dir = gloss.lemma.dataset.acronym desired_filename_without_extension = f'{idgloss}-{gloss.id}' - # use the file system command 'file' to determine the extension for the type of video file - desired_video_extension = video_file_type_extension(video_file_full_path) if glossvideo.version > 0: desired_extension = f'{desired_video_extension}.bak{glossvideo.id}' else: diff --git a/signbank/video/convertvideo.py b/signbank/video/convertvideo.py index 8cdedae94..5f29216e6 100755 --- a/signbank/video/convertvideo.py +++ b/signbank/video/convertvideo.py @@ -213,18 +213,34 @@ def make_thumbnail_video(sourcefile, targetfile): os.remove(temp_target) +# this is only for documentation purposes in the patterns, it's not a setting +# these were found to work properly on Ubuntu and match older files and older code ACCEPTABLE_VIDEO_EXTENSIONS = ['.mp4', '.mov', '.webm', '.m4v', '.mkv', '.m2v'] -def get_video_extension_from_stored_filepath(video_file_full_path): - file_path, file_extension = os.path.splitext(video_file_full_path) - if '.bak' in file_extension: - # this is a backup file, remove the extension again - file_path, file_extension = os.path.splitext(file_path) - if file_extension not in ACCEPTABLE_VIDEO_EXTENSIONS: - # some other extension is present in the filename - file_extension = '.mp4' - return file_extension +def extension_on_filename(filename): + # used to retrieve a video type file extension from a filename where there is no file + # if this is a backup file, then the extension at the end is not the video file type extension + # otherwise, just retrieve the normal extension from the filename + # caveat, some video files in the database have weird backup sequences and have no video extension + # the .mp4 is for those + filename_with_extension = os.path.basename(filename) + filename_without_extension, ext = os.path.splitext(os.path.basename(filename)) + + if ext in ACCEPTABLE_VIDEO_EXTENSIONS: + return ext + + m = re.search(r".+-(\d+)\.(mp4|m4v|mov|webm|mkv|m2v)\.(bak\d+)$", filename_with_extension) + if m: + return m.group(2) + m = re.search(r".+-(\d+)\.(mp4|m4v|mov|webm|mkv|m2v)\.(bak\d+)$", filename_without_extension) + if m: + return m.group(2) + # the function is only called if there is no file + # if we get here, the filename does not match any correct pattern and the extension + # does not match any video file type + # this allows the function to work on the development servers + return '.mp4' def video_file_type_extension(video_file_full_path): @@ -233,7 +249,7 @@ def video_file_type_extension(video_file_full_path): return '' if not os.path.exists(video_file_full_path): - return get_video_extension_from_stored_filepath(video_file_full_path) + return extension_on_filename(video_file_full_path) filetype_output = subprocess.run(["file", video_file_full_path], stdout=subprocess.PIPE) filetype = str(filetype_output.stdout) @@ -250,10 +266,10 @@ def video_file_type_extension(video_file_full_path): elif 'MPEG-2' in filetype: desired_video_extension = '.m2v' else: - # no match found, print something to the log and just keep using mp4 + # no match found, print something to the log and just keep using what's on the filename if DEBUG_VIDEOS: print('video:admin:convertvideo:video_file_type_extension:file:UNKNOWN ', filetype) - desired_video_extension = get_video_extension_from_stored_filepath(video_file_full_path) + desired_video_extension = extension_on_filename(video_file_full_path) return desired_video_extension diff --git a/signbank/video/models.py b/signbank/video/models.py index 996444303..d25115274 100755 --- a/signbank/video/models.py +++ b/signbank/video/models.py @@ -9,10 +9,8 @@ import time import stat import shutil -# from signbank.video.admin import filename_matches_nme, filename_matches_perspective - -from signbank.video.convertvideo import extract_frame, convert_video, probe_format, make_thumbnail_video, generate_image_sequence, remove_stills +from signbank.video.convertvideo import extract_frame, convert_video, make_thumbnail_video, generate_image_sequence, remove_stills from django.core.files.storage import FileSystemStorage from django.contrib.auth import models as authmodels @@ -50,7 +48,6 @@ def filename_matches_backup_video(filename): return re.search(r".+-(\d+)\.(mp4|m4v|mov|webm|mkv|m2v)\.(bak\d+)$", filename_with_extension) - class GlossVideoStorage(FileSystemStorage): """Implement our shadowing video storage system""" From 3c68049061b22eb3ef640a39a5ef5b38d3d758da Mon Sep 17 00:00:00 2001 From: susanodd Date: Mon, 24 Feb 2025 09:01:00 +0100 Subject: [PATCH 20/24] #1398: Per review, added doc-strings to column methods of GlossVideoAdmin --- signbank/video/admin.py | 42 ++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 971839e59..0d6240d65 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -441,7 +441,11 @@ class GlossVideoAdmin(admin.ModelAdmin): actions = [rename_extension_videos, remove_backups, renumber_backups, unlink_files] def video_file(self, obj=None): - # this will display the full path in the list view + """ + column VIDEO FILE + this will display the full path in the list view, also for non-existent files + this allows to browse the file paths also on the development servers + """ if obj is None or not str(obj.videofile): return "" video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) @@ -449,17 +453,28 @@ def video_file(self, obj=None): return video_file_full_path def perspective(self, obj=None): + """ + column PERSPECTIVE + This will be True if the object is of subclass GlossVideoPerspective + """ if obj is None: return "" return obj.is_glossvideoperspective() is True def NME(self, obj=None): + """ + column NME + This will be True if the object is of subclass GlossVideoNME + """ if obj is None: return "" return obj.is_glossvideonme() is True def file_timestamp(self, obj=None): - # if the file exists, this will display its timestamp in the list view + """ + column FILE TIMESTAMP + if the file exists, this will display its timestamp in the list view + """ if obj is None or not str(obj.videofile): return "" video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) @@ -468,7 +483,10 @@ def file_timestamp(self, obj=None): return DT.datetime.fromtimestamp(os.path.getctime(video_file_full_path)) def file_group(self, obj=None): - # this will display a group in the list view + """ + column FILE GROUP + if the file exists, this will display the file system group in the list view + """ if obj is None or not str(obj.videofile): return "" video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) @@ -478,7 +496,10 @@ def file_group(self, obj=None): return group def file_size(self, obj=None): - # this will display a group in the list view + """ + column FILE SIZE + if the file exists, this will display the file size in the list view + """ if obj is None or not str(obj.videofile): return "" video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) @@ -488,7 +509,10 @@ def file_size(self, obj=None): return size def permissions(self, obj=None): - # this will display a group in the list view + """ + column PERMISSIONS + if the file exists, this will display the file system permissions in the list view + """ if obj is None or not str(obj.videofile): return "" video_file_full_path = Path(WRITABLE_FOLDER, str(obj.videofile)) @@ -498,7 +522,10 @@ def permissions(self, obj=None): return stats def video_type(self, obj=None): - # if the file exists, this will display its timestamp in the list view + """ + column VIDEO TYPE + if the file exists, this will display the video type in file extension format + """ if obj is None or not str(obj.videofile): return "" video_file_full_path = os.path.join(WRITABLE_FOLDER, str(obj.videofile)) @@ -507,7 +534,7 @@ def video_type(self, obj=None): return video_file_type_extension(video_file_full_path) def get_list_display_links(self, request, list_display): - # do not allow the user to view individual revisions in list + # do not allow the user to click on data of individual elements in the list display self.list_display_links = (None, ) return self.list_display_links @@ -515,6 +542,7 @@ def has_add_permission(self, request): return False def has_delete_permission(self, request, obj=None): + # Only allow to delete objects without any file if not self.file_timestamp(obj): return True return False From cac71661f00ae0f3c5bb43bc156ca0a3470565ba Mon Sep 17 00:00:00 2001 From: susanodd Date: Tue, 25 Feb 2025 13:56:23 +0100 Subject: [PATCH 21/24] #1398: Per review: Move admin-command selected backup video files to trash location Files renamed to include dataset and 2char folder at front of filename to keep trash flat. --- signbank/settings/server_specific/default.py | 1 + signbank/video/admin.py | 39 ++++++++++++-------- signbank/video/models.py | 11 ++++++ 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/signbank/settings/server_specific/default.py b/signbank/settings/server_specific/default.py index de8d3540b..4c5c336e5 100644 --- a/signbank/settings/server_specific/default.py +++ b/signbank/settings/server_specific/default.py @@ -23,6 +23,7 @@ DATASET_METADATA_DIRECTORY = 'metadata_eafs' TEST_DATA_DIRECTORY = 'test_data' BACKUP_VIDEOS_FOLDER = 'video_backups' +DELETED_FILES_FOLDER = 'prullenmand' #Tmp folder to use TMP_DIR = '/tmp' diff --git a/signbank/video/admin.py b/signbank/video/admin.py index 0d6240d65..bf151f53a 100755 --- a/signbank/video/admin.py +++ b/signbank/video/admin.py @@ -1,18 +1,18 @@ from django.contrib import admin from signbank.video.models import (GlossVideo, GlossVideoHistory, AnnotatedVideo, ExampleVideoHistory, - filename_matches_nme, filename_matches_perspective, filename_matches_video, filename_matches_backup_video) + filename_matches_nme, filename_matches_perspective, filename_matches_video, filename_matches_backup_video, flattened_video_path) from signbank.dictionary.models import Dataset, AnnotatedGloss, Gloss from django.contrib.auth.models import User from signbank.settings.base import * -from signbank.settings.server_specific import WRITABLE_FOLDER, FILESYSTEM_SIGNBANK_GROUPS, DEBUG_VIDEOS +from signbank.settings.server_specific import WRITABLE_FOLDER, FILESYSTEM_SIGNBANK_GROUPS, DEBUG_VIDEOS, DELETED_FILES_FOLDER from django.utils.translation import gettext_lazy as _ from signbank.tools import get_two_letter_dir from signbank.video.convertvideo import video_file_type_extension -import re from pathlib import Path import os import stat +import shutil import datetime as DT @@ -325,12 +325,14 @@ def rename_extension_videos(modeladmin, request, queryset): glossvideo.save() -@admin.action(description="Remove selected backups") +@admin.action(description="Move selected backup files to trash") def remove_backups(modeladmin, request, queryset): """ Command for the GlossVideo objects selected in queryset The command appears in the admin pull-down list of commands for the selected gloss videos - The command removes the selected backup files + The command moves the selected backup files to the DELETED_FILES_FOLDER location + To prevent the gloss video object from pointing to the deleted files folder location + the name stored in the object is set to empty before deleting the object Other selected objects are ignored This allows the user to keep a number of the backup files by not selecting everything Called from GlossVideoAdmin @@ -347,18 +349,23 @@ def remove_backups(modeladmin, request, queryset): print('video:admin:remove_backups:delete object: ', relative_path) obj.delete() continue - # first remove the video file so the GlossVideo object can be deleted later + # move the video file to DELETED_FILES_FOLDER and erase the videofile name in the object # this is done to avoid the signals on GlossVideo delete - try: - os.unlink(obj.videofile.path) - os.remove(video_file_full_path) - if DEBUG_VIDEOS: - print('video:admin:remove_backups:os.remove: ', video_file_full_path) - except (OSError, PermissionError): - if DEBUG_VIDEOS: - print('Exception video:admin:remove_backups: could not delete video file: ', video_file_full_path) - continue - # only backup videos are deleted here + deleted_file_name = flattened_video_path(relative_path) + destination = os.path.join(WRITABLE_FOLDER, DELETED_FILES_FOLDER, deleted_file_name) + destination_dir = os.path.dirname(destination) + if not os.path.exists(destination_dir): + os.makedirs(destination_dir) + if os.path.isdir(destination_dir): + try: + obj.videofile.name = "" + obj.save() + shutil.move(video_file_full_path, destination) + if DEBUG_VIDEOS: + print('video:admin:remove_backups:shutil.move: ', video_file_full_path, destination) + except (OSError, PermissionError) as e: + print(e) + continue # the object does not point to anything anymore, so it can be deleted if DEBUG_VIDEOS: print('video:admin:remove_backups:delete object: ', relative_path) diff --git a/signbank/video/models.py b/signbank/video/models.py index d25115274..fbf7ca50f 100755 --- a/signbank/video/models.py +++ b/signbank/video/models.py @@ -48,6 +48,17 @@ def filename_matches_backup_video(filename): return re.search(r".+-(\d+)\.(mp4|m4v|mov|webm|mkv|m2v)\.(bak\d+)$", filename_with_extension) +def flattened_video_path(filename): + relative_path_components, filename_component = os.path.split(filename) + m = re.search(r"^glossvideo/(.+)/(..)$", relative_path_components) + if m: + # prefix the filename with part of the dataset-specific part of the relative path + dataset_folder = m.group(1) + two_char_folder = m.group(2) + return dataset_folder + '_' + two_char_folder + '_' + filename_component + return filename_component + + class GlossVideoStorage(FileSystemStorage): """Implement our shadowing video storage system""" From 7bbf95b6b1d71e4b8aad909f28972d0911ea3b97 Mon Sep 17 00:00:00 2001 From: susanodd Date: Wed, 26 Feb 2025 08:28:26 +0100 Subject: [PATCH 22/24] #1398: Per review: Added docstring and fstring to new flattened_video_path function. --- signbank/video/models.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/signbank/video/models.py b/signbank/video/models.py index fbf7ca50f..fa82f7863 100755 --- a/signbank/video/models.py +++ b/signbank/video/models.py @@ -48,15 +48,20 @@ def filename_matches_backup_video(filename): return re.search(r".+-(\d+)\.(mp4|m4v|mov|webm|mkv|m2v)\.(bak\d+)$", filename_with_extension) -def flattened_video_path(filename): - relative_path_components, filename_component = os.path.split(filename) - m = re.search(r"^glossvideo/(.+)/(..)$", relative_path_components) +def flattened_video_path(relative_path): + """ + This constructs the filename to be used in the DELETED_FILES_FOLDER + Take apart the gloss video relative path + If this succeeds, prefix the filename with the dataset-specific components + Otherwise just return the filename + """ + relative_path_folders, filename = os.path.split(relative_path) + m = re.search(r"^glossvideo/(.+)/(..)$", relative_path_folders) if m: - # prefix the filename with part of the dataset-specific part of the relative path dataset_folder = m.group(1) two_char_folder = m.group(2) - return dataset_folder + '_' + two_char_folder + '_' + filename_component - return filename_component + return f"{dataset_folder}_{two_char_folder}_{filename}" + return filename class GlossVideoStorage(FileSystemStorage): From c9e2a826b23e6913511c96ea7c7cc881ed2e4117 Mon Sep 17 00:00:00 2001 From: susanodd Date: Wed, 26 Feb 2025 10:35:00 +0100 Subject: [PATCH 23/24] #1398: Per review: Gloss Video Admin Wiki Text File Stored as a text file so it gets maintained in GitHub. --- signbank/video/video_admin_wiki.txt | 100 ++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 signbank/video/video_admin_wiki.txt diff --git a/signbank/video/video_admin_wiki.txt b/signbank/video/video_admin_wiki.txt new file mode 100644 index 000000000..153e3d45e --- /dev/null +++ b/signbank/video/video_admin_wiki.txt @@ -0,0 +1,100 @@ + +GLOSS VIDEO ADMIN + +COLUMNS + +The tabular display shows the following columns as indicated in list_display of the class GlossVideoAdmin: + + ID, GLOSS, VIDEO FILE, PERSPECTIVE, NME, FILE_TIMESTAMP, FILE GROUP, PERMISSIONS, FILE SIZE, VIDEO TYPE, VERSION + +Some of these column values are computed based on the associated video file. +For these, there are corresponding methods in the class. Django displays the columns in upper case. +The version column indicates a backup file when it's greater than 0. + +FILTERS + +Filters are available as shown in list_filter to reduce the amount of data shown in the table. +For each filter, a class model has been defined. The filters appear in the right-hand column of the admin. + +GlossVideoDatasetFilter (Dataset) filters on the dataset. A list of all the dataset acronyms is shown. + +GlossVideoFileSystemGroupFilter (File System Group) filters on the file system group. The selection in FILESYSTEM_SIGNBANK_GROUPS is shown. + +The remaining filters are all Boolean + +GlossVideoExistenceFilter (File Exists) filters on whether a video file exists for the gloss video object + +GlossVideoFileTypeFilter (MP4 File) filters on the type of the video file, MP4 File (True, False) + +GlossVideoNMEFilter (NME Video) filters on whether the gloss video object is a non-manual elements video + +GlossVideoPerspectiveFilter (Perspective Video) filters on whether the gloss video object is a perspective video + +GlossVideoFilenameFilter (Filename Correct) filters on whether the filename associated with the file is correct +Methods defined in video models.py are available for this. +They compare the filename to acceptable patterns for the type of file. +Some older files use an older pattern for the backup filenames that did not include the video file type extension in the filename. (See Commands) + + +GlossVideoBackupFilter (Backup Video) filters on whether the gloss video object is a backup video + + +SEARCH + +There is one search field input area in the Gloss Video Admin. +Here you can search on the initial text (regular expression carrot) that appears in the gloss annotation or lemma. +The Django query strings appear in search_fields: + +'^gloss__annotationidglosstranslation__text' search on the gloss annotation text, any language + +'^gloss__lemma__lemmaidglosstranslation__text' search on the lemma annotation text, any language + + +ACTIONS + +Django allows to make a selection from the objects displayed on the specific page of admin results. +To this queryset, a command can be applied to the selected objects. +These are shown in the actions field of the class. The commands are shown in a pull-down list. +The primary default action is Delete, which also appears in the pull-down. +Delete does not appear in the actions list of the class, but as a method of the class. + +Signbank only allows to delete gloss video objects in the admin when there is no video file. +This is because of the backup system, which is invoked via signals when an object is deleted. +As a safeguard, normal delete is thus not available if there is a video file. + +Due to legacy code, backup files have undergone name changes over the years. +Originally, the backup files had as an extension, sequences of ".bak.bak" added to them where the number of extensions corresponded to the version. +The video file type was omitted because all videos were converted to mp4. +However, with the introduction of webcam capture and the API, the various video formats were no longer converted. +It's also browser-specific whether a file is a video file. + +The Gloss Video Admin queryset commands are as follows: + + +"Rename normal video files to match type" (rename_extension_videos) + +This command only applies to selected normal video file objects, including backup files. +The filenames are updated to match the acceptable pattern. +This has the result of repairing legacy backup video names to match the new format. +It also makes the video file type file extension match the type of the video file. +This is necessary for legacy files where there was no video type in the filename. +It also applies to videos that were not converted to mp4 but include mp4 in the filename. + + +"Move selected backup files to trash" (remove_backups) + +This command moves the selected backup files to the DELETED_FILES_FOLDER location. +This only applies to objects in the query, allowing the user to keep a number of the backup files by not selecting everything. +The files are renamed as {dataset_acronym}_{two_char_lemma}_{filename}. This allows convenient sorting of the DELETED_FILES_FOLDER. + +"Renumber selected backups" (renumber_backups) + +If some backup files are missing or the objects have been deleted, the version numbers can be made sequential again with this command. +This also works for all the backup objects for the gloss of any selected objects, allowing to only select one of them, but renumber them all. + +"Set incorrect NME/Perspective filenames to empty string" (unlink_files) + +For some files in the database, a subclass perspective or NME video may be pointing to the normal video file. +These can be found using the filter Filename Correct plus the subclass filter. +Using this command, the name stored in the object can be set to empty to allow deletion of the object without deleting the file. +This command is only applied on subclass objects and when the filename is not correct. Other objects in the queryset are ignored. From 677a610b99bb0432d053360caa27bc37673ea698 Mon Sep 17 00:00:00 2001 From: susanodd Date: Thu, 27 Feb 2025 08:28:15 +0100 Subject: [PATCH 24/24] #1398: Per review: Enhanced documentation in text file for video admin wiki. Added text describing filenames and storage structure. This provides context for the commands and filters. --- signbank/video/video_admin_wiki.txt | 33 ++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/signbank/video/video_admin_wiki.txt b/signbank/video/video_admin_wiki.txt index 153e3d45e..e98e4a7d8 100644 --- a/signbank/video/video_admin_wiki.txt +++ b/signbank/video/video_admin_wiki.txt @@ -1,16 +1,47 @@ GLOSS VIDEO ADMIN + +Gloss Video Admin allows to view the filenames and file properties of stored video objects. +The video files themselves are stored in location GLOSS_VIDEO_DIRECTORY in a folder for the dataset. +The dataset folder is organised in sub-folders based on the first two characters of the lemma text of the gloss. +The lemma text is that of the default language of the dataset. The dataset folder is the dataset acronym. + +Caution: Modification of either the dataset acronym or its default language will rename and move all of the video files for the dataset. + + +The Gloss Video Admin is described below. + + +VIDEO FILENAMES + +Gloss video filenames have the following structure, where attribute idgloss is the lemma text in the default language of the dataset. +The description uses pseudo-code patterns that include regular expression syntax. Video file type extensions are shown explicitly. + +Primary video: {gloss.idgloss}-{gloss.id}.(mp4|m4v|mov|webm|mkv|m2v) + +Perspective video: {gloss.idgloss}-{gloss.id}_(left|right|nme_\d+_left|nme_\d+_right).(mp4|m4v|mov|webm|mkv|m2v) +The perspective video pattern also matches perspective NME video filenames. + +Non-manual elements video: {gloss.idgloss}-{gloss.id}_(nme_\d+|nme_\d+_left|nme_\d+_right).(mp4|m4v|mov|webm|mkv|m2v) +The ciphers string after the "nme_" in the pattern is the ordering index of the video, for identification and display. + +Backup video: {gloss.idgloss}-{gloss.id}.(mp4|m4v|mov|webm|mkv|m2v).(bak\d+) +The ciphers string after the "bak" is the ID of the backup video object. These are internal and not visible to users. + + + COLUMNS The tabular display shows the following columns as indicated in list_display of the class GlossVideoAdmin: - ID, GLOSS, VIDEO FILE, PERSPECTIVE, NME, FILE_TIMESTAMP, FILE GROUP, PERMISSIONS, FILE SIZE, VIDEO TYPE, VERSION + ID, GLOSS, VIDEO FILE, PERSPECTIVE, NME, FILE TIMESTAMP, FILE GROUP, PERMISSIONS, FILE SIZE, VIDEO TYPE, VERSION Some of these column values are computed based on the associated video file. For these, there are corresponding methods in the class. Django displays the columns in upper case. The version column indicates a backup file when it's greater than 0. + FILTERS Filters are available as shown in list_filter to reduce the amount of data shown in the table.