diff --git a/openstax/settings/base.py b/openstax/settings/base.py index fc8b8a5f9..a10ce2f13 100644 --- a/openstax/settings/base.py +++ b/openstax/settings/base.py @@ -249,7 +249,7 @@ 'oxauth', 'webinars', 'donations', - 'wagtailimportexport', + 'wagtail_transfer', 'versions', 'oxmenus', # wagtail @@ -270,6 +270,47 @@ 'wagtail.contrib.settings', ] +#################### +# Wagtail Transfer # +#################### + +WAGTAILTRANSFER_SECRET_KEY = os.getenv('WAGTAILTRANSFER_SECRET_KEY', 'change-me-in-production') +if ENVIRONMENT != 'local' and WAGTAILTRANSFER_SECRET_KEY == 'change-me-in-production': + raise RuntimeError( + "WAGTAILTRANSFER_SECRET_KEY must be set to a secure value in non-local environments. " + "Current value is the insecure default placeholder." + ) + +# Configure sources to import content from. +# Each environment should define the sources it can pull from. +# Example: on prod, you might pull from staging; on local, from staging or prod. +# Override in environment-specific settings or via env vars. +WAGTAILTRANSFER_SOURCES = {} + +_transfer_source_name = os.getenv('WAGTAILTRANSFER_SOURCE_NAME') +_transfer_source_url = os.getenv('WAGTAILTRANSFER_SOURCE_URL') +_transfer_source_key = os.getenv('WAGTAILTRANSFER_SOURCE_KEY') +_transfer_vars = { + 'WAGTAILTRANSFER_SOURCE_NAME': _transfer_source_name, + 'WAGTAILTRANSFER_SOURCE_URL': _transfer_source_url, + 'WAGTAILTRANSFER_SOURCE_KEY': _transfer_source_key, +} +_set_vars = {name for name, value in _transfer_vars.items() if value} +if _set_vars and len(_set_vars) != len(_transfer_vars): + missing = sorted(set(_transfer_vars.keys()) - _set_vars) + raise RuntimeError( + "Invalid Wagtail Transfer source configuration: " + "the environment variables WAGTAILTRANSFER_SOURCE_NAME, " + "WAGTAILTRANSFER_SOURCE_URL, and WAGTAILTRANSFER_SOURCE_KEY " + "must either all be set or all be unset. " + f"Currently missing: {', '.join(missing)}." + ) +if _transfer_source_name and _transfer_source_url and _transfer_source_key: + WAGTAILTRANSFER_SOURCES[_transfer_source_name] = { + 'BASE_URL': _transfer_source_url, + 'SECRET_KEY': _transfer_source_key, + } + ######## # Cron # ######## diff --git a/openstax/urls.py b/openstax/urls.py index 61f236e0f..53f4fac42 100644 --- a/openstax/urls.py +++ b/openstax/urls.py @@ -5,6 +5,7 @@ from wagtail.admin import urls as wagtailadmin_urls from wagtailautocomplete.urls.admin import urlpatterns as autocomplete_admin_urls from wagtail import urls as wagtail_urls +from wagtail_transfer import urls as wagtailtransfer_urls from wagtail.documents import urls as wagtaildocs_urls from accounts import urls as accounts_urls @@ -48,6 +49,8 @@ path('apps/cms/api/spike/', include(wagtail_urls)), path('sitemap.xml', sitemap), + path('wagtail-transfer/', include(wagtailtransfer_urls)), + # For anything not caught by a more specific rule above, hand over to Wagtail's serving mechanism path('', include(wagtail_urls)), ] diff --git a/requirements/base.txt b/requirements/base.txt index b70c013f4..c7c4e27a1 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -33,4 +33,5 @@ vcrpy==7.0.0 wagtail==7.3.1 wagtail-autocomplete==0.12.0 wagtail-modeladmin==2.2.0 +wagtail-transfer==0.11 whitenoise==6.9.0 diff --git a/wagtailimportexport/__init__.py b/wagtailimportexport/__init__.py deleted file mode 100644 index 27dd626de..000000000 --- a/wagtailimportexport/__init__.py +++ /dev/null @@ -1 +0,0 @@ -#default_app_config = 'wagtailimportexport.apps.WagtailImportExportAppConfig' \ No newline at end of file diff --git a/wagtailimportexport/admin_urls.py b/wagtailimportexport/admin_urls.py deleted file mode 100644 index a32cfdf76..000000000 --- a/wagtailimportexport/admin_urls.py +++ /dev/null @@ -1,11 +0,0 @@ -from django.urls import re_path - -from wagtailimportexport import views - - -app_name = 'wagtailimportexport' -urlpatterns = [ - re_path(r'^import-page/$', views.import_page, name='import-page'), - re_path(r'^export-page/$', views.export_page, name='export-page'), - re_path(r'^$', views.index, name='index'), -] diff --git a/wagtailimportexport/apps.py b/wagtailimportexport/apps.py deleted file mode 100644 index bce1cd0a0..000000000 --- a/wagtailimportexport/apps.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.apps import AppConfig - - -class WagtailImportExportAppConfig(AppConfig): - name = 'wagtailimportexport' - label = 'wagtailimportexport' - verbose_name = "Import/Export Tool" - default = True diff --git a/wagtailimportexport/config.py b/wagtailimportexport/config.py deleted file mode 100644 index a80908065..000000000 --- a/wagtailimportexport/config.py +++ /dev/null @@ -1,3 +0,0 @@ -app_settings = { - 'max_file_size': 20000000, # Limit max file size to 20.000.000 bytes if checkbox is selected. -} \ No newline at end of file diff --git a/wagtailimportexport/exporting.py b/wagtailimportexport/exporting.py deleted file mode 100644 index 60b357ba2..000000000 --- a/wagtailimportexport/exporting.py +++ /dev/null @@ -1,189 +0,0 @@ -import io -import json -import logging -import copy - -from django.core.files import File -from django.core.serializers.json import DjangoJSONEncoder -from django.db.models.base import ModelState -from django.db.models.fields.files import FieldFile - -from wagtail.models import Page -from wagtail.blocks import StreamValue -from wagtail.images.models import Image - -from wagtailimportexport import functions -from wagtailimportexport.config import app_settings -from wagtail.documents.models import Document - - -def export_page(settings={'root_page': None, 'export_unpublished': False, - 'export_documents': False, 'export_images': False, 'null_pk': True, - 'null_fk': False, 'null_users': False - }): - """ - Exports the root_page as well as its children (if the setting is set). - - Arguments: - settings -- A dictionary that holds settings from cleared form data. - - Returns: - A zip archive of the exported pages; if it fails at any point, returns - None and logs the error. - """ - - settings = copy.deepcopy(settings) - - # If root_page is not set, then set it the main directory as default. - if not settings['root_page']: - settings['root_page'] = Page.objects.filter(url_path='/').first() - - # Get the list of the pages, (that are the descendant of the root_page). - pages = Page.objects.descendant_of( - settings['root_page'], inclusive=True).order_by('path').specific() - - # Filter the pages if export_unpublished is set to false. - if not settings['export_unpublished']: - pages = pages.filter(live=True) - - # Initialize the variables. - page_data = [] - exported_paths = set() - - # Start looping through pages and export their content. - for (i, page) in enumerate(pages): - parent_path = page.path[:-(Page.steplen)] - - # skip over pages whose parents haven't already been exported - # (which means that export_unpublished is false and the parent was unpublished) - if i == 0 or (parent_path in exported_paths): - - # Turn page data to a dictionary. - data = json.loads(page.to_json()) - locale = data['locale'] - - # look up document titles - if page.content_type.model == 'book': - cover = functions.document_title(data['cover']) - title_image = functions.document_title(data['title_image']) - hi_res_pdf = functions.document_title(data['high_resolution_pdf']) - lo_res_pdf = functions.document_title(data['low_resolution_pdf']) - community_logo = functions.document_title(data['community_resource_logo']) - community_feature_link = functions.document_title(data['community_resource_feature_link']) - - # Get list (and metadata) of images and documents to be exported. - images = list_fileobjects(page, settings, Image) if settings['export_images'] else {} - documents = list_fileobjects(page, settings, Document) if settings['export_documents'] else {} - - # Remove FKs - if settings['null_fk']: - functions.null_fks(page, data) - - #Remove the owner of the page. - if settings['null_users'] and not data.get('owner'): - data['owner'] = None - - # Null all the images. - if settings['export_images']: - for image in images: - if data.get(image) is not None: - data[image] = None - - data['pk'] = None - data['locale'] = locale - # add document titles to data - if page.content_type.model == 'book': - data['cover'] = cover - data['title_image'] = title_image - data['high_resolution_pdf'] = hi_res_pdf - data['low_resolution_pdf'] = lo_res_pdf - data['community_resource_logo'] = community_logo - data['community_resource_feature_link'] = community_feature_link - - # Export page data. - page_data.append({ - 'content': data, - 'model': page.content_type.model, - 'app_label': page.content_type.app_label, - 'images': images, - 'documents': documents - }) - - exported_paths.add(page.path) - - return functions.zip_contents(page_data) - - -def list_fileobjects(page, settings, objtype): - """ - Returns a dict of all fields that has the related_model of objtype as well as their metadata. - - Arguments: - page -- Page instance with supported fields. - settings -- Settings dictionary from main method. - objtype -- Image, Document from Wagtail. - - Returns: - A dictionary of fields with their respective metadata. - """ - - data = json.loads(page.to_json()) - - if objtype == Image: - related_model_by = "" - elif objtype == Document: - related_model_by = "" - else: - return {} - - objects = {} - for field in page._meta.get_fields(): - if field.related_model and str(field.related_model) == related_model_by: - if data[field.name]: - - try: - # Get the object instance. - instance = objtype.objects.get(pk=data[field.name]) - - # Null the object if the filesize is larger. - if instance.file.size > app_settings['max_file_size'] and settings['ignore_large_files']: - objects[field.name] = None - else: - objects[field.name] = instance_to_data(instance, null_users=settings['null_users']) - - except (FileNotFoundError, objtype.DoesNotExist): - logging.error("File for " + str(field.name) + " is not found on the environment, skipping.") - objects[field.name] = None - - else: - objects[field.name] = None - - return objects - - -def instance_to_data(instance, null_users=False): - """ - A utility to create JSON-able data from a model instance. - - Arguments: - instance -- objects.get() object instance. - null_users -- Whether to null user references. - - Returns: - A dictionary of metadata of instance. - """ - - data = {} - - for key, value in instance.__dict__.items(): - if isinstance(value, ModelState): - continue - elif null_users == True and ('user_id' in key or 'owner' in key): - data[key] = None - elif isinstance(value, StreamValue): - data[key] = json.dumps(value.stream_data, cls=DjangoJSONEncoder) - elif isinstance(value, FieldFile) or isinstance(value, File): - data[key] = {'name': value.name, 'size': value.size} - else: - data[key] = value - return data \ No newline at end of file diff --git a/wagtailimportexport/forms.py b/wagtailimportexport/forms.py deleted file mode 100644 index 048a7fefb..000000000 --- a/wagtailimportexport/forms.py +++ /dev/null @@ -1,96 +0,0 @@ -from django import forms - -from wagtail.admin.widgets import AdminPageChooser -from wagtail.models import Page -from wagtail.admin import widgets as wagtailadmin_widgets - - -admin_page_params = { - 'can_choose_root': True, - 'show_edit_link': False, - 'user_perms': 'copy_to' -} - - -class ImportPage(forms.Form): - """ - This form renders the import fields for zip archives. - """ - - file = forms.FileField(label="Zip Archive to Import") - - parent_page = forms.ModelChoiceField( - queryset=Page.objects.all(), - widget=AdminPageChooser(**admin_page_params.copy()), - label="Destination Parent Page", - help_text="Imported pages will be created as children of this page." - ) - -class ExportPage(forms.Form): - """ - This form renders the export fields. - """ - - root_page = forms.ModelChoiceField( - queryset=Page.objects.all(), - widget=AdminPageChooser(**admin_page_params.copy()), - label="Root Page to Export", - help_text="All children pages (including the selected root page) will be exported." - ) - - export_unpublished = forms.BooleanField( - initial=True, - required=False, - label="Export Unpublished Pages", - help_text="If True, unpublished pages will be exported along with published pages.", - ) - - null_pk = forms.BooleanField( - widget=forms.HiddenInput(), - required = False, - initial=False, - label="Remove Primary Keys", - help_text="This is set to False as default and can be changed in code. Changing to True may break import functionality.", - ) - - null_fk = forms.BooleanField( - initial=True, - required=False, - label="Remove Foreign Keys", - help_text="If True, foreign keys will be nulled. Leave checked if exported archive will be imported to a different environment.", - ) - - null_users = forms.BooleanField( - initial=True, - required=False, - label="Remove User References", - help_text="If True, user fields (owner in pages, *user_id in images) will be nulled. Leave checked if exported archive will be imported to a different environment.", - ) - - export_images = forms.BooleanField( - initial=True, - required=False, - label="Export Images", - help_text="If True, image references will be nulled and images that are used on the page will be exported along with the rest of the content. Leave checked if exported archive will be imported to a different environment.", - ) - - export_documents = forms.BooleanField( - initial=True, - required=False, - label="Export Documents", - help_text="If True, document references will be nulled and documents that are used on the page will be exported along with the rest of the content. Leave checked if exported archive will be imported to a different environment.", - ) - - export_snippets = forms.BooleanField( - initial=True, - required=False, - label="Export Snippets", - help_text="If True, snippet references will be nulled and snippets that are used on the page will be exported along with the rest of the content. Leave checked if exported archive will be imported to a different environment.", - ) - - ignore_large_files = forms.BooleanField( - initial=True, - required=False, - label="Exclude Large Files", - help_text="If True, large files will be nullified and ignored during export.", - ) diff --git a/wagtailimportexport/functions.py b/wagtailimportexport/functions.py deleted file mode 100644 index 067ff1889..000000000 --- a/wagtailimportexport/functions.py +++ /dev/null @@ -1,246 +0,0 @@ -import tempfile -import zipfile -import os -import io -import json -import logging - -from django.core.files.storage import storages as get_storage_class -from django.core.serializers.json import DjangoJSONEncoder -from django.db.models.fields.related import ForeignKey -from django.db.models.fields.reverse_related import ManyToOneRel -from django.contrib.contenttypes.models import ContentType - -from wagtail.fields import StreamField -from wagtail.documents.models import Document - - -def null_pks(page, data): - """ - Nullifies primary keys within all supplied fields. - - Arguments: - page -- Page object. - data -- Page object in dictionary format. - - Returns: - N/A. Overwrites the argument. - """ - - # Nullify the main ID - data['id'] = None - data['pk'] = None - - # Loop through all fields. - for field_name, field_val in data.items(): - if type(field_val) != list: - continue - - for i, sub_item in enumerate(field_val): - if 'pk' in sub_item: - data[field_name][i]['pk'] = None - - -def find_null_child_blocks(subfield, location, data): - """ - Recursive function to find all children blocks - within streamfield and nullify fks. - - Arguments: - subfield -- A field. - location -- (Ordered) list of field keys that act - as a tree. - data -- Data object to overwrite the changes to. - - Returns: - N/A. Overwrites data. - """ - - # Some fields do not have child_blocks, and we should not - # investigate further if that's the case. - if "child_blocks" in subfield.__dict__.keys(): - - # Go through all fields. - for field_key, field_val in subfield.child_blocks.items(): - - # We want to catch the ForeignKey - if isinstance(field_val, ForeignKey): - # TODO: Implement overwriting. - pass - - # Recursive Calls - find_null_child_blocks(field_val, location + [field_key], data) - - -def find_null_child_relations(subfield, location, data): - """ - Recursive function to find all children relations - within manyotoone relationships and nullify fks. - - Arguments: - subfield -- A field. - location -- (Ordered) list of field keys that act - as a tree. - data -- Data object to overwrite the changes to. - - Returns: - N/A. Overwrites data. - """ - - # Some fields do not have related_model, and we should not - # investigate further if that's the case. - if "related_model" in subfield.__dict__.keys(): - - # Go through all fields. - for field in subfield.related_model._meta.fields: - - # We want to catch the ForeignKey - if isinstance(field, ForeignKey): - if not location[0] in data: - continue - - for i, value in enumerate(data[location[0]]): - if not field.name in data[location[0]][i]: - continue - - data[location[0]][i][field.name] = None - - -def null_fks(page, data): - """ - Nullifies foreign keys within all supplied fields. - - Arguments: - page -- Page object. - data -- Page object in dictionary format. - - Returns: - N/A. Overwrites the argument. - """ - - # Loop through all fields. - for field in page._meta.get_fields(): - - # Check whether the field is a ForeignKey. - # By nature, owner, content_type, live_revision - # are foreign keys defined by wagtail core pages. - if (isinstance(field, ForeignKey)): - data[field.name] = None - - # StreamFields often have foreign keys associated with them. - # if(isinstance(field, StreamField)): - # find_null_child_blocks(field.stream_block, [field.name], data) - # - # # Many to One relations often have foreign keys associated with them. - # if(isinstance(field, ManyToOneRel)): - # find_null_child_relations(field, [field.name], data) - - -def zip_contents(page_contents): - """ - Creates and returns a zip archive of all supplied items. - - Arguments: - page_contents -- A list of page dictionaries. - - Returns: - Zip file to be downloaded by the client. - """ - - file_storage = get_storage_class()() - - # Create a temporary directory. - with tempfile.TemporaryDirectory() as tempdir: - - # Create a temporary zip. - zfname = os.path.join(tempdir, 'content.zip') - - # Open the zip archive with write mode. - with zipfile.ZipFile(zfname, 'w') as zf: - - # Write the main content.json file to store all data. - zf.writestr( - 'content.json', - json.dumps(page_contents, indent=2, cls=DjangoJSONEncoder) - ) - - # Loop through pages to explore all used images and documents. - for page in page_contents: - - # Export all the images. - for image_def in page['images'].values(): - if not image_def: - continue - - filename = image_def['file']['name'] - - try: - with file_storage.open(filename, 'rb') as f: - zf.writestr(filename, f.read()) - except FileNotFoundError: - logging.error("File " + str(filename) + " is not found on local file storage and was not exported.") - - # Export all the documents. - for doc_def in page['documents'].values(): - if not doc_def: - continue - - filename = doc_def['file']['name'] - - try: - with file_storage.open(filename, 'rb') as f: - zf.writestr(filename, f.read()) - except FileNotFoundError: - logging.error("File " + str(filename) + " is not found on local file storage and was not exported.") - - with open(zfname, 'rb') as zf: - fd = zf.read() - - return io.BytesIO(fd) - - -def unzip_contents(zip_contents): - """ - Extracts all items in the zip archive and returns a mapping - of the contents, as well as their location in tempdir. - - Arguments: - zip_contents -- Zip file that is in memory. - - Returns: - Map of the extracted files. - """ - - # Create a temporary directory. - tempdir = tempfile.mkdtemp() - - # Extract all contents. - zip_contents.extractall(tempdir) - - # Return the mapping of all extracted members. - return {member: tempdir + '/' + member for member in zip_contents.namelist()} - - -def document_title(doc_pk): - doc = Document.objects.all().filter(pk=doc_pk) - if not doc: - return None - else: - return str(doc[0]) - - -def document_id(doc_title): - doc = Document.objects.all().filter(title=doc_title) - if not doc: - return None - else: - return doc[0].pk - - -def content_type_by_model(model): - content_type = ContentType.objects.all().filter(model=model) - if not content_type: - return None - else: - return str(content_type[0].pk) - diff --git a/wagtailimportexport/importing.py b/wagtailimportexport/importing.py deleted file mode 100644 index 8c61af851..000000000 --- a/wagtailimportexport/importing.py +++ /dev/null @@ -1,312 +0,0 @@ -import io -import json -import logging -import traceback -from zipfile import ZipFile - -from django.apps import apps -from django.core.files.images import ImageFile -from django.core.files.base import File -from django.contrib.contenttypes.models import ContentType -from django.db import models, transaction, IntegrityError - -from modelcluster.models import get_all_child_relations - -from wagtail.models import Page -from wagtail.images.models import Image - -from wagtailimportexport import functions -import snippets.models as snippets - - -def import_page(uploaded_archive, parent_page, overwrites={}): - """ - Imports uploaded_archive as children of parent_page. - - Arguments: - uploaded_archive -- A file object, which includes contents.json - and the media objects. - parent_page -- Page object, where the page(s) will be imported to. - - Returns: - numpages -- Integer value of number of pages that were successfully - imported. - numfails -- Integer value of number of pages that were failed to be - imported. - message -- String message to report any warning/issue. - """ - - # Read the zip archive and load as 'payload'. - payload = io.BytesIO(uploaded_archive.read()) - - # Open zip archive. - with ZipFile(payload, 'r') as zf: - try: - # Open content.json and load them into contents dictionary. - with zf.open('content.json') as mf: - contents = json.loads(mf.read().decode('utf-8-sig')) - error_msg = '' - - # First create the base Page records; these contain no foreign keys, so this allows us to - # build a complete mapping from old IDs to new IDs before we go on to importing the - # specific page models, which may require us to rewrite page IDs within foreign keys / rich - # text / streamfields. - page_content_type = ContentType.objects.get_for_model(Page) - - # Unzip all the files in the zip directory. - contents_mapping = functions.unzip_contents(zf) - - # Get the list of pages to skip. - existing_pages = list_existing_pages(contents) if not overwrites else [] - - # Dictionaries to store original paths. - pages_by_original_path = {} - pages_by_original_id = {} - - # Loop through all the pages. - for (i, page_record) in enumerate(contents): - - new_field_datas = {} - content_type = functions.content_type_by_model(page_record['model']) - #content_type = page_record['content']['content_type'] - - # Skip the existing pages. - if i in existing_pages: - error_msg = 'Import stopped. Duplicate slug: ' + str(page_record['content']['slug']) - continue - - # Reassign image IDs. - for (fieldname, filedata) in page_record["images"].items(): - - new_field_datas[fieldname] = None - - # Skip if the image is set to null. - if not filedata: - continue - - local_file_query = get_fileobject(filedata["file"]["name"].split("/")[-1], Image) - - local_file_id = local_file_query if local_file_query else create_fileobject( - filedata["title"], contents_mapping[filedata["file"]["name"]], Image) - - new_field_datas[fieldname] = local_file_id - - # Overwrite image and document IDs - for (field, new_value) in new_field_datas.items(): - page_record['content'][field] = new_value - - # Misc. overwrites - for (field, new_value) in overwrites.items(): - page_record['content'][field] = new_value - - if page_record['model'] == 'book': - # look up document ids - page_record['content']['cover'] = functions.document_id(page_record['content']['cover']) - page_record['content']['title_image'] = functions.document_id(page_record['content']['title_image']) - page_record['content']['high_resolution_pdf'] = functions.document_id(page_record['content']['high_resolution_pdf']) - page_record['content']['low_resolution_pdf'] = functions.document_id(page_record['content']['low_resolution_pdf']) - page_record['content']['community_resource_logo'] = functions.document_id(page_record['content']['community_resource_logo']) - page_record['content']['community_resource_feature_link'] = functions.document_id(page_record['content']['community_resource_feature_link']) - - # set page.pk to null if pk already exists - pages = Page.objects.all() - for p in pages: - if p.pk == page_record['content']['pk']: - page_record['content']['pk'] = None - break - - page_record['content']['content_type'] = content_type - # Create page instance. - page = Page.from_serializable_data(page_record['content']) - - original_path = page.path - original_id = page.id - - # Clear id and treebeard-related fields so that they get reassigned when we save via add_child - page.id = None - page.path = None - page.depth = None - page.numchild = 0 - page.url_path = None - page.content_type = page_content_type - - # Handle children of the imported page(s). - if i == 0: - parent_page.add_child(instance=page) - else: - # Child pages are created in the same sibling path order as the - # source tree because the export is ordered by path - parent_path = original_path[:-(Page.steplen)] - pages_by_original_path[parent_path].add_child(instance=page) - - pages_by_original_path[original_path] = page - pages_by_original_id[original_id] = page - - # Get the page model of the source page by app_label and model name - # The content type ID of the source page is not in general the same - # between the source and destination sites but the page model needs - # to exist on both. - try: - model = apps.get_model(page_record['app_label'], page_record['model']) - except LookupError: - logging.error("Importing file failed because the model " + page_record[ - 'model'] + " does not exist on this environment.") - return (0, 1, "Importing file failed because the model " + page_record[ - 'model'] + " does not exist on this environment.") - - specific_page = model.from_serializable_data(page_record['content'], check_fks=False, - strict_fks=False) - - base_page = pages_by_original_id[specific_page.id] - specific_page.page_ptr = base_page - specific_page.__dict__.update(base_page.__dict__) - specific_page.content_type = ContentType.objects.get_for_model(model) - update_page_references(specific_page, pages_by_original_id) - specific_page.save() - - return (len(contents) - len(existing_pages), len(existing_pages), error_msg) - - except LookupError as e: - # If content.json does not exist, then return the error, - # and terminate the import_page. - logging.error("Importing file failed because file does not exist: " + str(e)) - traceback.print_exception(type(e), e, e.__traceback__) - return (0, 1, "File does not exist: " + str(e)) - - return (0, 1, "") - - -def list_existing_pages(pages): - """ - Returns a list of pages that already exist in this - environment by looking up by slug. - - Arguments: - pages -- A list of pages in content.json - - Returns: - existing_pages -- List of pages that correspond to indexes - in 'pages'. - """ - - existing_pages = [] - - for (i, page_record) in enumerate(pages): - try: - # Trying to get the page. - localpage = Page.objects.get(slug=page_record['content']['slug']) - - if localpage: - existing_pages.append(i) - - except Page.DoesNotExist: - continue - - return existing_pages - - -def get_fileobject(title, objtype): - """ - Returns the id of the object if it exists, otherwise returns - False. - - Arguments: - title -- The filename to be queried. - objtype -- Image, Document from Wagtail. - - Returns: - False if the object does not exist in this environment, - object's integer ID if it does exist. - """ - - try: - # Check whether the object already exists. - localobj = objtype.objects.get(file=title) - - if localobj: - return localobj.id - - except objtype.DoesNotExist: - return False - - return False - - -def create_fileobject(title, uploaded_file, objtype): - """ - Creates a new object given the information and returns - the ID of the created object. Assumes the object with - title does not exist. - - Arguments: - title -- The filename of the object to be created. - uploaded_file -- The file object to create. - objtype -- Image, Document from Wagtail. - - Returns: - Integer ID of the created object if the creation is successful; - otherwise None. - """ - - try: - with open(uploaded_file, 'rb') as mf: - - # Create the file object based on objtype. - if objtype == File: - filedata = File(mf, name=mf.name.split("/")[-1]) - elif objtype == Image: - filedata = ImageFile(mf, name=mf.name.split("/")[-1]) - else: - return None - - try: - with transaction.atomic(): - # Create the object and return the ID. - localobj = objtype.objects.create(file=filedata, title=title) - return localobj.id - - except IntegrityError: - logging.error("Integrity error while uploading a file:", title) - return None - except FileNotFoundError: - logging.error("File " + uploaded_file + " is not found on imported archive, skipping.") - - return None - - -def update_page_references(model, pages_by_original_id): - """ - Updates the page references recursively. - - Arguments: - model -- - pages_by_original_id -- - - Returns: - N/A. Overwrites model attributes. - - """ - - for field in model._meta.get_fields(): - if isinstance(field, models.ForeignKey) and issubclass(field.related_model, Page): - linked_page_id = getattr(model, field.attname) - try: - # see if the linked page is one of the ones we're importing - linked_page = pages_by_original_id[linked_page_id] - except KeyError: - # any references to pages outside of the import should be left unchanged - continue - - # update fk to the linked page's new ID - setattr(model, field.attname, linked_page.id) - - # update references within inline child models, including the ParentalKey pointing back - # to the page - for rel in get_all_child_relations(model): - for child in getattr(model, rel.get_accessor_name()).all(): - # reset the child model's PK so that it will be inserted as a new record - # rather than updating an existing one - child.pk = None - # update page references on the child model, including the ParentalKey - update_page_references(child, pages_by_original_id) - \ No newline at end of file diff --git a/wagtailimportexport/migrations/__init__.py b/wagtailimportexport/migrations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/wagtailimportexport/templates/wagtailimportexport/export-page.html b/wagtailimportexport/templates/wagtailimportexport/export-page.html deleted file mode 100644 index e8f471b60..000000000 --- a/wagtailimportexport/templates/wagtailimportexport/export-page.html +++ /dev/null @@ -1,31 +0,0 @@ -{% extends "wagtailadmin/base.html" %} -{% load i18n %} -{% block titletag %}{% blocktrans %}Export Pages{% endblocktrans %}{% endblock %} -{% block content %} - {% trans "Export Pages" as title_str %} - {% include "wagtailadmin/shared/header.html" with title=title_str icon="download" %} - -
-
- {% csrf_token %} -
    - {% for field in form %} -
  • {% include "wagtailadmin/shared/field.html" %}
    • - {% endfor %} -
- - -
-
-{% endblock %} - -{% block extra_js %} - {{ block.super }} - {% include "wagtailadmin/pages/_editor_js.html" %} - {{ form.media.js }} -{% endblock %} - -{% block extra_css %} - {{ block.super }} - {{ form.media.css }} -{% endblock %} diff --git a/wagtailimportexport/templates/wagtailimportexport/import-page.html b/wagtailimportexport/templates/wagtailimportexport/import-page.html deleted file mode 100644 index 4f53e3198..000000000 --- a/wagtailimportexport/templates/wagtailimportexport/import-page.html +++ /dev/null @@ -1,31 +0,0 @@ -{% extends "wagtailadmin/base.html" %} -{% load i18n %} -{% block titletag %}{% blocktrans %}Import Pages{% endblocktrans %}{% endblock %} -{% block content %} - {% trans "Import Pages" as title_str %} - {% include "wagtailadmin/shared/header.html" with title=title_str icon="download" %} - -
-
- {% csrf_token %} -
    - {% for field in form %} -
  • {% include "wagtailadmin/shared/field.html" %}
    • - {% endfor %} -
- - -
-
-{% endblock %} - -{% block extra_js %} - {{ block.super }} - {% include "wagtailadmin/pages/_editor_js.html" %} - {{ form.media.js }} -{% endblock %} - -{% block extra_css %} - {{ block.super }} - {{ form.media.css }} -{% endblock %} \ No newline at end of file diff --git a/wagtailimportexport/templates/wagtailimportexport/index.html b/wagtailimportexport/templates/wagtailimportexport/index.html deleted file mode 100644 index 34cac3d5c..000000000 --- a/wagtailimportexport/templates/wagtailimportexport/index.html +++ /dev/null @@ -1,30 +0,0 @@ -{% extends "wagtailadmin/base.html" %} -{% load i18n %} -{% block titletag %}{% blocktrans %}Import/Export Pages{% endblocktrans %}{% endblock %} -{% block content %} - {% trans "Import/Export Pages" as title_str %} - {% include "wagtailadmin/shared/header.html" with title=title_str icon="download" %} - -
- -
-{% endblock %} diff --git a/wagtailimportexport/tests/__init__.py b/wagtailimportexport/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/wagtailimportexport/tests/test_functions.py b/wagtailimportexport/tests/test_functions.py deleted file mode 100644 index 4997a3903..000000000 --- a/wagtailimportexport/tests/test_functions.py +++ /dev/null @@ -1,32 +0,0 @@ -from django.test import TestCase - -from wagtailimportexport import functions - - -class TestNullPKs(TestCase): - """ - Test cases for null_pks method in functions.py - """ - def test_null(self): - pass - -class TestNullFKs(TestCase): - """ - Test cases for null_fks method in functions.py - """ - def test_null(self): - pass - -class TestZipContents(TestCase): - """ - Test cases for zip_contents method in functions.py - """ - def test_null(self): - pass - -class TestUnZipContents(TestCase): - """ - Test cases for unzip_contents method in functions.py - """ - def test_null(self): - pass \ No newline at end of file diff --git a/wagtailimportexport/tests/tests.py b/wagtailimportexport/tests/tests.py deleted file mode 100644 index ec9f65409..000000000 --- a/wagtailimportexport/tests/tests.py +++ /dev/null @@ -1,36 +0,0 @@ -from unittest import mock - -from django.forms import FileField, ModelChoiceField -from django.test import TestCase, Client -from django.core.files.uploadedfile import SimpleUploadedFile -from django.core.files import File - -from pages.models import HomePage -from wagtailimportexport.forms import ImportPage, ExportPage - - -class TemplateTests(TestCase): - def setUp(self): - self.client = Client() - - def test_import_template(self): - response = self.client.get('/admin/import-export/import-page/', follow=True) - self.assertEqual(response.status_code, 200) - - def test_export_template(self): - response = self.client.get('/admin/import-export/export-page/', follow=True) - self.assertEqual(response.status_code, 200) - - def test_import_form(self): - zip = SimpleUploadedFile("test.zip", b"file content", content_type="application/zip") - form_data = {"parent_page": 1} - form = ImportPage(form_data, files={'file': zip}) - self.assertTrue(form.is_valid(), form.errors) - - def test_export_form(self): - form_data = {"root_page": 1} - form = ExportPage(form_data) - self.assertTrue(form.is_valid(), form.errors) - - - diff --git a/wagtailimportexport/views.py b/wagtailimportexport/views.py deleted file mode 100644 index 4759ae640..000000000 --- a/wagtailimportexport/views.py +++ /dev/null @@ -1,102 +0,0 @@ -from django.http import JsonResponse -from django.shortcuts import redirect, render -from django.urls import reverse -from django.utils.translation import ngettext -from django.http import HttpResponse - -from wagtail.admin import messages - -from wagtailimportexport import forms, importing, exporting - - -def index(request): - """ - View for main menu of the Import/Export tool. Provides a list - of features. - """ - return render(request, 'wagtailimportexport/index.html') - -def import_page(request): - """ - View for the import page. - """ - if request.method == 'POST': - form = forms.ImportPage(request.POST, request.FILES) - - if form.is_valid(): - - # Read fields on the submitted form. - form_file = form.cleaned_data['file'] - form_parentpage = form.cleaned_data['parent_page'] - - # Import pages and get the response. - num_uploaded, num_failed, response = importing.import_page(form_file, form_parentpage) - - # Show messages depending on the response. - if not num_failed: - # All pages are imported. - messages.success( - request, ngettext("Imported %(count)s page.", "Imported %(count)s pages.", num_uploaded) - % {'count': num_uploaded} - ) - elif not num_uploaded: - # None of the pages are imported. - messages.error( - request, ngettext("Failed to import %(count)s page. %(reason)s", "Failed to import %(count)s pages. %(reason)s", num_failed) - % {'count': num_failed, 'reason': response} - ) - else: - # Some pages are imported and some failed. - messages.warning( - request, ngettext("Failed to import %(failed)s out of %(total)s page. %(reason)s", "Failed to import %(failed)s out of %(total)s pages. %(reason)s", num_failed + num_uploaded) - % {'failed': num_failed, 'total': num_failed + num_uploaded, 'reason': response} - ) - - # Redirect client to the parent page view on admin. - return redirect('wagtailadmin_explore', form_parentpage.pk) - else: - form = forms.ImportPage() - - # Redirect client to form. - return render(request, 'wagtailimportexport/import-page.html', { - 'form': form, - }) - -def export_page(request): - """ - View for the export page. - """ - - if request.method == 'POST': - form = forms.ExportPage(request.POST) - - if form.is_valid(): - export_file = exporting.export_page(settings=form.cleaned_data) - - if export_file: - # Grab ZIP file from in-memory, make response with correct MIME-type - response = HttpResponse(export_file.getvalue(), content_type = "application/x-zip-compressed") - - # ..and correct content-disposition - response['Content-Disposition'] = 'attachment; filename=wagtail-export.zip' - - return response - else: - form = forms.ExportPage() - - messages.error( - request, "Failed to generate an export file. Please refer to the logs for further details." - ) - - # Redirect client to form. - return render(request, 'wagtailimportexport/export-page.html', { - 'form': form, - }) - - else: - form = forms.ExportPage() - - # Redirect client to form. - return render(request, 'wagtailimportexport/export-page.html', { - 'form': form, - }) diff --git a/wagtailimportexport/wagtail_hooks.py b/wagtailimportexport/wagtail_hooks.py deleted file mode 100644 index 7001d7487..000000000 --- a/wagtailimportexport/wagtail_hooks.py +++ /dev/null @@ -1,37 +0,0 @@ -from django.urls import include, path -from django.urls import reverse - -from wagtail import hooks -from wagtail.admin.menu import MenuItem - -from wagtailimportexport import admin_urls - - -@hooks.register('register_admin_urls') -def register_admin_urls(): - """ - Register 'import-export/' url path to admin urls. - """ - return [ - path(r'import-export/', include(admin_urls, namespace='wagtailimportexport')), - ] - - -class ImportExportMenuItem(MenuItem): - """ - Add the menu item to admin side menu. This will be only shown if the user is - superuser. This will be only shown if the user is - superuser. - """ - def is_shown(self, request): - return request.user.is_superuser - - -@hooks.register('register_admin_menu_item') -def register_import_export_menu_item(): - """ - Add the menu item to admin side menu. - """ - return ImportExportMenuItem( - 'Import / Export', reverse('wagtailimportexport:index'), classname='icon icon-download', order=800 - )