Skip to content

Commit

Permalink
Possible fix for issue #464 - Enable Zip64.
Browse files Browse the repository at this point in the history
  • Loading branch information
@FormerLurker
FormerLurker committed Mar 1, 2020
1 parent 8ef2991 commit 815cd3f
Show file tree
Hide file tree
Showing 4 changed files with 154 additions and 118 deletions.
10 changes: 10 additions & 0 deletions octoprint_octolapse/error_messages.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,16 @@
"name": "No Files Were Found",
"description": "No files were found within the archive.",
"help_link": "error_help_rendering_archive_import_no_files_found.md"
},
'zip_file_too_large': {
"name": "The Archive is Too Large",
"description": "The zip archive is too large to import.",
"help_link": "error_help_rendering_archive_import_zip_file_too_large.md"
},
'zip_file_corrupt': {
"name": "The Archive is Corrupt",
"description": "The zip archive is too large to import.",

This comment has been minimized.

Sign in to view

Copy link
@EleRas

EleRas Mar 2, 2020

This seems to be the wrong error message

This comment has been minimized.

Sign in to view

Copy link
@FormerLurker

FormerLurker Mar 2, 2020

Author Owner

You are correct! Thanks for letting me know. Will push a fix after work.
"help_link": "error_help_rendering_archive_import_zip_file_corrupt.md"
}
}
}
Expand Down
249 changes: 131 additions & 118 deletions octoprint_octolapse/render.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import time
import json
import copy
from zipfile import ZipFile
import zipfile as zipfile
from csv import DictReader
# sarge was added to the additional requirements for the plugin
import datetime
Expand Down Expand Up @@ -520,46 +520,48 @@ def archive_unfinished_job(
os.makedirs(target_directory)
except OSError:
pass
try:
with zipfile.ZipFile(target_path, mode='w', allowZip64=True) as snapshot_archive:
# add the job info
timelapse_info_path = os.path.join(job_directory,
utility.TimelapseJobInfo.timelapse_info_file_name)

with ZipFile(target_path, 'w') as snapshot_archive:
# add the job info
timelapse_info_path = os.path.join(job_directory,
utility.TimelapseJobInfo.timelapse_info_file_name)

snapshot_files = os.listdir(camera_directory)
if not progress_total_steps:
progress_total_steps = len(snapshot_files)

if os.path.exists(timelapse_info_path):
snapshot_archive.write(
os.path.join(job_directory,
utility.TimelapseJobInfo.timelapse_info_file_name),
os.path.join(job_guid, utility.TimelapseJobInfo.timelapse_info_file_name)
)
snapshot_files = os.listdir(camera_directory)
if not progress_total_steps:
progress_total_steps = len(snapshot_files)

progress_current_step += 1
if progress_callback:
progress_callback(progress_key, progress_current_step, progress_total_steps)

for name in snapshot_files:
file_path = os.path.join(camera_directory, name)
# ensure that all files we add to the archive were created by Octolapse, and are useful
# for rendering.
if os.path.isfile(file_path) and (
utility.is_valid_snapshot_extension(utility.get_extension_from_filename(name)) or
CameraInfo.is_camera_info_file(name) or
SnapshotMetadata.is_metadata_file(name) or
OctolapseSettings.is_camera_settings_file(name) or
OctolapseSettings.is_rendering_settings_file(name)
):
if os.path.exists(timelapse_info_path):
snapshot_archive.write(
file_path,
os.path.join(job_guid, camera_guid, name)
os.path.join(job_directory,
utility.TimelapseJobInfo.timelapse_info_file_name),
os.path.join(job_guid, utility.TimelapseJobInfo.timelapse_info_file_name)
)

progress_current_step += 1
if progress_callback:
progress_callback(progress_key, progress_current_step, progress_total_steps)

for name in snapshot_files:
file_path = os.path.join(camera_directory, name)
# ensure that all files we add to the archive were created by Octolapse, and are useful
# for rendering.
if os.path.isfile(file_path) and (
utility.is_valid_snapshot_extension(utility.get_extension_from_filename(name)) or
CameraInfo.is_camera_info_file(name) or
SnapshotMetadata.is_metadata_file(name) or
OctolapseSettings.is_camera_settings_file(name) or
OctolapseSettings.is_rendering_settings_file(name)
):
snapshot_archive.write(
file_path,
os.path.join(job_guid, camera_guid, name)
)
progress_current_step += 1
if progress_callback:
progress_callback(progress_key, progress_current_step, progress_total_steps)
except zipfile.LargeZipFile as e:
logger.exception("The zip file is too large to open.")
raise e
if is_download:
metadata = self._get_metadata_for_rendering_files(job_guid, camera_guid, temporary_directory)
target_extension = utility.get_extension_from_full_path(target_path)
Expand All @@ -584,95 +586,106 @@ def import_snapshot_archive(self, snapshot_archive_path, prevent_archive=False):
}
root_job_guid = "{0}".format(uuid.uuid4())
root_camera_guid = "{0}".format(uuid.uuid4())

with ZipFile(snapshot_archive_path) as zip_file:
archive_files_temp_dict = {} # a temporary dict to hold values while we construct the jobs
fileinfos = [x for x in zip_file.infolist() if not x.filename.startswith("__MACOSX") ]
for fileinfo in fileinfos:
# see if the current item is a directory
if not fileinfo.filename.endswith(os.sep):
parts = utility.split_all(fileinfo.filename)
name = os.path.basename(fileinfo.filename)
name_without_extension = utility.get_filename_from_full_path(name)
extension = utility.get_extension_from_filename(name).lower()
item = {
"name": name,
"fileinfo": fileinfo
}
location_type = None
job_guid = None
camera_guid = None
file_type = None
if len(parts) == 1:
job_guid = root_job_guid
camera_guid = root_camera_guid
elif len(parts) == 2 and _is_valid_uuid(parts[0]):
job_guid = parts[0].lower()
elif len(parts) == 3 and _is_valid_uuid(parts[0]) and _is_valid_uuid(parts[1]):
job_guid = parts[0].lower()
camera_guid = parts[1].lower()
else:
continue

if job_guid not in archive_files_temp_dict:
archive_files_temp_dict[job_guid] = {
'cameras': {},
'file': None
try:
with zipfile.ZipFile(snapshot_archive_path, mode="r", allowZip64=True) as zip_file:
archive_files_temp_dict = {} # a temporary dict to hold values while we construct the jobs
fileinfos = [x for x in zip_file.infolist() if not x.filename.startswith("__MACOSX") ]
for fileinfo in fileinfos:
# see if the current item is a directory
if not fileinfo.filename.endswith(os.sep):
parts = utility.split_all(fileinfo.filename)
name = os.path.basename(fileinfo.filename)
name_without_extension = utility.get_filename_from_full_path(name)
extension = utility.get_extension_from_filename(name).lower()
item = {
"name": name,
"fileinfo": fileinfo
}
if camera_guid and camera_guid not in archive_files_temp_dict[job_guid]["cameras"]:
archive_files_temp_dict[job_guid]['cameras'][camera_guid] = []

# this file is in the root. See what kind of file this is
if utility.TimelapseJobInfo.is_timelapse_info_file(name):
item["name"] = utility.TimelapseJobInfo.timelapse_info_file_name
# preserve case of the name, but keep the extension lower case
archive_files_temp_dict[job_guid]["file"] = item
else:
if utility.is_valid_snapshot_extension(extension):
# preserve case of the name, but keep the extension lower case
file_name = "{0}.{1}".format(name_without_extension, extension)
elif CameraInfo.is_camera_info_file(name):
file_name = CameraInfo.camera_info_filename
elif SnapshotMetadata.is_metadata_file(name):
file_name = SnapshotMetadata.METADATA_FILE_NAME
elif OctolapseSettings.is_camera_settings_file(name):
file_name = OctolapseSettings.camera_settings_file_name
elif OctolapseSettings.is_rendering_settings_file(name):
file_name = OctolapseSettings.rendering_settings_file_name
location_type = None
job_guid = None
camera_guid = None
file_type = None
if len(parts) == 1:
job_guid = root_job_guid
camera_guid = root_camera_guid
elif len(parts) == 2 and _is_valid_uuid(parts[0]):
job_guid = parts[0].lower()
elif len(parts) == 3 and _is_valid_uuid(parts[0]) and _is_valid_uuid(parts[1]):
job_guid = parts[0].lower()
camera_guid = parts[1].lower()
else:
continue
item["name"] = file_name
archive_files_temp_dict[job_guid]['cameras'][camera_guid].append(item)

# now replace all of the job guids with new ones to prevent conflicts with existing unfinished
# rendering jobs.
for key in archive_files_temp_dict.keys():
archive_files_dict["{0}".format(uuid.uuid4())] = archive_files_temp_dict[key]
archive_files_temp_dict = {}

# now create the directories and files and place them in the temp snapshot directory
for job_guid, job in iteritems(archive_files_dict):
job_path = utility.get_temporary_snapshot_job_path(temporary_directory, job_guid)
if not os.path.isdir(job_path):
os.makedirs(job_path)
job_info_file = job["file"]
if job_info_file:
file_path = os.path.join(job_path, job_info_file["name"])
with zip_file.open(job_info_file["fileinfo"]) as info_file:
with open(file_path, 'wb') as target_file:
target_file.write(info_file.read())
for camera_guid, camera in iteritems(job["cameras"]):
camera_path = utility.get_temporary_snapshot_job_camera_path(
temporary_directory, job_guid, camera_guid
)
if not os.path.isdir(camera_path):
os.makedirs(camera_path)
for camera_fileinfo in camera:
file_path = os.path.join(camera_path, camera_fileinfo["name"])
with zip_file.open(camera_fileinfo["fileinfo"]) as camera_file:
with open(file_path, 'wb') as target_file:
target_file.write(camera_file.read())

if job_guid not in archive_files_temp_dict:
archive_files_temp_dict[job_guid] = {
'cameras': {},
'file': None
}
if camera_guid and camera_guid not in archive_files_temp_dict[job_guid]["cameras"]:
archive_files_temp_dict[job_guid]['cameras'][camera_guid] = []

# this file is in the root. See what kind of file this is
if utility.TimelapseJobInfo.is_timelapse_info_file(name):
item["name"] = utility.TimelapseJobInfo.timelapse_info_file_name
# preserve case of the name, but keep the extension lower case
archive_files_temp_dict[job_guid]["file"] = item
else:
if utility.is_valid_snapshot_extension(extension):
# preserve case of the name, but keep the extension lower case
file_name = "{0}.{1}".format(name_without_extension, extension)
elif CameraInfo.is_camera_info_file(name):
file_name = CameraInfo.camera_info_filename
elif SnapshotMetadata.is_metadata_file(name):
file_name = SnapshotMetadata.METADATA_FILE_NAME
elif OctolapseSettings.is_camera_settings_file(name):
file_name = OctolapseSettings.camera_settings_file_name
elif OctolapseSettings.is_rendering_settings_file(name):
file_name = OctolapseSettings.rendering_settings_file_name
else:
continue
item["name"] = file_name
archive_files_temp_dict[job_guid]['cameras'][camera_guid].append(item)

# now replace all of the job guids with new ones to prevent conflicts with existing unfinished
# rendering jobs.
for key in archive_files_temp_dict.keys():
archive_files_dict["{0}".format(uuid.uuid4())] = archive_files_temp_dict[key]
archive_files_temp_dict = {}

# now create the directories and files and place them in the temp snapshot directory
for job_guid, job in iteritems(archive_files_dict):
job_path = utility.get_temporary_snapshot_job_path(temporary_directory, job_guid)
if not os.path.isdir(job_path):
os.makedirs(job_path)
job_info_file = job["file"]
if job_info_file:
file_path = os.path.join(job_path, job_info_file["name"])
with zip_file.open(job_info_file["fileinfo"]) as info_file:
with open(file_path, 'wb') as target_file:
target_file.write(info_file.read())
for camera_guid, camera in iteritems(job["cameras"]):
camera_path = utility.get_temporary_snapshot_job_camera_path(
temporary_directory, job_guid, camera_guid
)
if not os.path.isdir(camera_path):
os.makedirs(camera_path)
for camera_fileinfo in camera:
file_path = os.path.join(camera_path, camera_fileinfo["name"])
with zip_file.open(camera_fileinfo["fileinfo"]) as camera_file:
with open(file_path, 'wb') as target_file:
target_file.write(camera_file.read())
except zipfile.LargeZipFile:
logger.exception("The zip file at '%s' is too large to open.", snapshot_archive_path)
return {
'success': False,
'error_keys': ['rendering', 'archive', 'import', 'zip_file_too_large']
}
except zipfile.BadZipfile:
logger.exception("The zip file at '%s' appears to be corrupt.", snapshot_archive_path)
return {
'success': False,
'error_keys': ['rendering', 'archive', 'import', 'zip_file_corrupt']
}
# now we should have extracted all of the items, add the job to the queue for these cameras
has_created_jobs = False
for job_guid, job in iteritems(archive_files_dict):
Expand Down
6 changes: 6 additions & 0 deletions ...olapse/static/docs/help/error_help_rendering_archive_import_zip_file_corrupt.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
The zip file you are attempting to import is either in the wrong format, or is corrupted.

### Possible Solutions
1. Make sure your archive is in zip format.
2. Extract and re-create your archive. It is OK to put all of your files in the root of the zip.
3. Re-upload the zip file. It is possible that the file was corrupted during upload.
7 changes: 7 additions & 0 deletions ...apse/static/docs/help/error_help_rendering_archive_import_zip_file_too_large.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Due to limitations of either your python version, file system, or OS, the zip file you are trying to import is too large to be opened.

### Possible Solutions

1. Upgrade to python 3. This requires Octoprint 1.4.0 or above.
2. Reduce the zip file size. You can either remove images or reduce the resolution.
3. Change your SD card's file system. Fat32 is limited to files smaller than 4 GB.

0 comments on commit 815cd3f

Please sign in to comment.