[qt_common] asynchronous, cancellable import/export

Signed-off-by: crueter <crueter@eden-emu.dev>

src/qt_common/CMakeLists.txt

@@ -25,6 +25,7 @@ add_library(qt_common STATIC
     qt_applet_util.h qt_applet_util.cpp
     qt_progress_dialog.h qt_progress_dialog.cpp
     qt_string_lookup.h
+    qt_compress.h qt_compress.cpp
 
 )
 

src/qt_common/qt_compress.cpp

@@ -0,0 +1,351 @@
+#include "qt_compress.h"
+#include "quazipfileinfo.h"
+
+#include <QDirIterator>
+
+/** This is a modified version of JlCompress **/
+namespace QtCommon::Compress {
+
+bool compressDir(QString fileCompressed,
+                 QString dir,
+                 const JlCompress::Options &options,
+                 QtCommon::QtProgressCallback callback)
+{
+    // Create zip
+    QuaZip zip(fileCompressed);
+    QDir().mkpath(QFileInfo(fileCompressed).absolutePath());
+    if (!zip.open(QuaZip::mdCreate)) {
+        QFile::remove(fileCompressed);
+        return false;
+    }
+
+    // See how big the overall fs structure is
+    // good approx. of total progress
+    // TODO(crueter): QDirListing impl
+    QDirIterator iter(dir,
+                      QDir::NoDotAndDotDot | QDir::Hidden | QDir::Files,
+                      QDirIterator::Subdirectories);
+
+    std::size_t total = 0;
+    while (iter.hasNext()) {
+        total += iter.nextFileInfo().size();
+    }
+
+    std::size_t progress = 0;
+    callback(total, progress);
+
+    // Add the files and subdirectories
+    if (!compressSubDir(&zip, dir, dir, options, total, progress, callback)) {
+        QFile::remove(fileCompressed);
+        return false;
+    }
+
+    // Close zip
+    zip.close();
+    if (zip.getZipError() != 0) {
+        QFile::remove(fileCompressed);
+        return false;
+    }
+
+    return true;
+}
+
+bool compressSubDir(QuaZip *zip,
+                    QString dir,
+                    QString origDir,
+                    const JlCompress::Options &options,
+                    std::size_t total,
+                    std::size_t &progress,
+                    QtProgressCallback callback)
+{
+    // zip: object where to add the file
+    // dir: current real directory
+    // origDir: original real directory
+    // (path(dir)-path(origDir)) = path inside the zip object
+
+    if (!zip)
+        return false;
+    if (zip->getMode() != QuaZip::mdCreate && zip->getMode() != QuaZip::mdAppend
+        && zip->getMode() != QuaZip::mdAdd)
+        return false;
+
+    QDir directory(dir);
+    if (!directory.exists())
+        return false;
+
+
+    QDir origDirectory(origDir);
+    if (dir != origDir) {
+        QuaZipFile dirZipFile(zip);
+        std::unique_ptr<QuaZipNewInfo> qzni;
+        if (options.getDateTime().isNull()) {
+            qzni = std::make_unique<QuaZipNewInfo>(origDirectory.relativeFilePath(dir)
+                                                       + QLatin1String("/"),
+                                                   dir);
+        } else {
+            qzni = std::make_unique<QuaZipNewInfo>(origDirectory.relativeFilePath(dir)
+                                                       + QLatin1String("/"),
+                                                   dir,
+                                                   options.getDateTime());
+        }
+        if (!dirZipFile.open(QIODevice::WriteOnly, *qzni, nullptr, 0, 0)) {
+            return false;
+        }
+        dirZipFile.close();
+    }
+
+    // For each subfolder
+    QFileInfoList subfiles = directory.entryInfoList(QDir::AllDirs | QDir::NoDotAndDotDot
+                                                     | QDir::Hidden | QDir::Dirs);
+    for (const auto &file : std::as_const(subfiles)) {
+        if (!compressSubDir(
+                zip, file.absoluteFilePath(), origDir, options, total, progress, callback)) {
+            return false;
+        }
+    }
+
+    // For each file in directory
+    QFileInfoList files = directory.entryInfoList(QDir::Hidden | QDir::Files);
+    for (const auto &file : std::as_const(files)) {
+        // If it's not a file or it's the compressed file being created
+        if (!file.isFile() || file.absoluteFilePath() == zip->getZipName())
+            continue;
+
+        // Create relative name for the compressed file
+        QString filename = origDirectory.relativeFilePath(file.absoluteFilePath());
+
+        // Compress the file
+        if (!compressFile(zip, file.absoluteFilePath(), filename, options, total, progress, callback)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+bool compressFile(QuaZip *zip,
+                  QString fileName,
+                  QString fileDest,
+                  const JlCompress::Options &options,
+                  std::size_t total,
+                  std::size_t &progress,
+                  QtCommon::QtProgressCallback callback)
+{
+    // zip: object where to add the file
+    // fileName: real file name
+    // fileDest: file name inside the zip object
+
+    if (!zip)
+        return false;
+    if (zip->getMode() != QuaZip::mdCreate && zip->getMode() != QuaZip::mdAppend
+        && zip->getMode() != QuaZip::mdAdd)
+        return false;
+
+    QuaZipFile outFile(zip);
+    if (options.getDateTime().isNull()) {
+        if (!outFile.open(QIODevice::WriteOnly,
+                          QuaZipNewInfo(fileDest, fileName),
+                          nullptr,
+                          0,
+                          options.getCompressionMethod(),
+                          options.getCompressionLevel()))
+            return false;
+    } else {
+        if (!outFile.open(QIODevice::WriteOnly,
+                          QuaZipNewInfo(fileDest, fileName, options.getDateTime()),
+                          nullptr,
+                          0,
+                          options.getCompressionMethod(),
+                          options.getCompressionLevel()))
+            return false;
+    }
+
+    QFileInfo input(fileName);
+    if (quazip_is_symlink(input)) {
+        // Not sure if we should use any specialized codecs here.
+        // After all, a symlink IS just a byte array. And
+        // this is mostly for Linux, where UTF-8 is ubiquitous these days.
+        QString path = quazip_symlink_target(input);
+        QString relativePath = input.dir().relativeFilePath(path);
+        outFile.write(QFile::encodeName(relativePath));
+    } else {
+        QFile inFile;
+        inFile.setFileName(fileName);
+        if (!inFile.open(QIODevice::ReadOnly)) {
+            return false;
+        }
+        if (!copyData(inFile, outFile, total, progress, callback) || outFile.getZipError() != UNZ_OK) {
+            return false;
+        }
+        inFile.close();
+    }
+
+    outFile.close();
+    return outFile.getZipError() == UNZ_OK;
+}
+
+bool copyData(QIODevice &inFile,
+              QIODevice &outFile,
+              std::size_t total,
+              std::size_t &progress,
+              QtProgressCallback callback)
+{
+    while (!inFile.atEnd()) {
+        char buf[4096];
+        qint64 readLen = inFile.read(buf, 4096);
+        if (readLen <= 0)
+            return false;
+        if (outFile.write(buf, readLen) != readLen)
+            return false;
+
+        progress += readLen;
+        if (!callback(total, progress)) {
+            return false;
+        }
+    }
+    return true;
+}
+
+QStringList extractDir(QString fileCompressed, QString dir, QtCommon::QtProgressCallback callback)
+{
+    // Open zip
+    QuaZip zip(fileCompressed);
+    return extractDir(zip, dir, callback);
+}
+
+QStringList extractDir(QuaZip &zip, const QString &dir, QtCommon::QtProgressCallback callback)
+{
+    if (!zip.open(QuaZip::mdUnzip)) {
+        return QStringList();
+    }
+    QString cleanDir = QDir::cleanPath(dir);
+    QDir directory(cleanDir);
+    QString absCleanDir = directory.absolutePath();
+    if (!absCleanDir.endsWith(QLatin1Char('/'))) // It only ends with / if it's the FS root.
+        absCleanDir += QLatin1Char('/');
+    QStringList extracted;
+    if (!zip.goToFirstFile()) {
+        return QStringList();
+    }
+
+    std::size_t total = 0;
+    for (const QuaZipFileInfo64 &info : zip.getFileInfoList64()) {
+        total += info.uncompressedSize;
+    }
+
+    std::size_t progress = 0;
+    callback(total, progress);
+
+    do {
+        QString name = zip.getCurrentFileName();
+        QString absFilePath = directory.absoluteFilePath(name);
+        QString absCleanPath = QDir::cleanPath(absFilePath);
+        if (!absCleanPath.startsWith(absCleanDir))
+            continue;
+        if (!extractFile(&zip, QLatin1String(""), absFilePath, total, progress, callback)) {
+            removeFile(extracted);
+            return QStringList();
+        }
+        extracted.append(absFilePath);
+    } while (zip.goToNextFile());
+
+    // Close zip
+    zip.close();
+    if (zip.getZipError() != 0) {
+        removeFile(extracted);
+        return QStringList();
+    }
+
+    return extracted;
+}
+
+bool extractFile(QuaZip *zip,
+                 QString fileName,
+                 QString fileDest,
+                 std::size_t total,
+                 std::size_t &progress,
+                 QtCommon::QtProgressCallback callback)
+{
+    // zip: object where to add the file
+    // filename: real file name
+    // fileincompress: file name of the compressed file
+
+    if (!zip)
+        return false;
+    if (zip->getMode() != QuaZip::mdUnzip)
+        return false;
+
+    if (!fileName.isEmpty())
+        zip->setCurrentFile(fileName);
+    QuaZipFile inFile(zip);
+    if (!inFile.open(QIODevice::ReadOnly) || inFile.getZipError() != UNZ_OK)
+        return false;
+
+    // Check existence of resulting file
+    QDir curDir;
+    if (fileDest.endsWith(QLatin1String("/"))) {
+        if (!curDir.mkpath(fileDest)) {
+            return false;
+        }
+    } else {
+        if (!curDir.mkpath(QFileInfo(fileDest).absolutePath())) {
+            return false;
+        }
+    }
+
+    QuaZipFileInfo64 info;
+    if (!zip->getCurrentFileInfo(&info))
+        return false;
+
+    QFile::Permissions srcPerm = info.getPermissions();
+    if (fileDest.endsWith(QLatin1String("/")) && QFileInfo(fileDest).isDir()) {
+        if (srcPerm != 0) {
+            QFile(fileDest).setPermissions(srcPerm);
+        }
+        return true;
+    }
+
+    if (info.isSymbolicLink()) {
+        QString target = QFile::decodeName(inFile.readAll());
+        return QFile::link(target, fileDest);
+    }
+
+    // Open resulting file
+    QFile outFile;
+    outFile.setFileName(fileDest);
+    if (!outFile.open(QIODevice::WriteOnly))
+        return false;
+
+    // Copy data
+    if (!copyData(inFile, outFile, total, progress, callback) || inFile.getZipError() != UNZ_OK) {
+        outFile.close();
+        removeFile(QStringList(fileDest));
+        return false;
+    }
+    outFile.close();
+
+    // Close file
+    inFile.close();
+    if (inFile.getZipError() != UNZ_OK) {
+        removeFile(QStringList(fileDest));
+        return false;
+    }
+
+    if (srcPerm != 0) {
+        outFile.setPermissions(srcPerm);
+    }
+    return true;
+}
+
+bool removeFile(QStringList listFile)
+{
+    bool ret = true;
+    // For each file
+    for (int i = 0; i < listFile.count(); i++) {
+        // Remove
+        ret = ret && QFile::remove(listFile.at(i));
+    }
+    return ret;
+}
+
+} // namespace QtCommon::Compress

src/qt_common/qt_compress.h

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <QDir>
+#include <QString>
+#include <JlCompress.h>
+#include "qt_common/qt_common.h"
+
+/** This is a modified version of JlCompress **/
+namespace QtCommon::Compress {
+
+/**
+ * @brief Compress an entire directory and report its progress.
+ * @param fileCompressed Destination file
+ * @param dir            The directory to compress
+ * @param options        Compression level, etc
+ * @param callback       Callback that takes in two std::size_t (total, progress) and returns false if the current operation should be cancelled.
+ */
+bool compressDir(QString fileCompressed,
+                 QString dir,
+                 const JlCompress::Options& options = JlCompress::Options(),
+                 QtCommon::QtProgressCallback callback = {});
+
+// Internal //
+bool compressSubDir(QuaZip *zip,
+                    QString dir,
+                    QString origDir,
+                    const JlCompress::Options &options,
+                    std::size_t total,
+                    std::size_t &progress,
+                    QtCommon::QtProgressCallback callback);
+
+bool compressFile(QuaZip *zip,
+                  QString fileName,
+                  QString fileDest,
+                  const JlCompress::Options &options,
+                  std::size_t total,
+                  std::size_t &progress,
+                  QtCommon::QtProgressCallback callback);
+
+bool copyData(QIODevice &inFile,
+              QIODevice &outFile,
+              std::size_t total,
+              std::size_t &progress,
+              QtCommon::QtProgressCallback callback);
+
+// Extract //
+
+/**
+ * @brief Extract a zip file and report its progress.
+ * @param fileCompressed Compressed file
+ * @param dir            The directory to push the results to
+ * @param callback       Callback that takes in two std::size_t (total, progress) and returns false if the current operation should be cancelled.
+ */
+QStringList extractDir(QString fileCompressed, QString dir, QtCommon::QtProgressCallback callback = {});
+
+// Internal //
+QStringList extractDir(QuaZip &zip, const QString &dir, QtCommon::QtProgressCallback callback);
+
+bool extractFile(QuaZip *zip,
+                 QString fileName,
+                 QString fileDest,
+                 std::size_t total,
+                 std::size_t &progress,
+                 QtCommon::QtProgressCallback callback);
+
+bool removeFile(QStringList listFile);
+
+}

src/qt_common/qt_content_util.cpp

@@ -8,6 +8,7 @@
 #include "frontend_common/content_manager.h"
 #include "frontend_common/firmware_manager.h"
 #include "qt_common/qt_common.h"
+#include "qt_common/qt_compress.h"
 #include "qt_common/qt_game_util.h"
 #include "qt_common/qt_progress_dialog.h"
 #include "qt_frontend_util.h"
@@ -385,59 +386,75 @@ void ClearDataDir(FrontendCommon::DataManager::DataDir dir)
 
 void ExportDataDir(FrontendCommon::DataManager::DataDir data_dir, std::function<void()> callback)
 {
+    using namespace QtCommon::Frontend;
     const std::string dir = FrontendCommon::DataManager::GetDataDir(data_dir);
 
     const QString zip_dump_location
-        = QtCommon::Frontend::GetSaveFileName(tr("Select Export Location"),
+        = GetSaveFileName(tr("Select Export Location"),
                                               QStringLiteral("export.zip"),
                                               tr("Zipped Archives (*.zip)"));
 
     if (zip_dump_location.isEmpty())
         return;
 
-    QMetaObject::Connection* connection = new QMetaObject::Connection;
-    *connection = QObject::connect(qApp, &QGuiApplication::aboutToQuit, rootObject, [=]() mutable {
-        QtCommon::Frontend::Warning(tr("Still Exporting"),
-                                    tr("Eden is still exporting some data, and will continue "
-                                       "running in the background until it's done."));
-    });
-
-    QtCommon::Frontend::QtProgressDialog* progress = new QtCommon::Frontend::QtProgressDialog(
-        tr("Compressing, this may take a while..."), tr("Background"), 0, 0, rootObject);
+    QtProgressDialog* progress = new QtProgressDialog(
+        tr("Exporting data. This may take a while..."), tr("Cancel"), 0, 100, rootObject);
 
+    progress->setWindowTitle(tr("Exporting"));
     progress->setWindowModality(Qt::WindowModal);
+    progress->setMinimumDuration(100);
+    progress->setAutoClose(false);
+    progress->setAutoReset(false);
     progress->show();
+
+    // Qt's wasCanceled seems to be wonky
+    bool was_cancelled = false;
+
+    QObject::connect(progress, &QtProgressDialog::canceled, rootObject, [=]() mutable {
+        was_cancelled = false;
+    });
+
     QGuiApplication::processEvents();
 
-    QFuture<bool> future = QtConcurrent::run([&]() {
-        return JlCompress::compressDir(zip_dump_location,
-                                       QString::fromStdString(dir),
-                                       true,
-                                       QDir::Hidden | QDir::Files | QDir::Dirs);
+    auto progress_callback = [=](size_t total_size, size_t processed_size) {
+        QMetaObject::invokeMethod(progress,
+                                  &QtProgressDialog::setValue,
+                                  static_cast<int>((processed_size * 100) / total_size));
+
+        return !progress->wasCanceled();
+    };
+
+    QFuture<bool> future = QtConcurrent::run([=]() {
+        return QtCommon::Compress::compressDir(zip_dump_location,
+                                               QString::fromStdString(dir),
+                                               JlCompress::Options(),
+                                               progress_callback);
     });
 
     QFutureWatcher<bool>* watcher = new QFutureWatcher<bool>(rootObject);
 
     QObject::connect(watcher, &QFutureWatcher<bool>::finished, rootObject, [=]() {
         progress->close();
-        progress->deleteLater();
-        QObject::disconnect(*connection);
-        delete connection;
 
-        if (watcher->result()) {
-            QtCommon::Frontend::Information(tr("Exported Successfully"),
+        if (was_cancelled) {
+            Information(tr("Export Cancelled"),
+                                            tr("Export was cancelled by the user."));
+        } else if (watcher->result()) {
+            Information(tr("Exported Successfully"),
                                             tr("Data was exported successfully."));
         } else {
-            QtCommon::Frontend::Critical(
+            Critical(
                 tr("Export Failed"),
                 tr("Ensure you have write permissions on the targeted directory and try again."));
         }
 
+        progress->deleteLater();
         watcher->deleteLater();
-        callback();
+        if (callback) callback();
     });
 
     watcher->setFuture(future);
+
 }
 
 void ImportDataDir(FrontendCommon::DataManager::DataDir data_dir, std::function<void()> callback)
@@ -462,39 +479,54 @@ void ImportDataDir(FrontendCommon::DataManager::DataDir data_dir, std::function<
     if (button != QMessageBox::Yes)
         return;
 
-    FrontendCommon::DataManager::ClearDir(data_dir);
-
-    QMetaObject::Connection* connection = new QMetaObject::Connection;
-    *connection = QObject::connect(qApp, &QGuiApplication::aboutToQuit, rootObject, [=]() mutable {
-        Warning(tr("Still Importing"),
-                tr("Eden is still importing some data, and will continue "
-                   "running in the background until it's done."));
-    });
-
-    QtProgressDialog* progress = new QtProgressDialog(tr("Decompressing, this may take a while..."),
-                                                      tr("Background"),
-                                                      0,
+    QtProgressDialog* progress = new QtProgressDialog(tr("Importing data. This may take a while..."),
+                                                      tr("Cancel"),
                                                       0,
+                                                      100,
                                                       rootObject);
 
+    progress->setWindowTitle(tr("Importing"));
     progress->setWindowModality(Qt::WindowModal);
+    progress->setMinimumDuration(100);
+    progress->setAutoClose(false);
+    progress->setAutoReset(false);
     progress->show();
+    progress->setValue(0);
+
+    // Qt's wasCanceled seems to be wonky
+    bool was_cancelled = false;
+
+    QObject::connect(progress, &QtProgressDialog::canceled, rootObject, [=]() mutable {
+        was_cancelled = false;
+    });
+
     QGuiApplication::processEvents();
 
+    FrontendCommon::DataManager::ClearDir(data_dir);
+
+    auto progress_callback = [=](size_t total_size, size_t processed_size) {
+        QMetaObject::invokeMethod(progress,
+                                  &QtProgressDialog::setValue,
+                                  static_cast<int>((processed_size * 100) / total_size));
+
+        return !progress->wasCanceled();
+    };
+
     QFuture<bool> future = QtConcurrent::run([=]() {
-        return !JlCompress::extractDir(zip_dump_location,
-                                       QString::fromStdString(dir)).empty();
+        return !QtCommon::Compress::extractDir(zip_dump_location,
+                                               QString::fromStdString(dir),
+                                               progress_callback)
+                    .empty();
     });
 
     QFutureWatcher<bool>* watcher = new QFutureWatcher<bool>(rootObject);
 
     QObject::connect(watcher, &QFutureWatcher<bool>::finished, rootObject, [=]() {
         progress->close();
-        progress->deleteLater();
-        QObject::disconnect(*connection);
-        delete connection;
 
-        if (watcher->result()) {
+        if (was_cancelled) {
+            Information(tr("Import Cancelled"), tr("Import was cancelled by the user."));
+        } else if (watcher->result()) {
             Information(tr("Imported Successfully"), tr("Data was imported successfully."));
         } else {
             Critical(
@@ -502,8 +534,9 @@ void ImportDataDir(FrontendCommon::DataManager::DataDir data_dir, std::function<
                 tr("Ensure you have read permissions on the targeted directory and try again."));
         }
 
+        progress->deleteLater();
         watcher->deleteLater();
-        callback();
+        if (callback) callback();
     });
 
     watcher->setFuture(future);