Teach 'binarycreator' and 'repogen' to repack packages from repository

To both tools added options:
--repository                  The directory containing the available repository.
--ignore-invalid-repositories Ignore all invalid repositories instead of aborting.

Documentation added to ifw-tools.html page.

Task-number: QTIFW-925
Change-Id: I36519385df6166d0e450c0ef9d7df44c8611d6a6
Reviewed-by: Leena Miettinen <riitta-leena.miettinen@qt.io>
Reviewed-by: Katja Marttila <katja.marttila@qt.io>

doc/installerfw.qdoc

@@ -822,6 +822,11 @@
             \li Use \c directory as the \l{Package Directory Structure}
                 {package directory}.
                 Defaults to the current working directory.
+        \row
+            \li --repository directory
+            \li Use \c directory as the repository directory
+                with packages to repack.
+                This entry can be given multiple times.
         \row
             \li -n or --online-only
             \li Compile without any component in the installer binary.
@@ -853,6 +858,10 @@
             \li --ignore-invalid-packages
             \li Ignore component or package directories that do not have valid
                 metadata information (package.xml) to make testing faster.
+        \row
+            \li --ignore-invalid-repositories
+            \li Ignore repository directories that do not have valid
+                metadata information (Updates.xml) instead of aborting.
         \row
             \li -v or --verbose
             \li Display debug output.
@@ -907,6 +916,9 @@
     specify the location in the installer configuration file when creating an
     installer for it.
 
+    You can use an existing repository to repack packages to another
+    repository or offline installer.
+
     \section2 Summary of repogen Parameters
 
     \table
@@ -917,6 +929,12 @@
             \li -p or --packages directory
             \li Use \c directory as the \l{Package Directory Structure}
                 {package directory}. This is mandatory.
+        \row
+            \li --repository directory
+            \li Use \c directory as the repository directory
+                with packages to repack (not to confuse with the mandatory target
+                repository directory).
+                This entry can be given multiple times.
         \row
             \li repository directory
             \li Target directory for the repository. During an initial installation, the directory

tools/binarycreator/binarycreator.cpp

@@ -598,6 +598,7 @@ int main(int argc, char **argv)
     QString target;
     QString configFile;
     QStringList packagesDirectories;
+    QStringList repositoryDirectories;
     bool onlineOnly = false;
     bool offlineOnly = false;
     QStringList resources;
@@ -621,6 +622,16 @@ int main(int argc, char **argv)
                     "specified location."));
             }
             packagesDirectories.append(*it);
+        } else if (*it == QLatin1String("--repository")) {
+            ++it;
+            if (it == args.end()) {
+                return printErrorAndUsageAndExit(QString::fromLatin1("Error: Repository parameter missing argument."));
+            }
+            if (QFileInfo(*it).exists()) {
+                repositoryDirectories.append(*it);
+            } else {
+                return printErrorAndUsageAndExit(QString::fromLatin1("Error: Only local filesystem repositories now supported."));
+            }
         } else if (*it == QLatin1String("-e") || *it == QLatin1String("--exclude")) {
             ++it;
             if (!filteredPackages.isEmpty())
@@ -733,8 +744,8 @@ int main(int argc, char **argv)
     if (configFile.isEmpty())
         return printErrorAndUsageAndExit(QString::fromLatin1("Error: No configuration file selected."));
 
-    if (packagesDirectories.isEmpty())
-        return printErrorAndUsageAndExit(QString::fromLatin1("Error: Package directory parameter missing."));
+    if (packagesDirectories.isEmpty() && repositoryDirectories.isEmpty())
+        return printErrorAndUsageAndExit(QString::fromLatin1("Error: Both Package directory and Repository parameters missing."));
 
     qDebug() << "Parsed arguments, ok.";
 
@@ -752,14 +763,29 @@ int main(int argc, char **argv)
 
         // Note: the order here is important
 
-        // 1; create the list of available packages
-        QInstallerTools::PackageInfoVector packages =
-            QInstallerTools::createListOfPackages(packagesDirectories, &filteredPackages, ftype);
+        QInstallerTools::PackageInfoVector packages;
 
-        // 2; copy the packages data and setup the packages vector with the files we copied,
+        // 1; update the list of available compressed packages
+        if (!repositoryDirectories.isEmpty()) {
+            // 1.1; search packages
+            QInstallerTools::PackageInfoVector precompressedPackages = QInstallerTools::createListOfRepositoryPackages(repositoryDirectories,
+                &filteredPackages, ftype);
+            // 1.2; add to common vector
+            packages.append(precompressedPackages);
+        }
+
+        // 2; update the list of available prepared packages
+        if (!packagesDirectories.isEmpty()) {
+            // 2.1; search packages
+            QInstallerTools::PackageInfoVector preparedPackages = QInstallerTools::createListOfPackages(packagesDirectories,
+                &filteredPackages, ftype);
+            // 2.2; copy the packages data and setup the packages vector with the files we copied,
             //    must happen before copying meta data because files will be compressed if
             //    needed and meta data generation relies on this
-        QInstallerTools::copyComponentData(packagesDirectories, tmpRepoDir, &packages);
+            QInstallerTools::copyComponentData(packagesDirectories, tmpRepoDir, &preparedPackages);
+            // 2.3; add to common vector
+            packages.append(preparedPackages);
+        }
 
         // 3; copy the meta data of the available packages, generate Updates.xml
         QInstallerTools::copyMetaData(tmpMetaDir, tmpRepoDir, packages, settings

tools/common/repositorygen.cpp

@@ -27,11 +27,13 @@
 **************************************************************************/
 #include "repositorygen.h"
 
+#include <constants.h>
 #include <fileio.h>
 #include <fileutils.h>
 #include <errors.h>
 #include <globals.h>
 #include <lib7z_create.h>
+#include <lib7z_extract.h>
 #include <lib7z_facade.h>
 #include <lib7z_list.h>
 #include <settings.h>
@@ -53,6 +55,8 @@ void QInstallerTools::printRepositoryGenOptions()
 {
     std::cout << "  -p|--packages dir         The directory containing the available packages." << std::endl;
     std::cout << "                            This entry can be given multiple times." << std::endl;
+    std::cout << "  --repository dir          The directory containing the available repository." << std::endl;
+    std::cout << "                            This entry can be given multiple times." << std::endl;
 
     std::cout << "  -e|--exclude p1,...,pn    Exclude the given packages." << std::endl;
     std::cout << "  -i|--include p1,...,pn    Include the given packages and their dependencies" << std::endl;
@@ -60,6 +64,7 @@ void QInstallerTools::printRepositoryGenOptions()
 
     std::cout << "  --ignore-translations     Do not use any translation" << std::endl;
     std::cout << "  --ignore-invalid-packages Ignore all invalid packages instead of aborting." << std::endl;
+    std::cout << "  --ignore-invalid-repositories Ignore all invalid repositories instead of aborting." << std::endl;
 }
 
 QString QInstallerTools::makePathAbsolute(const QString &path)
@@ -153,6 +158,7 @@ void QInstallerTools::copyMetaData(const QString &_targetDir, const QString &met
     }
 
     foreach (const PackageInfo &info, packages) {
+        if (info.metaFile.isEmpty() && info.metaNode.isEmpty()) {
             if (!QDir(targetDir).mkpath(info.name))
                 throw QInstaller::Error(QString::fromLatin1("Cannot create directory \"%1\".").arg(info.name));
 
@@ -366,7 +372,22 @@ void QInstallerTools::copyMetaData(const QString &_targetDir, const QString &met
                 }
                 update.appendChild(package.firstChildElement(QLatin1String("Licenses")).cloneNode());
             }
+        } else {
+            // Extract metadata from archive
+            QFile metaFile(info.metaFile);
+            QInstaller::openForRead(&metaFile);
+            Lib7z::extractArchive(&metaFile, targetDir);
+
+            // Restore "PackageUpdate" node;
+            QDomDocument update;
+            if (!update.setContent(info.metaNode)) {
+                throw QInstaller::Error(QString::fromLatin1("Cannot restore \"PackageUpdate\" description for node %1").arg(info.name));
             }
+
+            root.appendChild(update.documentElement());
+        }
+    }
+
     doc.appendChild(root);
 
     QFile targetUpdatesXml(targetDir + QLatin1String("/Updates.xml"));
@@ -483,6 +504,114 @@ PackageInfoVector QInstallerTools::createListOfPackages(const QStringList &packa
     return dict;
 }
 
+PackageInfoVector QInstallerTools::createListOfRepositoryPackages(const QStringList &repositoryDirectories,
+    QStringList *packagesToFilter, FilterType filterType)
+{
+    qDebug() << "Collecting information about available repository packages...";
+
+    bool ignoreInvalidRepositories = qApp->arguments().contains(QString::fromLatin1("--ignore-invalid-repositories"));
+
+    PackageInfoVector dict;
+    QFileInfoList entries;
+    foreach (const QString &repositoryDirectory, repositoryDirectories)
+        entries.append(QFileInfo(repositoryDirectory));
+    for (QFileInfoList::const_iterator it = entries.constBegin(); it != entries.constEnd(); ++it) {
+
+        qDebug() << "Process repository" << it->fileName();
+
+        QFile file(QString::fromLatin1("%1/Updates.xml").arg(it->filePath()));
+
+        QFileInfo fileInfo(file);
+        if (!fileInfo.exists()) {
+            if (ignoreInvalidRepositories) {
+                qDebug() << "- skip invalid repository";
+                continue;
+            }
+            throw QInstaller::Error(QString::fromLatin1("Repository \"%1\" does not contain a update "
+                "description (Updates.xml is missing).").arg(QDir::toNativeSeparators(it->fileName())));
+        }
+        if (!file.open(QIODevice::ReadOnly)) {
+            qDebug() << "Cannot open Updates.xml for reading:" << file.errorString();
+            continue;
+        }
+
+        QString error;
+        QDomDocument doc;
+        if (!doc.setContent(&file, &error)) {
+            qDebug().nospace() << "Cannot fetch a valid version of Updates.xml from repository "
+                               << it->fileName() << ": " << error;
+            continue;
+        }
+        file.close();
+
+        const QDomElement root = doc.documentElement();
+        if (root.tagName() != QLatin1String("Updates")) {
+            throw QInstaller::Error(QCoreApplication::translate("QInstaller",
+                "Invalid content in \"%1\".").arg(QDir::toNativeSeparators(file.fileName())));
+        }
+
+        const QDomNodeList children = root.childNodes();
+        for (int i = 0; i < children.count(); ++i) {
+            const QDomElement el = children.at(i).toElement();
+            if ((!el.isNull()) && (el.tagName() == QLatin1String("PackageUpdate"))) {
+                QInstallerTools::PackageInfo info;
+
+                QDomElement c1 = el.firstChildElement(QInstaller::scName);
+                if (!c1.isNull())
+                    info.name = c1.text();
+                else
+                    continue;
+                if (filterType == Exclude) {
+                    // Check for current package in exclude list, if found, skip it
+                    if (packagesToFilter->contains(info.name)) {
+                        continue;
+                    }
+                } else {
+                    // Check for current package in include list, if not found, skip it
+                    if (!packagesToFilter->contains(info.name))
+                        continue;
+                }
+                c1 = el.firstChildElement(QInstaller::scVersion);
+                if (!c1.isNull())
+                    info.version = c1.text();
+                else
+                    continue;
+
+                info.directory = QString::fromLatin1("%1/%2").arg(it->filePath(), info.name);
+                info.metaFile = QString::fromLatin1("%1/%3%2").arg(info.directory,
+                    QString::fromLatin1("meta.7z"), info.version);
+
+                const QDomNodeList c2 = el.childNodes();
+                for (int j = 0; j < c2.count(); ++j) {
+                    if (c2.at(j).toElement().tagName() == QInstaller::scDependencies)
+                        info.dependencies = c2.at(j).toElement().text()
+                            .split(QInstaller::commaRegExp(), QString::SkipEmptyParts);
+                    else if (c2.at(j).toElement().tagName() == QInstaller::scDownloadableArchives) {
+                        QStringList names = c2.at(j).toElement().text()
+                            .split(QInstaller::commaRegExp(), QString::SkipEmptyParts);
+                        foreach (const QString &name, names) {
+                            info.copiedFiles.append(QString::fromLatin1("%1/%3%2").arg(info.directory,
+                                name, info.version));
+                            info.copiedFiles.append(QString::fromLatin1("%1/%3%2.sha1").arg(info.directory,
+                                name, info.version));
+                        }
+                    }
+                }
+                QString metaString;
+                {
+                    QTextStream metaStream(&metaString);
+                    el.save(metaStream, 0);
+                }
+                info.metaNode = metaString;
+                dict.push_back(info);
+                qDebug() << "- it provides the package" << info.name << " - " << info.version;
+            }
+        }
+    }
+
+    return dict;
+}
+
 QHash<QString, QString> QInstallerTools::buildPathToVersionMapping(const PackageInfoVector &info)
 {
     QHash<QString, QString> map;
@@ -568,6 +697,7 @@ void QInstallerTools::copyComponentData(const QStringList &packageDirs, const QS
                 .arg(name));
         }
 
+        if (info.copiedFiles.isEmpty()) {
             QStringList compressedFiles;
             QStringList filesToCompress;
             foreach (const QString &packageDir, packageDirs) {
@@ -634,5 +764,17 @@ void QInstallerTools::copyComponentData(const QStringList &packageDirs, const QS
                     throw;
                 }
             }
+        } else {
+            foreach (const QString &file, (*infos)[i].copiedFiles) {
+                QFileInfo fromInfo(file);
+                QFile from(file);
+                QString target = QString::fromLatin1("%1/%2").arg(namedRepoDir, fromInfo.fileName());
+                qDebug() << "Copying file from" << from.fileName() << "to" << target;
+                if (!from.copy(target)) {
+                    throw QInstaller::Error(QString::fromLatin1("Cannot copy file \"%1\" to \"%2\": %3")
+                        .arg(QDir::toNativeSeparators(from.fileName()), QDir::toNativeSeparators(target), from.errorString()));
+                }
+            }
+        }
     }
 }

tools/common/repositorygen.h

@@ -44,6 +44,8 @@ struct PackageInfo
     QString directory;
     QStringList dependencies;
     QStringList copiedFiles;
+    QString metaFile;
+    QString metaNode;
 };
 typedef QVector<PackageInfo> PackageInfoVector;
 
@@ -58,6 +60,10 @@ void copyWithException(const QString &source, const QString &target, const QStri
 
 PackageInfoVector createListOfPackages(const QStringList &packagesDirectories, QStringList *packagesToFilter,
     FilterType ftype);
+
+PackageInfoVector createListOfRepositoryPackages(const QStringList &repositoryDirectories, QStringList *packagesToFilter,
+    FilterType filterType);
+
 QHash<QString, QString> buildPathToVersionMapping(const PackageInfoVector &info);
 
 void compressMetaDirectories(const QString &repoDir, const QString &baseDir,

tools/repogen/repogen.cpp

@@ -95,6 +95,7 @@ int main(int argc, char** argv)
         QStringList filteredPackages;
         bool updateExistingRepository = false;
         QStringList packagesDirectories;
+        QStringList repositoryDirectories;
         QInstallerTools::FilterType filterType = QInstallerTools::Exclude;
         bool remove = false;
         bool updateExistingRepositoryWithNewComponents = false;
@@ -156,6 +157,19 @@ int main(int argc, char** argv)
 
                 packagesDirectories.append(args.first());
                 args.removeFirst();
+            } else if (args.first() == QLatin1String("--repository")) {
+                args.removeFirst();
+                if (args.isEmpty()) {
+                    return printErrorAndUsageAndExit(QCoreApplication::translate("QInstaller",
+                        "Error: Repository parameter missing argument"));
+                }
+
+                if (!QFileInfo(args.first()).exists()) {
+                    return printErrorAndUsageAndExit(QCoreApplication::translate("QInstaller",
+                        "Error: Only local filesystem repositories now supported"));
+                }
+                repositoryDirectories.append(args.first());
+                args.removeFirst();
             } else if (args.first() == QLatin1String("--ignore-translations")
                 || args.first() == QLatin1String("--ignore-invalid-packages")) {
                     args.removeFirst();
@@ -168,7 +182,7 @@ int main(int argc, char** argv)
             }
         }
 
-        if (packagesDirectories.isEmpty() || (args.count() != 1)) {
+        if ((packagesDirectories.isEmpty() && repositoryDirectories.isEmpty()) || (args.count() != 1)) {
                 printUsage();
                 return 1;
         }
@@ -190,8 +204,15 @@ int main(int argc, char** argv)
                 "Repository target directory \"%1\" already exists.").arg(QDir::toNativeSeparators(repositoryDir)));
         }
 
-        QInstallerTools::PackageInfoVector packages = QInstallerTools::createListOfPackages(packagesDirectories,
+        QInstallerTools::PackageInfoVector packages;
+
+        QInstallerTools::PackageInfoVector precompressedPackages = QInstallerTools::createListOfRepositoryPackages(repositoryDirectories,
             &filteredPackages, filterType);
+        packages.append(precompressedPackages);
+
+        QInstallerTools::PackageInfoVector preparedPackages = QInstallerTools::createListOfPackages(packagesDirectories,
+            &filteredPackages, filterType);
+        packages.append(preparedPackages);
 
         if (updateExistingRepositoryWithNewComponents) {
             QDomDocument doc;
@@ -251,7 +272,10 @@ int main(int argc, char** argv)
         QTemporaryDir tmp;
         tmp.setAutoRemove(false);
         tmpMetaDir = tmp.path();
-        QInstallerTools::copyComponentData(packagesDirectories, repositoryDir, &packages);
+        QStringList directories;
+        directories.append(packagesDirectories);
+        directories.append(repositoryDirectories);
+        QInstallerTools::copyComponentData(directories, repositoryDir, &packages);
         QInstallerTools::copyMetaData(tmpMetaDir, repositoryDir, packages, QLatin1String("{AnyApplication}"),
             QLatin1String(QUOTE(IFW_REPOSITORY_FORMAT_VERSION)));
         QInstallerTools::compressMetaDirectories(tmpMetaDir, tmpMetaDir, pathToVersionMapping);