QuasarApp/installer-framework
4
0
Fork 0
mirror of https://github.com/QuasarApp/installer-framework.git synced 2025-05-03 00:39:33 +00:00
Code Issues Projects Releases Wiki Activity

Teach 'binarycreator' and 'repogen' to repack packages from repository

Browse Source 
To both tools added options:
--repository                  The directory containing the available repository.
--ignore-invalid-repositories Ignore all invalid repositories instead of aborting.

Documentation added to ifw-tools.html page.

Task-number: QTIFW-925
Change-Id: I36519385df6166d0e450c0ef9d7df44c8611d6a6
Reviewed-by: Leena Miettinen <riitta-leena.miettinen@qt.io>
Reviewed-by: Katja Marttila <katja.marttila@qt.io>
This commit is contained in:
Konstantin Podsvirov 2017-02-03 16:56:37 +03:00 committed by Katja Marttila
parent 6f6a632b2d
commit 2385184b8b
5 changed files with 472 additions and 256 deletions

18
doc/installerfw.qdoc

@ -822,6 +822,11 @@
\li Use \c directory as the \l{Package Directory Structure} \li Use \c directory as the \l{Package Directory Structure}
{package directory}. {package directory}.
Defaults to the current working directory. Defaults to the current working directory.
\row
\li --repository directory
\li Use \c directory as the repository directory
with packages to repack.
This entry can be given multiple times.
\row \row
\li -n or --online-only \li -n or --online-only
\li Compile without any component in the installer binary. \li Compile without any component in the installer binary.
@ -853,6 +858,10 @@
\li --ignore-invalid-packages \li --ignore-invalid-packages
\li Ignore component or package directories that do not have valid \li Ignore component or package directories that do not have valid
metadata information (package.xml) to make testing faster. metadata information (package.xml) to make testing faster.
\row
\li --ignore-invalid-repositories
\li Ignore repository directories that do not have valid
metadata information (Updates.xml) instead of aborting.
\row \row
\li -v or --verbose \li -v or --verbose
\li Display debug output. \li Display debug output.
@ -907,6 +916,9 @@
specify the location in the installer configuration file when creating an specify the location in the installer configuration file when creating an
installer for it. installer for it.
You can use an existing repository to repack packages to another
repository or offline installer.
\section2 Summary of repogen Parameters \section2 Summary of repogen Parameters
\table \table
@ -917,6 +929,12 @@
\li -p or --packages directory \li -p or --packages directory
\li Use \c directory as the \l{Package Directory Structure} \li Use \c directory as the \l{Package Directory Structure}
{package directory}. This is mandatory. {package directory}. This is mandatory.
\row
\li --repository directory
\li Use \c directory as the repository directory
with packages to repack (not to confuse with the mandatory target
repository directory).
This entry can be given multiple times.
\row \row
\li repository directory \li repository directory
\li Target directory for the repository. During an initial installation, the directory \li Target directory for the repository. During an initial installation, the directory

44
tools/binarycreator/binarycreator.cpp

@ -598,6 +598,7 @@ int main(int argc, char **argv)
QString target; QString target;
QString configFile; QString configFile;
QStringList packagesDirectories; QStringList packagesDirectories;
QStringList repositoryDirectories;
bool onlineOnly = false; bool onlineOnly = false;
bool offlineOnly = false; bool offlineOnly = false;
QStringList resources; QStringList resources;
@ -621,6 +622,16 @@ int main(int argc, char **argv)
"specified location.")); "specified location."));
} }
packagesDirectories.append(*it); packagesDirectories.append(*it);
} else if (*it == QLatin1String("--repository")) {
++it;
if (it == args.end()) {
return printErrorAndUsageAndExit(QString::fromLatin1("Error: Repository parameter missing argument."));
}
if (QFileInfo(*it).exists()) {
repositoryDirectories.append(*it);
} else {
return printErrorAndUsageAndExit(QString::fromLatin1("Error: Only local filesystem repositories now supported."));
}
} else if (*it == QLatin1String("-e") || *it == QLatin1String("--exclude")) { } else if (*it == QLatin1String("-e") || *it == QLatin1String("--exclude")) {
++it; ++it;
if (!filteredPackages.isEmpty()) if (!filteredPackages.isEmpty())
@ -733,8 +744,8 @@ int main(int argc, char **argv)
if (configFile.isEmpty()) if (configFile.isEmpty())
return printErrorAndUsageAndExit(QString::fromLatin1("Error: No configuration file selected.")); return printErrorAndUsageAndExit(QString::fromLatin1("Error: No configuration file selected."));
if (packagesDirectories.isEmpty()) if (packagesDirectories.isEmpty() && repositoryDirectories.isEmpty())
return printErrorAndUsageAndExit(QString::fromLatin1("Error: Package directory parameter missing.")); return printErrorAndUsageAndExit(QString::fromLatin1("Error: Both Package directory and Repository parameters missing."));
qDebug() << "Parsed arguments, ok."; qDebug() << "Parsed arguments, ok.";
@ -752,14 +763,29 @@ int main(int argc, char **argv)
// Note: the order here is important // Note: the order here is important
// 1; create the list of available packages QInstallerTools::PackageInfoVector packages;
QInstallerTools::PackageInfoVector packages =
QInstallerTools::createListOfPackages(packagesDirectories, &filteredPackages, ftype);
// 2; copy the packages data and setup the packages vector with the files we copied, // 1; update the list of available compressed packages
// must happen before copying meta data because files will be compressed if if (!repositoryDirectories.isEmpty()) {
// needed and meta data generation relies on this // 1.1; search packages
QInstallerTools::copyComponentData(packagesDirectories, tmpRepoDir, &packages); QInstallerTools::PackageInfoVector precompressedPackages = QInstallerTools::createListOfRepositoryPackages(repositoryDirectories,
&filteredPackages, ftype);
// 1.2; add to common vector
packages.append(precompressedPackages);
}
// 2; update the list of available prepared packages
if (!packagesDirectories.isEmpty()) {
// 2.1; search packages
QInstallerTools::PackageInfoVector preparedPackages = QInstallerTools::createListOfPackages(packagesDirectories,
&filteredPackages, ftype);
// 2.2; copy the packages data and setup the packages vector with the files we copied,
// must happen before copying meta data because files will be compressed if
// needed and meta data generation relies on this
QInstallerTools::copyComponentData(packagesDirectories, tmpRepoDir, &preparedPackages);
// 2.3; add to common vector
packages.append(preparedPackages);
}
// 3; copy the meta data of the available packages, generate Updates.xml // 3; copy the meta data of the available packages, generate Updates.xml
QInstallerTools::copyMetaData(tmpMetaDir, tmpRepoDir, packages, settings QInstallerTools::copyMetaData(tmpMetaDir, tmpRepoDir, packages, settings

630
tools/common/repositorygen.cpp

@ -27,11 +27,13 @@
**************************************************************************/ **************************************************************************/
#include "repositorygen.h" #include "repositorygen.h"
#include <constants.h>
#include <fileio.h> #include <fileio.h>
#include <fileutils.h> #include <fileutils.h>
#include <errors.h> #include <errors.h>
#include <globals.h> #include <globals.h>
#include <lib7z_create.h> #include <lib7z_create.h>
#include <lib7z_extract.h>
#include <lib7z_facade.h> #include <lib7z_facade.h>
#include <lib7z_list.h> #include <lib7z_list.h>
#include <settings.h> #include <settings.h>
@ -53,6 +55,8 @@ void QInstallerTools::printRepositoryGenOptions()
{ {
std::cout << " -p|--packages dir The directory containing the available packages." << std::endl; std::cout << " -p|--packages dir The directory containing the available packages." << std::endl;
std::cout << " This entry can be given multiple times." << std::endl; std::cout << " This entry can be given multiple times." << std::endl;
std::cout << " --repository dir The directory containing the available repository." << std::endl;
std::cout << " This entry can be given multiple times." << std::endl;
std::cout << " -e|--exclude p1,...,pn Exclude the given packages." << std::endl; std::cout << " -e|--exclude p1,...,pn Exclude the given packages." << std::endl;
std::cout << " -i|--include p1,...,pn Include the given packages and their dependencies" << std::endl; std::cout << " -i|--include p1,...,pn Include the given packages and their dependencies" << std::endl;
@ -60,6 +64,7 @@ void QInstallerTools::printRepositoryGenOptions()
std::cout << " --ignore-translations Do not use any translation" << std::endl; std::cout << " --ignore-translations Do not use any translation" << std::endl;
std::cout << " --ignore-invalid-packages Ignore all invalid packages instead of aborting." << std::endl; std::cout << " --ignore-invalid-packages Ignore all invalid packages instead of aborting." << std::endl;
std::cout << " --ignore-invalid-repositories Ignore all invalid repositories instead of aborting." << std::endl;
} }
QString QInstallerTools::makePathAbsolute(const QString &path) QString QInstallerTools::makePathAbsolute(const QString &path)
@ -153,220 +158,236 @@ void QInstallerTools::copyMetaData(const QString &_targetDir, const QString &met
} }
foreach (const PackageInfo &info, packages) { foreach (const PackageInfo &info, packages) {
if (!QDir(targetDir).mkpath(info.name)) if (info.metaFile.isEmpty() && info.metaNode.isEmpty()) {
throw QInstaller::Error(QString::fromLatin1("Cannot create directory \"%1\".").arg(info.name)); if (!QDir(targetDir).mkpath(info.name))
throw QInstaller::Error(QString::fromLatin1("Cannot create directory \"%1\".").arg(info.name));
const QString packageXmlPath = QString::fromLatin1("%1/meta/package.xml").arg(info.directory); const QString packageXmlPath = QString::fromLatin1("%1/meta/package.xml").arg(info.directory);
qDebug() << "Copy meta data for package" << info.name << "using" << packageXmlPath; qDebug() << "Copy meta data for package" << info.name << "using" << packageXmlPath;
QFile file(packageXmlPath); QFile file(packageXmlPath);
QInstaller::openForRead(&file); QInstaller::openForRead(&file);
QString errMsg; QString errMsg;
int line = 0; int line = 0;
int column = 0; int column = 0;
QDomDocument packageXml; QDomDocument packageXml;
if (!packageXml.setContent(&file, &errMsg, &line, &column)) { if (!packageXml.setContent(&file, &errMsg, &line, &column)) {
throw QInstaller::Error(QString::fromLatin1("Cannot parse \"%1\": line: %2, column: %3: %4 (%5)") throw QInstaller::Error(QString::fromLatin1("Cannot parse \"%1\": line: %2, column: %3: %4 (%5)")
.arg(QDir::toNativeSeparators(packageXmlPath)).arg(line).arg(column).arg(errMsg, info.name)); .arg(QDir::toNativeSeparators(packageXmlPath)).arg(line).arg(column).arg(errMsg, info.name));
}
QDomElement update = doc.createElement(QLatin1String("PackageUpdate"));
QDomNode nameElement = update.appendChild(doc.createElement(QLatin1String("Name")));
nameElement.appendChild(doc.createTextNode(info.name));
// list of current unused or later transformed tags
QStringList blackList;
blackList << QLatin1String("UserInterfaces") << QLatin1String("Translations") <<
QLatin1String("Licenses") << QLatin1String("Name");
bool foundDefault = false;
bool foundVirtual = false;
bool foundDisplayName = false;
bool foundDownloadableArchives = false;
bool foundCheckable = false;
const QDomNode package = packageXml.firstChildElement(QLatin1String("Package"));
const QDomNodeList childNodes = package.childNodes();
for (int i = 0; i < childNodes.count(); ++i) {
const QDomNode node = childNodes.at(i);
const QString key = node.nodeName();
if (key == QLatin1String("Default"))
foundDefault = true;
if (key == QLatin1String("Virtual"))
foundVirtual = true;
if (key == QLatin1String("DisplayName"))
foundDisplayName = true;
if (key == QLatin1String("DownloadableArchives"))
foundDownloadableArchives = true;
if (key == QLatin1String("Checkable"))
foundCheckable = true;
if (node.isComment() || blackList.contains(key))
continue; // just skip comments and some tags...
QDomElement element = doc.createElement(key);
for (int j = 0; j < node.attributes().size(); ++j) {
element.setAttribute(node.attributes().item(j).toAttr().name(),
node.attributes().item(j).toAttr().value());
} }
update.appendChild(element).appendChild(doc.createTextNode(node.toElement().text()));
}
if (foundDefault && foundVirtual) { QDomElement update = doc.createElement(QLatin1String("PackageUpdate"));
throw QInstaller::Error(QString::fromLatin1("Error: <Default> and <Virtual> elements are " QDomNode nameElement = update.appendChild(doc.createElement(QLatin1String("Name")));
"mutually exclusive in file \"%1\".").arg(QDir::toNativeSeparators(packageXmlPath))); nameElement.appendChild(doc.createTextNode(info.name));
}
if (foundDefault && foundCheckable) { // list of current unused or later transformed tags
throw QInstaller::Error(QString::fromLatin1("Error: <Default> and <Checkable>" QStringList blackList;
"elements are mutually exclusive in file \"%1\".") blackList << QLatin1String("UserInterfaces") << QLatin1String("Translations") <<
.arg(QDir::toNativeSeparators(packageXmlPath))); QLatin1String("Licenses") << QLatin1String("Name");
}
if (!foundDisplayName) { bool foundDefault = false;
qWarning() << "No DisplayName tag found at" << info.name << ", using component Name instead."; bool foundVirtual = false;
QDomElement displayNameElement = doc.createElement(QLatin1String("DisplayName")); bool foundDisplayName = false;
update.appendChild(displayNameElement).appendChild(doc.createTextNode(info.name)); bool foundDownloadableArchives = false;
} bool foundCheckable = false;
const QDomNode package = packageXml.firstChildElement(QLatin1String("Package"));
const QDomNodeList childNodes = package.childNodes();
for (int i = 0; i < childNodes.count(); ++i) {
const QDomNode node = childNodes.at(i);
const QString key = node.nodeName();
// get the size of the data if (key == QLatin1String("Default"))
quint64 componentSize = 0; foundDefault = true;
quint64 compressedComponentSize = 0; if (key == QLatin1String("Virtual"))
foundVirtual = true;
if (key == QLatin1String("DisplayName"))
foundDisplayName = true;
if (key == QLatin1String("DownloadableArchives"))
foundDownloadableArchives = true;
if (key == QLatin1String("Checkable"))
foundCheckable = true;
if (node.isComment() || blackList.contains(key))
continue; // just skip comments and some tags...
const QDir::Filters filters = QDir::Files | QDir::NoDotAndDotDot; QDomElement element = doc.createElement(key);
const QDir dataDir = QString::fromLatin1("%1/%2/data").arg(metaDataDir, info.name); for (int j = 0; j < node.attributes().size(); ++j) {
const QFileInfoList entries = dataDir.exists() ? dataDir.entryInfoList(filters | QDir::Dirs) element.setAttribute(node.attributes().item(j).toAttr().name(),
: QDir(QString::fromLatin1("%1/%2").arg(metaDataDir, info.name)).entryInfoList(filters); node.attributes().item(j).toAttr().value());
qDebug() << "calculate size of directory" << dataDir.absolutePath(); }
foreach (const QFileInfo &fi, entries) { update.appendChild(element).appendChild(doc.createTextNode(node.toElement().text()));
try { }
if (fi.isDir()) {
QDirIterator recursDirIt(fi.filePath(), QDirIterator::Subdirectories); if (foundDefault && foundVirtual) {
while (recursDirIt.hasNext()) { throw QInstaller::Error(QString::fromLatin1("Error: <Default> and <Virtual> elements are "
recursDirIt.next(); "mutually exclusive in file \"%1\".").arg(QDir::toNativeSeparators(packageXmlPath)));
const quint64 size = QInstaller::fileSize(recursDirIt.fileInfo()); }
if (foundDefault && foundCheckable) {
throw QInstaller::Error(QString::fromLatin1("Error: <Default> and <Checkable>"
"elements are mutually exclusive in file \"%1\".")
.arg(QDir::toNativeSeparators(packageXmlPath)));
}
if (!foundDisplayName) {
qWarning() << "No DisplayName tag found at" << info.name << ", using component Name instead.";
QDomElement displayNameElement = doc.createElement(QLatin1String("DisplayName"));
update.appendChild(displayNameElement).appendChild(doc.createTextNode(info.name));
}
// get the size of the data
quint64 componentSize = 0;
quint64 compressedComponentSize = 0;
const QDir::Filters filters = QDir::Files | QDir::NoDotAndDotDot;
const QDir dataDir = QString::fromLatin1("%1/%2/data").arg(metaDataDir, info.name);
const QFileInfoList entries = dataDir.exists() ? dataDir.entryInfoList(filters | QDir::Dirs)
: QDir(QString::fromLatin1("%1/%2").arg(metaDataDir, info.name)).entryInfoList(filters);
qDebug() << "calculate size of directory" << dataDir.absolutePath();
foreach (const QFileInfo &fi, entries) {
try {
if (fi.isDir()) {
QDirIterator recursDirIt(fi.filePath(), QDirIterator::Subdirectories);
while (recursDirIt.hasNext()) {
recursDirIt.next();
const quint64 size = QInstaller::fileSize(recursDirIt.fileInfo());
componentSize += size;
compressedComponentSize += size;
}
} else if (Lib7z::isSupportedArchive(fi.filePath())) {
// if it's an archive already, list its files and sum the uncompressed sizes
QFile archive(fi.filePath());
compressedComponentSize += archive.size();
QInstaller::openForRead(&archive);
QVector<Lib7z::File>::const_iterator fileIt;
const QVector<Lib7z::File> files = Lib7z::listArchive(&archive);
for (fileIt = files.begin(); fileIt != files.end(); ++fileIt)
componentSize += fileIt->uncompressedSize;
} else {
// otherwise just add its size
const quint64 size = QInstaller::fileSize(fi);
componentSize += size; componentSize += size;
compressedComponentSize += size; compressedComponentSize += size;
} }
} else if (Lib7z::isSupportedArchive(fi.filePath())) { } catch (const QInstaller::Error &error) {
// if it's an archive already, list its files and sum the uncompressed sizes qDebug().noquote() << error.message();
QFile archive(fi.filePath()); } catch(...) {
compressedComponentSize += archive.size(); // ignore, that's just about the sizes - and size doesn't matter, you know?
QInstaller::openForRead(&archive);
QVector<Lib7z::File>::const_iterator fileIt;
const QVector<Lib7z::File> files = Lib7z::listArchive(&archive);
for (fileIt = files.begin(); fileIt != files.end(); ++fileIt)
componentSize += fileIt->uncompressedSize;
} else {
// otherwise just add its size
const quint64 size = QInstaller::fileSize(fi);
componentSize += size;
compressedComponentSize += size;
}
} catch (const QInstaller::Error &error) {
qDebug().noquote() << error.message();
} catch(...) {
// ignore, that's just about the sizes - and size doesn't matter, you know?
}
}
QDomElement fileElement = doc.createElement(QLatin1String("UpdateFile"));
fileElement.setAttribute(QLatin1String("UncompressedSize"), componentSize);
fileElement.setAttribute(QLatin1String("CompressedSize"), compressedComponentSize);
// adding the OS attribute to be compatible with old sdks
fileElement.setAttribute(QLatin1String("OS"), QLatin1String("Any"));
update.appendChild(fileElement);
root.appendChild(update);
// copy script file
const QString script = package.firstChildElement(QLatin1String("Script")).text();
if (!script.isEmpty()) {
QFile scriptFile(QString::fromLatin1("%1/meta/%2").arg(info.directory, script));
if (!scriptFile.open(QIODevice::ReadOnly | QIODevice::Text)) {
throw QInstaller::Error(QString::fromLatin1("Cannot open component script at \"%1\".")
.arg(QDir::toNativeSeparators(scriptFile.fileName())));
}
const QString scriptContent = QLatin1String("(function() {")
+ QString::fromUtf8(scriptFile.readAll())
+ QLatin1String(";"
" if (typeof Component == \"undefined\")"
" throw \"Missing Component constructor. Please check your script.\";"
"})();");
// if the user isn't aware of the downloadable archives value we will add it automatically later
foundDownloadableArchives |= scriptContent.contains(QLatin1String("addDownloadableArchive"))
|| scriptContent.contains(QLatin1String("removeDownloadableArchive"));
static QInstaller::ScriptEngine testScriptEngine;
const QJSValue value = testScriptEngine.evaluate(scriptContent, scriptFile.fileName());
if (value.isError()) {
throw QInstaller::Error(QString::fromLatin1("Exception while loading component "
"script at \"%1\": %2").arg(QDir::toNativeSeparators(scriptFile.fileName()),
value.toString().isEmpty() ?
QString::fromLatin1("Unknown error.") : value.toString()));
}
const QString toLocation(QString::fromLatin1("%1/%2/%3").arg(targetDir, info.name, script));
copyWithException(scriptFile.fileName(), toLocation, QInstaller::scScript);
}
// write DownloadableArchives tag if that is missed by the user
if (!foundDownloadableArchives && !info.copiedFiles.isEmpty()) {
QStringList realContentFiles;
foreach (const QString &filePath, info.copiedFiles) {
if (!filePath.endsWith(QLatin1String(".sha1"), Qt::CaseInsensitive)) {
const QString fileName = QFileInfo(filePath).fileName();
// remove unnecessary version string from filename and add it to the list
realContentFiles.append(fileName.mid(info.version.count()));
} }
} }
update.appendChild(doc.createElement(QLatin1String("DownloadableArchives"))).appendChild(doc QDomElement fileElement = doc.createElement(QLatin1String("UpdateFile"));
.createTextNode(realContentFiles.join(QChar::fromLatin1(',')))); fileElement.setAttribute(QLatin1String("UncompressedSize"), componentSize);
} fileElement.setAttribute(QLatin1String("CompressedSize"), compressedComponentSize);
// adding the OS attribute to be compatible with old sdks
fileElement.setAttribute(QLatin1String("OS"), QLatin1String("Any"));
update.appendChild(fileElement);
// copy user interfaces root.appendChild(update);
const QStringList uiFiles = copyFilesFromNode(QLatin1String("UserInterfaces"),
QLatin1String("UserInterface"), QString(), QLatin1String("user interface"), package, info,
targetDir);
if (!uiFiles.isEmpty()) {
update.appendChild(doc.createElement(QLatin1String("UserInterfaces"))).appendChild(doc
.createTextNode(uiFiles.join(QChar::fromLatin1(','))));
}
// copy translations // copy script file
QStringList trFiles; const QString script = package.firstChildElement(QLatin1String("Script")).text();
if (!qApp->arguments().contains(QString::fromLatin1("--ignore-translations"))) { if (!script.isEmpty()) {
trFiles = copyFilesFromNode(QLatin1String("Translations"), QLatin1String("Translation"), QFile scriptFile(QString::fromLatin1("%1/meta/%2").arg(info.directory, script));
QString(), QLatin1String("translation"), package, info, targetDir); if (!scriptFile.open(QIODevice::ReadOnly | QIODevice::Text)) {
if (!trFiles.isEmpty()) { throw QInstaller::Error(QString::fromLatin1("Cannot open component script at \"%1\".")
update.appendChild(doc.createElement(QLatin1String("Translations"))).appendChild(doc .arg(QDir::toNativeSeparators(scriptFile.fileName())));
.createTextNode(trFiles.join(QChar::fromLatin1(',')))); }
const QString scriptContent = QLatin1String("(function() {")
+ QString::fromUtf8(scriptFile.readAll())
+ QLatin1String(";"
" if (typeof Component == \"undefined\")"
" throw \"Missing Component constructor. Please check your script.\";"
"})();");
// if the user isn't aware of the downloadable archives value we will add it automatically later
foundDownloadableArchives |= scriptContent.contains(QLatin1String("addDownloadableArchive"))
|| scriptContent.contains(QLatin1String("removeDownloadableArchive"));
static QInstaller::ScriptEngine testScriptEngine;
const QJSValue value = testScriptEngine.evaluate(scriptContent, scriptFile.fileName());
if (value.isError()) {
throw QInstaller::Error(QString::fromLatin1("Exception while loading component "
"script at \"%1\": %2").arg(QDir::toNativeSeparators(scriptFile.fileName()),
value.toString().isEmpty() ?
QString::fromLatin1("Unknown error.") : value.toString()));
}
const QString toLocation(QString::fromLatin1("%1/%2/%3").arg(targetDir, info.name, script));
copyWithException(scriptFile.fileName(), toLocation, QInstaller::scScript);
} }
}
// copy license files // write DownloadableArchives tag if that is missed by the user
const QStringList licenses = copyFilesFromNode(QLatin1String("Licenses"), QLatin1String("License"), if (!foundDownloadableArchives && !info.copiedFiles.isEmpty()) {
QLatin1String("file"), QLatin1String("license"), package, info, targetDir); QStringList realContentFiles;
if (!licenses.isEmpty()) { foreach (const QString &filePath, info.copiedFiles) {
foreach (const QString &trFile, trFiles) { if (!filePath.endsWith(QLatin1String(".sha1"), Qt::CaseInsensitive)) {
// Copy translated license file based on the assumption that it will have the same base name const QString fileName = QFileInfo(filePath).fileName();
// as the original license plus the file name of an existing translation file without suffix. // remove unnecessary version string from filename and add it to the list
foreach (const QString &license, licenses) { realContentFiles.append(fileName.mid(info.version.count()));
const QFileInfo untranslated(license); }
const QString translatedLicense = QString::fromLatin1("%2_%3.%4").arg(untranslated }
.baseName(), QFileInfo(trFile).baseName(), untranslated.completeSuffix());
// ignore copy failure, that's just about the translations update.appendChild(doc.createElement(QLatin1String("DownloadableArchives"))).appendChild(doc
QFile::copy(QString::fromLatin1("%1/meta/%2").arg(info.directory).arg(translatedLicense), .createTextNode(realContentFiles.join(QChar::fromLatin1(','))));
QString::fromLatin1("%1/%2/%3").arg(targetDir, info.name, translatedLicense)); }
// copy user interfaces
const QStringList uiFiles = copyFilesFromNode(QLatin1String("UserInterfaces"),
QLatin1String("UserInterface"), QString(), QLatin1String("user interface"), package, info,
targetDir);
if (!uiFiles.isEmpty()) {
update.appendChild(doc.createElement(QLatin1String("UserInterfaces"))).appendChild(doc
.createTextNode(uiFiles.join(QChar::fromLatin1(','))));
}
// copy translations
QStringList trFiles;
if (!qApp->arguments().contains(QString::fromLatin1("--ignore-translations"))) {
trFiles = copyFilesFromNode(QLatin1String("Translations"), QLatin1String("Translation"),
QString(), QLatin1String("translation"), package, info, targetDir);
if (!trFiles.isEmpty()) {
update.appendChild(doc.createElement(QLatin1String("Translations"))).appendChild(doc
.createTextNode(trFiles.join(QChar::fromLatin1(','))));
} }
} }
update.appendChild(package.firstChildElement(QLatin1String("Licenses")).cloneNode());
// copy license files
const QStringList licenses = copyFilesFromNode(QLatin1String("Licenses"), QLatin1String("License"),
QLatin1String("file"), QLatin1String("license"), package, info, targetDir);
if (!licenses.isEmpty()) {
foreach (const QString &trFile, trFiles) {
// Copy translated license file based on the assumption that it will have the same base name
// as the original license plus the file name of an existing translation file without suffix.
foreach (const QString &license, licenses) {
const QFileInfo untranslated(license);
const QString translatedLicense = QString::fromLatin1("%2_%3.%4").arg(untranslated
.baseName(), QFileInfo(trFile).baseName(), untranslated.completeSuffix());
// ignore copy failure, that's just about the translations
QFile::copy(QString::fromLatin1("%1/meta/%2").arg(info.directory).arg(translatedLicense),
QString::fromLatin1("%1/%2/%3").arg(targetDir, info.name, translatedLicense));
}
}
update.appendChild(package.firstChildElement(QLatin1String("Licenses")).cloneNode());
}
} else {
// Extract metadata from archive
QFile metaFile(info.metaFile);
QInstaller::openForRead(&metaFile);
Lib7z::extractArchive(&metaFile, targetDir);
// Restore "PackageUpdate" node;
QDomDocument update;
if (!update.setContent(info.metaNode)) {
throw QInstaller::Error(QString::fromLatin1("Cannot restore \"PackageUpdate\" description for node %1").arg(info.name));
}
root.appendChild(update.documentElement());
} }
} }
doc.appendChild(root); doc.appendChild(root);
QFile targetUpdatesXml(targetDir + QLatin1String("/Updates.xml")); QFile targetUpdatesXml(targetDir + QLatin1String("/Updates.xml"));
@ -483,6 +504,114 @@ PackageInfoVector QInstallerTools::createListOfPackages(const QStringList &packa
return dict; return dict;
} }
PackageInfoVector QInstallerTools::createListOfRepositoryPackages(const QStringList &repositoryDirectories,
QStringList *packagesToFilter, FilterType filterType)
{
qDebug() << "Collecting information about available repository packages...";
bool ignoreInvalidRepositories = qApp->arguments().contains(QString::fromLatin1("--ignore-invalid-repositories"));
PackageInfoVector dict;
QFileInfoList entries;
foreach (const QString &repositoryDirectory, repositoryDirectories)
entries.append(QFileInfo(repositoryDirectory));
for (QFileInfoList::const_iterator it = entries.constBegin(); it != entries.constEnd(); ++it) {
qDebug() << "Process repository" << it->fileName();
QFile file(QString::fromLatin1("%1/Updates.xml").arg(it->filePath()));
QFileInfo fileInfo(file);
if (!fileInfo.exists()) {
if (ignoreInvalidRepositories) {
qDebug() << "- skip invalid repository";
continue;
}
throw QInstaller::Error(QString::fromLatin1("Repository \"%1\" does not contain a update "
"description (Updates.xml is missing).").arg(QDir::toNativeSeparators(it->fileName())));
}
if (!file.open(QIODevice::ReadOnly)) {
qDebug() << "Cannot open Updates.xml for reading:" << file.errorString();
continue;
}
QString error;
QDomDocument doc;
if (!doc.setContent(&file, &error)) {
qDebug().nospace() << "Cannot fetch a valid version of Updates.xml from repository "
<< it->fileName() << ": " << error;
continue;
}
file.close();
const QDomElement root = doc.documentElement();
if (root.tagName() != QLatin1String("Updates")) {
throw QInstaller::Error(QCoreApplication::translate("QInstaller",
"Invalid content in \"%1\".").arg(QDir::toNativeSeparators(file.fileName())));
}
const QDomNodeList children = root.childNodes();
for (int i = 0; i < children.count(); ++i) {
const QDomElement el = children.at(i).toElement();
if ((!el.isNull()) && (el.tagName() == QLatin1String("PackageUpdate"))) {
QInstallerTools::PackageInfo info;
QDomElement c1 = el.firstChildElement(QInstaller::scName);
if (!c1.isNull())
info.name = c1.text();
else
continue;
if (filterType == Exclude) {
// Check for current package in exclude list, if found, skip it
if (packagesToFilter->contains(info.name)) {
continue;
}
} else {
// Check for current package in include list, if not found, skip it
if (!packagesToFilter->contains(info.name))
continue;
}
c1 = el.firstChildElement(QInstaller::scVersion);
if (!c1.isNull())
info.version = c1.text();
else
continue;
info.directory = QString::fromLatin1("%1/%2").arg(it->filePath(), info.name);
info.metaFile = QString::fromLatin1("%1/%3%2").arg(info.directory,
QString::fromLatin1("meta.7z"), info.version);
const QDomNodeList c2 = el.childNodes();
for (int j = 0; j < c2.count(); ++j) {
if (c2.at(j).toElement().tagName() == QInstaller::scDependencies)
info.dependencies = c2.at(j).toElement().text()
.split(QInstaller::commaRegExp(), QString::SkipEmptyParts);
else if (c2.at(j).toElement().tagName() == QInstaller::scDownloadableArchives) {
QStringList names = c2.at(j).toElement().text()
.split(QInstaller::commaRegExp(), QString::SkipEmptyParts);
foreach (const QString &name, names) {
info.copiedFiles.append(QString::fromLatin1("%1/%3%2").arg(info.directory,
name, info.version));
info.copiedFiles.append(QString::fromLatin1("%1/%3%2.sha1").arg(info.directory,
name, info.version));
}
}
}
QString metaString;
{
QTextStream metaStream(&metaString);
el.save(metaStream, 0);
}
info.metaNode = metaString;
dict.push_back(info);
qDebug() << "- it provides the package" << info.name << " - " << info.version;
}
}
}
return dict;
}
QHash<QString, QString> QInstallerTools::buildPathToVersionMapping(const PackageInfoVector &info) QHash<QString, QString> QInstallerTools::buildPathToVersionMapping(const PackageInfoVector &info)
{ {
QHash<QString, QString> map; QHash<QString, QString> map;
@ -568,70 +697,83 @@ void QInstallerTools::copyComponentData(const QStringList &packageDirs, const QS
.arg(name)); .arg(name));
} }
QStringList compressedFiles; if (info.copiedFiles.isEmpty()) {
QStringList filesToCompress; QStringList compressedFiles;
foreach (const QString &packageDir, packageDirs) { QStringList filesToCompress;
const QDir dataDir(QString::fromLatin1("%1/%2/data").arg(packageDir, name)); foreach (const QString &packageDir, packageDirs) {
foreach (const QString &entry, dataDir.entryList(QDir::Dirs | QDir::NoDotAndDotDot | QDir::Files)) { const QDir dataDir(QString::fromLatin1("%1/%2/data").arg(packageDir, name));
QFileInfo fileInfo(dataDir.absoluteFilePath(entry)); foreach (const QString &entry, dataDir.entryList(QDir::Dirs | QDir::NoDotAndDotDot | QDir::Files)) {
if (fileInfo.isFile() && !fileInfo.isSymLink()) { QFileInfo fileInfo(dataDir.absoluteFilePath(entry));
const QString absoluteEntryFilePath = dataDir.absoluteFilePath(entry); if (fileInfo.isFile() && !fileInfo.isSymLink()) {
if (Lib7z::isSupportedArchive(absoluteEntryFilePath)) { const QString absoluteEntryFilePath = dataDir.absoluteFilePath(entry);
QFile tmp(absoluteEntryFilePath); if (Lib7z::isSupportedArchive(absoluteEntryFilePath)) {
QString target = QString::fromLatin1("%1/%3%2").arg(namedRepoDir, entry, info.version); QFile tmp(absoluteEntryFilePath);
qDebug() << "Copying archive from" << tmp.fileName() << "to" << target; QString target = QString::fromLatin1("%1/%3%2").arg(namedRepoDir, entry, info.version);
if (!tmp.copy(target)) { qDebug() << "Copying archive from" << tmp.fileName() << "to" << target;
throw QInstaller::Error(QString::fromLatin1("Cannot copy file \"%1\" to \"%2\": %3") if (!tmp.copy(target)) {
.arg(QDir::toNativeSeparators(tmp.fileName()), QDir::toNativeSeparators(target), tmp.errorString())); throw QInstaller::Error(QString::fromLatin1("Cannot copy file \"%1\" to \"%2\": %3")
.arg(QDir::toNativeSeparators(tmp.fileName()), QDir::toNativeSeparators(target), tmp.errorString()));
}
compressedFiles.append(target);
} else {
filesToCompress.append(absoluteEntryFilePath);
} }
} else if (fileInfo.isDir()) {
qDebug() << "Compressing data directory" << entry;
QString target = QString::fromLatin1("%1/%3%2.7z").arg(namedRepoDir, entry, info.version);
Lib7z::createArchive(target, QStringList() << dataDir.absoluteFilePath(entry),
Lib7z::QTmpFile::No);
compressedFiles.append(target); compressedFiles.append(target);
} else { } else if (fileInfo.isSymLink()) {
filesToCompress.append(absoluteEntryFilePath); filesToCompress.append(dataDir.absoluteFilePath(entry));
} }
} else if (fileInfo.isDir()) {
qDebug() << "Compressing data directory" << entry;
QString target = QString::fromLatin1("%1/%3%2.7z").arg(namedRepoDir, entry, info.version);
Lib7z::createArchive(target, QStringList() << dataDir.absoluteFilePath(entry),
Lib7z::QTmpFile::No);
compressedFiles.append(target);
} else if (fileInfo.isSymLink()) {
filesToCompress.append(dataDir.absoluteFilePath(entry));
} }
} }
}
if (!filesToCompress.isEmpty()) { if (!filesToCompress.isEmpty()) {
qDebug() << "Compressing files found in data directory:" << filesToCompress; qDebug() << "Compressing files found in data directory:" << filesToCompress;
QString target = QString::fromLatin1("%1/%3%2").arg(namedRepoDir, QLatin1String("content.7z"), QString target = QString::fromLatin1("%1/%3%2").arg(namedRepoDir, QLatin1String("content.7z"),
info.version); info.version);
Lib7z::createArchive(target, filesToCompress, Lib7z::QTmpFile::No); Lib7z::createArchive(target, filesToCompress, Lib7z::QTmpFile::No);
compressedFiles.append(target); compressedFiles.append(target);
} }
foreach (const QString &target, compressedFiles) { foreach (const QString &target, compressedFiles) {
(*infos)[i].copiedFiles.append(target); (*infos)[i].copiedFiles.append(target);
QFile archiveFile(target); QFile archiveFile(target);
QFile archiveHashFile(archiveFile.fileName() + QLatin1String(".sha1")); QFile archiveHashFile(archiveFile.fileName() + QLatin1String(".sha1"));
qDebug() << "Hash is stored in" << archiveHashFile.fileName(); qDebug() << "Hash is stored in" << archiveHashFile.fileName();
qDebug() << "Creating hash of archive" << archiveFile.fileName(); qDebug() << "Creating hash of archive" << archiveFile.fileName();
try { try {
QInstaller::openForRead(&archiveFile); QInstaller::openForRead(&archiveFile);
const QByteArray hashOfArchiveData = QInstaller::calculateHash(&archiveFile, const QByteArray hashOfArchiveData = QInstaller::calculateHash(&archiveFile,
QCryptographicHash::Sha1).toHex(); QCryptographicHash::Sha1).toHex();
archiveFile.close(); archiveFile.close();
QInstaller::openForWrite(&archiveHashFile); QInstaller::openForWrite(&archiveHashFile);
archiveHashFile.write(hashOfArchiveData); archiveHashFile.write(hashOfArchiveData);
qDebug() << "Generated sha1 hash:" << hashOfArchiveData; qDebug() << "Generated sha1 hash:" << hashOfArchiveData;
(*infos)[i].copiedFiles.append(archiveHashFile.fileName()); (*infos)[i].copiedFiles.append(archiveHashFile.fileName());
archiveHashFile.close(); archiveHashFile.close();
} catch (const QInstaller::Error &/*e*/) { } catch (const QInstaller::Error &/*e*/) {
archiveFile.close(); archiveFile.close();
archiveHashFile.close(); archiveHashFile.close();
throw; throw;
}
}
} else {
foreach (const QString &file, (*infos)[i].copiedFiles) {
QFileInfo fromInfo(file);
QFile from(file);
QString target = QString::fromLatin1("%1/%2").arg(namedRepoDir, fromInfo.fileName());
qDebug() << "Copying file from" << from.fileName() << "to" << target;
if (!from.copy(target)) {
throw QInstaller::Error(QString::fromLatin1("Cannot copy file \"%1\" to \"%2\": %3")
.arg(QDir::toNativeSeparators(from.fileName()), QDir::toNativeSeparators(target), from.errorString()));
}
} }
} }
} }

6
tools/common/repositorygen.h

@ -44,6 +44,8 @@ struct PackageInfo
QString directory; QString directory;
QStringList dependencies; QStringList dependencies;
QStringList copiedFiles; QStringList copiedFiles;
QString metaFile;
QString metaNode;
}; };
typedef QVector<PackageInfo> PackageInfoVector; typedef QVector<PackageInfo> PackageInfoVector;
@ -58,6 +60,10 @@ void copyWithException(const QString &source, const QString &target, const QStri
PackageInfoVector createListOfPackages(const QStringList &packagesDirectories, QStringList *packagesToFilter, PackageInfoVector createListOfPackages(const QStringList &packagesDirectories, QStringList *packagesToFilter,
FilterType ftype); FilterType ftype);
PackageInfoVector createListOfRepositoryPackages(const QStringList &repositoryDirectories, QStringList *packagesToFilter,
FilterType filterType);
QHash<QString, QString> buildPathToVersionMapping(const PackageInfoVector &info); QHash<QString, QString> buildPathToVersionMapping(const PackageInfoVector &info);
void compressMetaDirectories(const QString &repoDir, const QString &baseDir, void compressMetaDirectories(const QString &repoDir, const QString &baseDir,

30
tools/repogen/repogen.cpp

@ -95,6 +95,7 @@ int main(int argc, char** argv)
QStringList filteredPackages; QStringList filteredPackages;
bool updateExistingRepository = false; bool updateExistingRepository = false;
QStringList packagesDirectories; QStringList packagesDirectories;
QStringList repositoryDirectories;
QInstallerTools::FilterType filterType = QInstallerTools::Exclude; QInstallerTools::FilterType filterType = QInstallerTools::Exclude;
bool remove = false; bool remove = false;
bool updateExistingRepositoryWithNewComponents = false; bool updateExistingRepositoryWithNewComponents = false;
@ -156,6 +157,19 @@ int main(int argc, char** argv)
packagesDirectories.append(args.first()); packagesDirectories.append(args.first());
args.removeFirst(); args.removeFirst();
} else if (args.first() == QLatin1String("--repository")) {
args.removeFirst();
if (args.isEmpty()) {
return printErrorAndUsageAndExit(QCoreApplication::translate("QInstaller",
"Error: Repository parameter missing argument"));
}
if (!QFileInfo(args.first()).exists()) {
return printErrorAndUsageAndExit(QCoreApplication::translate("QInstaller",
"Error: Only local filesystem repositories now supported"));
}
repositoryDirectories.append(args.first());
args.removeFirst();
} else if (args.first() == QLatin1String("--ignore-translations") } else if (args.first() == QLatin1String("--ignore-translations")
|| args.first() == QLatin1String("--ignore-invalid-packages")) { || args.first() == QLatin1String("--ignore-invalid-packages")) {
args.removeFirst(); args.removeFirst();
@ -168,7 +182,7 @@ int main(int argc, char** argv)
} }
} }
if (packagesDirectories.isEmpty() || (args.count() != 1)) { if ((packagesDirectories.isEmpty() && repositoryDirectories.isEmpty()) || (args.count() != 1)) {
printUsage(); printUsage();
return 1; return 1;
} }
@ -190,8 +204,15 @@ int main(int argc, char** argv)
"Repository target directory \"%1\" already exists.").arg(QDir::toNativeSeparators(repositoryDir))); "Repository target directory \"%1\" already exists.").arg(QDir::toNativeSeparators(repositoryDir)));
} }
QInstallerTools::PackageInfoVector packages = QInstallerTools::createListOfPackages(packagesDirectories, QInstallerTools::PackageInfoVector packages;
QInstallerTools::PackageInfoVector precompressedPackages = QInstallerTools::createListOfRepositoryPackages(repositoryDirectories,
&filteredPackages, filterType); &filteredPackages, filterType);
packages.append(precompressedPackages);
QInstallerTools::PackageInfoVector preparedPackages = QInstallerTools::createListOfPackages(packagesDirectories,
&filteredPackages, filterType);
packages.append(preparedPackages);
if (updateExistingRepositoryWithNewComponents) { if (updateExistingRepositoryWithNewComponents) {
QDomDocument doc; QDomDocument doc;
@ -251,7 +272,10 @@ int main(int argc, char** argv)
QTemporaryDir tmp; QTemporaryDir tmp;
tmp.setAutoRemove(false); tmp.setAutoRemove(false);
tmpMetaDir = tmp.path(); tmpMetaDir = tmp.path();
QInstallerTools::copyComponentData(packagesDirectories, repositoryDir, &packages); QStringList directories;
directories.append(packagesDirectories);
directories.append(repositoryDirectories);
QInstallerTools::copyComponentData(directories, repositoryDir, &packages);
QInstallerTools::copyMetaData(tmpMetaDir, repositoryDir, packages, QLatin1String("{AnyApplication}"), QInstallerTools::copyMetaData(tmpMetaDir, repositoryDir, packages, QLatin1String("{AnyApplication}"),
QLatin1String(QUOTE(IFW_REPOSITORY_FORMAT_VERSION))); QLatin1String(QUOTE(IFW_REPOSITORY_FORMAT_VERSION)));
QInstallerTools::compressMetaDirectories(tmpMetaDir, tmpMetaDir, pathToVersionMapping); QInstallerTools::compressMetaDirectories(tmpMetaDir, tmpMetaDir, pathToVersionMapping);