Improve substitution progress indicator

E.g.

  $ nix build --store local?root=/tmp/nix nixpkgs.firefox-unwrapped
  [0/1 built, 1/97/98 fetched, 65.8/92.8 MiB DL, 203.2/309.2 MiB copied] downloading 'https://cache.nixos.org/nar/1czm9fk0svacy4h6a3fzkpafi4f7a9gml36kk8cq1igaghbspg3k.nar.xz'
This commit is contained in:
Eelco Dolstra 2017-08-14 20:14:55 +02:00
parent b29b6feaba
commit c36467ad2e
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
5 changed files with 93 additions and 86 deletions

View file

@ -9,6 +9,7 @@
#include "finally.hh" #include "finally.hh"
#include "compression.hh" #include "compression.hh"
#include "json.hh" #include "json.hh"
#include "nar-info.hh"
#include <algorithm> #include <algorithm>
#include <iostream> #include <iostream>
@ -201,6 +202,16 @@ struct Child
}; };
template<typename T>
struct MaintainCount
{
T & counter;
T delta;
MaintainCount(T & counter, T delta) : counter(counter), delta(delta) { counter += delta; }
~MaintainCount() { counter -= delta; }
};
/* The worker class. */ /* The worker class. */
class Worker class Worker
{ {
@ -244,6 +255,8 @@ private:
public: public:
const Activity act;
/* Set if at least one derivation had a BuildError (i.e. permanent /* Set if at least one derivation had a BuildError (i.e. permanent
failure). */ failure). */
bool permanentFailure; bool permanentFailure;
@ -255,6 +268,11 @@ public:
std::unique_ptr<HookInstance> hook; std::unique_ptr<HookInstance> hook;
uint64_t expectedDownloadSize = 0;
uint64_t doneDownloadSize = 0;
uint64_t expectedNarSize = 0;
uint64_t doneNarSize = 0;
Worker(LocalStore & store); Worker(LocalStore & store);
~Worker(); ~Worker();
@ -313,6 +331,12 @@ public:
bool pathContentsGood(const Path & path); bool pathContentsGood(const Path & path);
void markContentsGood(const Path & path); void markContentsGood(const Path & path);
void updateProgress()
{
logger->event(evSetExpected, act, actDownload, expectedDownloadSize + doneDownloadSize);
logger->event(evSetExpected, act, actCopyPath, expectedNarSize + doneNarSize);
}
}; };
@ -3304,6 +3328,8 @@ private:
storePath when doing a repair. */ storePath when doing a repair. */
Path destPath; Path destPath;
std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedNar, maintainExpectedDownload;
typedef void (SubstitutionGoal::*GoalState)(); typedef void (SubstitutionGoal::*GoalState)();
GoalState state; GoalState state;
@ -3430,6 +3456,18 @@ void SubstitutionGoal::tryNext()
return; return;
} }
/* Update the total expected download size. */
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, narInfo->narSize);
maintainExpectedDownload =
narInfo && narInfo->fileSize
? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize)
: nullptr;
worker.updateProgress();
hasSubstitute = true; hasSubstitute = true;
/* Bail out early if this substituter lacks a valid /* Bail out early if this substituter lacks a valid
@ -3538,6 +3576,17 @@ void SubstitutionGoal::finished()
printMsg(lvlChatty, printMsg(lvlChatty,
format("substitution of path '%1%' succeeded") % storePath); format("substitution of path '%1%' succeeded") % storePath);
if (maintainExpectedDownload) {
auto fileSize = maintainExpectedDownload->delta;
maintainExpectedDownload.reset();
worker.doneDownloadSize += fileSize;
}
worker.doneNarSize += maintainExpectedNar->delta;
maintainExpectedNar.reset();
worker.updateProgress();
amDone(ecSuccess); amDone(ecSuccess);
} }
@ -3560,7 +3609,8 @@ static bool working = false;
Worker::Worker(LocalStore & store) Worker::Worker(LocalStore & store)
: store(store) : act(actRealise)
, store(store)
{ {
/* Debugging: prevent recursive workers. */ /* Debugging: prevent recursive workers. */
if (working) abort(); if (working) abort();
@ -3581,6 +3631,9 @@ Worker::~Worker()
are in trouble, since goals may call childTerminated() etc. in are in trouble, since goals may call childTerminated() etc. in
their destructors). */ their destructors). */
topGoals.clear(); topGoals.clear();
assert(expectedDownloadSize == 0);
assert(expectedNarSize == 0);
} }

View file

@ -83,12 +83,12 @@ struct CurlDownloader : public Downloader
std::string encoding; std::string encoding;
DownloadItem(CurlDownloader & downloader, const DownloadRequest & request) DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
: downloader(downloader), request(request) : downloader(downloader)
, request(request)
, act(actDownload, fmt("downloading '%s'", request.uri))
{ {
if (!request.expectedETag.empty()) if (!request.expectedETag.empty())
requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
logger->event(evDownloadCreated, act, request.uri);
} }
~DownloadItem() ~DownloadItem()
@ -105,7 +105,6 @@ struct CurlDownloader : public Downloader
} catch (...) { } catch (...) {
ignoreException(); ignoreException();
} }
logger->event(evDownloadDestroyed, act);
} }
template<class T> template<class T>
@ -168,7 +167,7 @@ struct CurlDownloader : public Downloader
int progressCallback(double dltotal, double dlnow) int progressCallback(double dltotal, double dlnow)
{ {
logger->event(evDownloadProgress, act, dltotal, dlnow); act.progress(dlnow, dltotal);
return _isInterrupted; return _isInterrupted;
} }
@ -267,7 +266,7 @@ struct CurlDownloader : public Downloader
try { try {
result.data = decodeContent(encoding, ref<std::string>(result.data)); result.data = decodeContent(encoding, ref<std::string>(result.data));
callSuccess(success, failure, const_cast<const DownloadResult &>(result)); callSuccess(success, failure, const_cast<const DownloadResult &>(result));
logger->event(evDownloadSucceeded, act, result.data->size()); act.progress(result.data->size(), result.data->size());
} catch (...) { } catch (...) {
done = true; done = true;
callFailure(failure, std::current_exception()); callFailure(failure, std::current_exception());

View file

@ -16,6 +16,8 @@ typedef enum {
typedef enum { typedef enum {
actUnknown = 0, actUnknown = 0,
actCopyPath = 100, actCopyPath = 100,
actDownload = 101,
actRealise = 102,
} ActivityType; } ActivityType;
class Activity class Activity
@ -38,10 +40,6 @@ typedef enum {
evBuildStarted = 1, evBuildStarted = 1,
evBuildOutput = 2, evBuildOutput = 2,
evBuildFinished = 3, evBuildFinished = 3,
evDownloadCreated = 4,
evDownloadDestroyed = 5,
evDownloadProgress = 6,
evDownloadSucceeded = 7,
evSubstitutionCreated = 8, evSubstitutionCreated = 8,
evSubstitutionStarted = 9, evSubstitutionStarted = 9,
evSubstitutionFinished = 10, evSubstitutionFinished = 10,

View file

@ -229,7 +229,7 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables
PathSet InstallablesCommand::toStorePaths(ref<Store> store, ToStorePathsMode mode) PathSet InstallablesCommand::toStorePaths(ref<Store> store, ToStorePathsMode mode)
{ {
if (mode != DryRun) if (mode != Build)
settings.readOnlyMode = true; settings.readOnlyMode = true;
PathSet outPaths, buildables; PathSet outPaths, buildables;

View file

@ -25,14 +25,6 @@ private:
std::map<ActivityType, uint64_t> expectedByType; std::map<ActivityType, uint64_t> expectedByType;
}; };
struct DownloadInfo
{
std::string uri;
uint64_t current = 0;
uint64_t expected = 0;
DownloadInfo(const std::string & uri) : uri(uri) { }
};
struct CopyInfo struct CopyInfo
{ {
uint64_t expected = 0; uint64_t expected = 0;
@ -58,9 +50,6 @@ private:
std::set<Activity::t> runningSubstitutions; std::set<Activity::t> runningSubstitutions;
uint64_t succeededSubstitutions = 0; uint64_t succeededSubstitutions = 0;
uint64_t downloadedBytes = 0; // finished downloads
std::map<Activity::t, DownloadInfo> downloads;
std::map<Activity::t, CopyInfo> runningCopies; std::map<Activity::t, CopyInfo> runningCopies;
std::list<ActInfo> activities; std::list<ActInfo> activities;
@ -155,10 +144,16 @@ public:
if (!state.activities.empty()) { if (!state.activities.empty()) {
if (!status.empty()) line += " "; if (!status.empty()) line += " ";
auto i = state.activities.rbegin(); auto i = state.activities.rbegin();
line += i->s;
if (!i->s2.empty()) { while (i != state.activities.rend() && i->s.empty() && i->s2.empty())
line += ": "; ++i;
line += i->s2;
if (i != state.activities.rend()) {
line += i->s;
if (!i->s2.empty()) {
line += ": ";
line += i->s2;
}
} }
} }
@ -172,9 +167,13 @@ public:
std::string res; std::string res;
if (state.failedBuilds) { auto add = [&](const std::string & s) {
if (!res.empty()) res += ", "; if (!res.empty()) res += ", ";
res += fmt(ANSI_RED "%d failed" ANSI_NORMAL, state.failedBuilds); res += s;
};
if (state.failedBuilds) {
add(fmt(ANSI_RED "%d failed" ANSI_NORMAL, state.failedBuilds));
} }
if (!state.builds.empty() || state.succeededBuilds) if (!state.builds.empty() || state.succeededBuilds)
@ -195,39 +194,31 @@ public:
state.succeededSubstitutions + state.substitutions.size()); state.succeededSubstitutions + state.substitutions.size());
} }
if (!state.downloads.empty() || state.downloadedBytes) {
if (!res.empty()) res += ", ";
uint64_t expected = state.downloadedBytes, current = state.downloadedBytes;
for (auto & i : state.downloads) {
expected += i.second.expected;
current += i.second.current;
}
res += fmt("%1$.0f/%2$.0f KiB", current / 1024.0, expected / 1024.0);
}
if (!state.runningCopies.empty()) { if (!state.runningCopies.empty()) {
if (!res.empty()) res += ", ";
uint64_t copied = 0, expected = 0; uint64_t copied = 0, expected = 0;
for (auto & i : state.runningCopies) { for (auto & i : state.runningCopies) {
copied += i.second.copied; copied += i.second.copied;
expected += i.second.expected - (i.second.done - i.second.copied); expected += i.second.expected - (i.second.done - i.second.copied);
} }
res += fmt("%d/%d copied", copied, expected); add(fmt("%d/%d copied", copied, expected));
} }
auto & act = state.activitiesByType[actCopyPath]; auto showActivity = [&](ActivityType type, const std::string & f) {
uint64_t done = act.done, expected = act.done; auto & act = state.activitiesByType[type];
for (auto & j : act.its) { uint64_t done = act.done, expected = act.done;
done += j.second->done; for (auto & j : act.its) {
expected += j.second->expected; done += j.second->done;
} expected += j.second->expected;
}
expected = std::max(expected, act.expected); expected = std::max(expected, act.expected);
if (done || expected) { if (done || expected)
if (!res.empty()) res += ", "; add(fmt(f, done / MiB, expected / MiB));
res += fmt("%1$.1f/%2$.1f MiB copied", done / MiB, expected / MiB); };
}
showActivity(actDownload, "%1$.1f/%2$.1f MiB DL");
showActivity(actCopyPath, "%1$.1f/%2$.1f MiB copied");
return res; return res;
} }
@ -318,40 +309,6 @@ public:
deleteActivity(*state, act); deleteActivity(*state, act);
} }
if (ev.type == evDownloadCreated) {
Activity::t act = ev.getI(0);
std::string uri = ev.getS(1);
state->downloads.emplace(act, DownloadInfo{uri});
if (state->runningSubstitutions.empty()) // FIXME: hack
createActivity(*state, act, fmt("downloading " ANSI_BOLD "%s" ANSI_NORMAL "", uri));
}
if (ev.type == evDownloadProgress) {
Activity::t act = ev.getI(0);
auto i = state->downloads.find(act);
assert(i != state->downloads.end());
i->second.expected = ev.getI(1);
i->second.current = ev.getI(2);
}
if (ev.type == evDownloadSucceeded) {
Activity::t act = ev.getI(0);
auto i = state->downloads.find(act);
assert(i != state->downloads.end());
state->downloadedBytes += ev.getI(1);
state->downloads.erase(i);
deleteActivity(*state, act);
}
if (ev.type == evDownloadDestroyed) {
Activity::t act = ev.getI(0);
auto i = state->downloads.find(act);
if (i != state->downloads.end()) {
state->downloads.erase(i);
deleteActivity(*state, act);
}
}
if (ev.type == evCopyProgress) { if (ev.type == evCopyProgress) {
Activity::t act = ev.getI(0); Activity::t act = ev.getI(0);
auto & i = state->runningCopies[act]; auto & i = state->runningCopies[act];