Add automatic garbage collection

Nix can now automatically run the garbage collector during builds or
while adding paths to the store. The option "min-free = <bytes>"
specifies that Nix should run the garbage collector whenever free
space in the Nix store drops below <bytes>. It will then delete
garbage until "max-free" bytes are available.

Garbage collection during builds is asynchronous; running builds are
not paused and new builds are not blocked. However, there also is a
synchronous GC run prior to the first build/substitution.

Currently, no old GC roots are deleted (as in "nix-collect-garbage
-d").
This commit is contained in:
Eelco Dolstra 2017-09-05 20:43:42 +02:00
parent b932ea58ec
commit 0b606aad46
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
6 changed files with 127 additions and 1 deletions

View file

@ -393,6 +393,10 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"
package repository.</para> package repository.</para>
</listitem> </listitem>
<listitem>
<para>Automatic garbage collection.</para>
</listitem>
</itemizedlist> </itemizedlist>
<para>This release has contributions from TBD.</para> <para>This release has contributions from TBD.</para>

View file

@ -3957,6 +3957,8 @@ void Worker::run(const Goals & _topGoals)
checkInterrupt(); checkInterrupt();
store.autoGC(false);
/* Call every wake goal (in the ordering established by /* Call every wake goal (in the ordering established by
CompareGoalPtrs). */ CompareGoalPtrs). */
while (!awake.empty() && !topGoals.empty()) { while (!awake.empty() && !topGoals.empty()) {
@ -4014,6 +4016,9 @@ void Worker::waitForInput()
is a build timeout, then wait for input until the first is a build timeout, then wait for input until the first
deadline for any child. */ deadline for any child. */
auto nearest = steady_time_point::max(); // nearest deadline auto nearest = steady_time_point::max(); // nearest deadline
if (settings.minFree.get() != 0)
// Periodicallty wake up to see if we need to run the garbage collector.
nearest = before + std::chrono::seconds(10);
for (auto & i : children) { for (auto & i : children) {
if (!i.respectTimeouts) continue; if (!i.respectTimeouts) continue;
if (0 != settings.maxSilentTime) if (0 != settings.maxSilentTime)

View file

@ -1,6 +1,7 @@
#include "derivations.hh" #include "derivations.hh"
#include "globals.hh" #include "globals.hh"
#include "local-store.hh" #include "local-store.hh"
#include "finally.hh"
#include <functional> #include <functional>
#include <queue> #include <queue>
@ -9,6 +10,7 @@
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/statvfs.h>
#include <errno.h> #include <errno.h>
#include <fcntl.h> #include <fcntl.h>
#include <unistd.h> #include <unistd.h>
@ -845,4 +847,72 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
} }
void LocalStore::autoGC(bool sync)
{
auto getAvail = [this]() {
struct statvfs st;
if (statvfs(realStoreDir.c_str(), &st))
throw SysError("getting filesystem info about '%s'", realStoreDir);
return (uint64_t) st.f_bavail * st.f_bsize;
};
std::shared_future<void> future;
{
auto state(_state.lock());
if (state->gcRunning) {
future = state->gcFuture;
debug("waiting for auto-GC to finish");
goto sync;
}
auto now = std::chrono::steady_clock::now();
if (now < state->lastGCCheck + std::chrono::seconds(5)) return;
auto avail = getAvail();
state->lastGCCheck = now;
if (avail >= settings.minFree || avail >= settings.maxFree) return;
if (avail > state->availAfterGC * 0.97) return;
state->gcRunning = true;
std::promise<void> promise;
future = state->gcFuture = promise.get_future().share();
std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable {
/* Wake up any threads waiting for the auto-GC to finish. */
Finally wakeup([&]() {
auto state(_state.lock());
state->gcRunning = false;
state->lastGCCheck = std::chrono::steady_clock::now();
promise.set_value();
});
printInfo("running auto-GC to free %d bytes", settings.maxFree - avail);
GCOptions options;
options.maxFreed = settings.maxFree - avail;
GCResults results;
collectGarbage(options, results);
_state.lock()->availAfterGC = getAvail();
}).detach();
}
sync:
// Wait for the future outside of the state lock.
if (sync) future.get();
}
} }

View file

@ -4,8 +4,9 @@
#include "config.hh" #include "config.hh"
#include <map> #include <map>
#include <sys/types.h> #include <limits>
#include <sys/types.h>
namespace nix { namespace nix {
@ -342,6 +343,13 @@ public:
Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors", Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
"A list of servers used by builtins.fetchurl to fetch files by hash."}; "A list of servers used by builtins.fetchurl to fetch files by hash."};
Setting<uint64_t> minFree{this, 0, "min-free",
"Automatically run the garbage collector when free disk space drops below the specified amount."};
Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free",
"Stop deleting garbage when free disk space is above the specified amount."};
}; };

View file

@ -244,6 +244,18 @@ LocalStore::LocalStore(const Params & params)
LocalStore::~LocalStore() LocalStore::~LocalStore()
{ {
std::shared_future<void> future;
{
auto state(_state.lock());
if (state->gcRunning)
future = state->gcFuture;
}
if (future.valid()) {
printError("waiting for auto-GC to finish on exit...");
future.get();
}
try { try {
auto state(_state.lock()); auto state(_state.lock());
@ -991,6 +1003,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
StringSource source(*nar); StringSource source(*nar);
restorePath(realPath, source); restorePath(realPath, source);
autoGC();
canonicalisePathMetaData(realPath, -1); canonicalisePathMetaData(realPath, -1);
optimisePath(realPath); // FIXME: combine with hashPath() optimisePath(realPath); // FIXME: combine with hashPath()
@ -1025,6 +1039,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
deletePath(realPath); deletePath(realPath);
autoGC();
if (recursive) { if (recursive) {
StringSource source(dump); StringSource source(dump);
restorePath(realPath, source); restorePath(realPath, source);
@ -1097,6 +1113,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
deletePath(realPath); deletePath(realPath);
autoGC();
writeFile(realPath, s); writeFile(realPath, s);
canonicalisePathMetaData(realPath, -1); canonicalisePathMetaData(realPath, -1);

View file

@ -7,6 +7,8 @@
#include "sync.hh" #include "sync.hh"
#include "util.hh" #include "util.hh"
#include <chrono>
#include <future>
#include <string> #include <string>
#include <unordered_set> #include <unordered_set>
@ -60,6 +62,21 @@ private:
/* The file to which we write our temporary roots. */ /* The file to which we write our temporary roots. */
AutoCloseFD fdTempRoots; AutoCloseFD fdTempRoots;
/* The last time we checked whether to do an auto-GC, or an
auto-GC finished. */
std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
/* Whether auto-GC is running. If so, get gcFuture to wait for
the GC to finish. */
bool gcRunning = false;
std::shared_future<void> gcFuture;
/* How much disk space was available after the previous
auto-GC. If the current available disk space is below
minFree but not much below availAfterGC, then there is no
point in starting a new GC. */
uint64_t availAfterGC = std::numeric_limits<uint64_t>::max();
}; };
Sync<State, std::recursive_mutex> _state; Sync<State, std::recursive_mutex> _state;
@ -196,6 +213,10 @@ public:
void addSignatures(const Path & storePath, const StringSet & sigs) override; void addSignatures(const Path & storePath, const StringSet & sigs) override;
/* If free disk space in /nix/store if below minFree, delete
garbage until it exceeds maxFree. */
void autoGC(bool sync = true);
private: private:
int getSchema(); int getSchema();