Add support for brotli compression

Build logs on cache.nixos.org are compressed using Brotli (since this
allows them to be decompressed automatically by Chrome and Firefox),
so it's handy if "nix log" can decompress them.
This commit is contained in:
Eelco Dolstra 2017-03-13 14:40:15 +01:00
parent 73d7a51ee6
commit e8186085e0
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
7 changed files with 38 additions and 5 deletions

View file

@ -14,6 +14,7 @@ LIBLZMA_LIBS = @LIBLZMA_LIBS@
SQLITE3_LIBS = @SQLITE3_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@
bash = @bash@ bash = @bash@
bindir = @bindir@ bindir = @bindir@
bro = @bro@
datadir = @datadir@ datadir = @datadir@
datarootdir = @datarootdir@ datarootdir = @datarootdir@
docdir = @docdir@ docdir = @docdir@

View file

@ -128,6 +128,7 @@ NEED_PROG(gzip, gzip)
NEED_PROG(xz, xz) NEED_PROG(xz, xz)
AC_PATH_PROG(dot, dot) AC_PATH_PROG(dot, dot)
AC_PATH_PROG(pv, pv, pv) AC_PATH_PROG(pv, pv, pv)
NEED_PROG(bro, bro)
# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). # Test that Perl has the open/fork feature (Perl 5.8.0 and beyond).

View file

@ -24,7 +24,8 @@ let
inherit officialRelease; inherit officialRelease;
buildInputs = buildInputs =
[ curl bison flex perl libxml2 libxslt bzip2 xz [ curl bison flex perl libxml2 libxslt
bzip2 xz brotli
pkgconfig sqlite libsodium boehmgc pkgconfig sqlite libsodium boehmgc
docbook5 docbook5_xsl docbook5 docbook5_xsl
autoconf-archive autoconf-archive
@ -73,7 +74,10 @@ let
src = tarball; src = tarball;
buildInputs = buildInputs =
[ curl perl bzip2 xz openssl pkgconfig sqlite boehmgc ] [ curl perl
bzip2 xz brotli
openssl pkgconfig sqlite boehmgc
]
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
(aws-sdk-cpp.override { (aws-sdk-cpp.override {

View file

@ -6,7 +6,8 @@ with import <nixpkgs> {};
name = "nix"; name = "nix";
buildInputs = buildInputs =
[ curl bison flex perl libxml2 libxslt bzip2 xz [ curl bison flex perl libxml2 libxslt
bzip2 xz brotli
pkgconfig sqlite libsodium boehmgc pkgconfig sqlite libsodium boehmgc
docbook5 docbook5_xsl docbook5 docbook5_xsl
autoconf-archive autoconf-archive

View file

@ -5,6 +5,8 @@
#include "store-api.hh" #include "store-api.hh"
#include "archive.hh" #include "archive.hh"
#include "s3.hh" #include "s3.hh"
#include "compression.hh"
#ifdef ENABLE_S3 #ifdef ENABLE_S3
#include <aws/core/client/ClientConfiguration.h> #include <aws/core/client/ClientConfiguration.h>
#endif #endif
@ -70,6 +72,8 @@ struct CurlDownloader : public Downloader
struct curl_slist * requestHeaders = 0; struct curl_slist * requestHeaders = 0;
std::string encoding;
DownloadItem(CurlDownloader & downloader, const DownloadRequest & request) DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
: downloader(downloader), request(request) : downloader(downloader), request(request)
{ {
@ -127,6 +131,7 @@ struct CurlDownloader : public Downloader
auto ss = tokenizeString<vector<string>>(line, " "); auto ss = tokenizeString<vector<string>>(line, " ");
status = ss.size() >= 2 ? ss[1] : ""; status = ss.size() >= 2 ? ss[1] : "";
result.data = std::make_shared<std::string>(); result.data = std::make_shared<std::string>();
encoding = "";
} else { } else {
auto i = line.find(':'); auto i = line.find(':');
if (i != string::npos) { if (i != string::npos) {
@ -142,7 +147,8 @@ struct CurlDownloader : public Downloader
debug(format("shutting down on 200 HTTP response with expected ETag")); debug(format("shutting down on 200 HTTP response with expected ETag"));
return 0; return 0;
} }
} } else if (name == "content-encoding")
encoding = trim(string(line, i + 1));;
} }
} }
return realSize; return realSize;
@ -268,7 +274,18 @@ struct CurlDownloader : public Downloader
{ {
result.cached = httpStatus == 304; result.cached = httpStatus == 304;
done = true; done = true;
callSuccess(success, failure, const_cast<const DownloadResult &>(result));
/* Ad hoc support for brotli, since curl doesn't do
this yet. */
try {
if (encoding == "br")
result.data = decompress("br", *result.data);
callSuccess(success, failure, const_cast<const DownloadResult &>(result));
} catch (...) {
done = true;
callFailure(failure, std::current_exception());
}
} else { } else {
Error err = Error err =
(httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) ? NotFound : (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) ? NotFound :

View file

@ -89,6 +89,11 @@ static ref<std::string> decompressBzip2(const std::string & in)
} }
} }
static ref<std::string> decompressBrotli(const std::string & in)
{
return make_ref<std::string>(runProgram(BRO, true, {"-d"}, in));
}
ref<std::string> compress(const std::string & method, const std::string & in) ref<std::string> compress(const std::string & method, const std::string & in)
{ {
StringSink ssink; StringSink ssink;
@ -106,6 +111,8 @@ ref<std::string> decompress(const std::string & method, const std::string & in)
return decompressXZ(in); return decompressXZ(in);
else if (method == "bzip2") else if (method == "bzip2")
return decompressBzip2(in); return decompressBzip2(in);
else if (method == "br")
return decompressBrotli(in);
else else
throw UnknownCompressionMethod(format("unknown compression method %s") % method); throw UnknownCompressionMethod(format("unknown compression method %s") % method);
} }

View file

@ -9,3 +9,5 @@ libutil_SOURCES := $(wildcard $(d)/*.cc)
libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS)
libutil_LIBS = libformat libutil_LIBS = libformat
libutil_CXXFLAGS = -DBRO=\"$(bro)\"