picnoir
/
nix-gl-host
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Hot Cache: use the DSO last write time/size instead of content hash 

After profiling a nixglhost hot run, it turns out that we were
spending more than 98% of the run time reading and sha256-hashing
files.

Let's give up on content hashing the files and assume that using their
name, size and last write time is good enough.

On a hot run, we reduce the run time from about 3s to 0.3s on a
nvme-powered ryzen 7 desktop.

I guess this 10x speedup probably worth the little cache corectness we
lose on the way.
This commit is contained in:
Félix Baylac Jacqué 2022-12-14 17:44:11 +01:00
parent 61a5fcdae8
commit 3ff2f01812
5 changed files with 82 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
src/nixglhost_wrapper.py
View File

@ -14,7 +14,7 @@ from glob import glob
from typing import List, Literal, Dict, Tuple, TypedDict, TextIO, Optional from typing import List, Literal, Dict, Tuple, TypedDict, TextIO, Optional
IN_NIX_STORE = False IN_NIX_STORE = False
CACHE_VERSION = 2 CACHE_VERSION = 3
if IN_NIX_STORE: if IN_NIX_STORE:
@ -30,47 +30,55 @@ class ResolvedLib:
together with some metadata helping us to uniquely identify it.""" together with some metadata helping us to uniquely identify it."""
def __init__( def __init__(
self, name: str, dirpath: str, fullpath: str, sha256: Optional[str] = None self,
name: str,
dirpath: str,
fullpath: str,
last_modification: Optional[float] = None,
size: Optional[int] = None,
): ):
self.name: str = name self.name: str = name
self.dirpath: str = dirpath self.dirpath: str = dirpath
self.fullpath: str = fullpath self.fullpath: str = fullpath
if sha256 is None: if size is None or last_modification is None:
h = hashlib.sha256() stat = os.stat(fullpath)
with open(fullpath, "rb") as f: self.last_modification: float = stat.st_atime
h.update(f.read()) self.size: int = stat.st_size
sha: str = h.hexdigest()
else: else:
sha = sha256 self.last_modification = last_modification
self.sha256: str = sha self.size = size
def __repr__(self): def __repr__(self):
return ( return f"ResolvedLib<{self.name}, {self.dirpath}, {self.fullpath}, {self.last_modification}, {self.size}>"
f"ResolvedLib<{self.name}, {self.dirpath}, {self.fullpath}, {self.sha256}>"
)
def to_dict(self) -> Dict: def to_dict(self) -> Dict:
return { return {
"name": self.name, "name": self.name,
"dirpath": self.dirpath, "dirpath": self.dirpath,
"fullpath": self.fullpath, "fullpath": self.fullpath,
"sha256": self.sha256, "last_modification": self.last_modification,
"size": self.size,
} }
def __hash__(self): def __hash__(self):
return hash((self.name, self.dirpath, self.fullpath, self.sha256)) return hash(
(self.name, self.dirpath, self.fullpath, self.last_modification, self.size)
)
def __eq__(self, o): def __eq__(self, o):
return ( return (
self.name == o.name self.name == o.name
and self.fullpath == o.fullpath and self.fullpath == o.fullpath
and self.sha256 == o.sha256
and self.dirpath == o.dirpath and self.dirpath == o.dirpath
and self.last_modification == o.last_modification
and self.size == o.size
) )
@classmethod @classmethod
def from_dict(cls, d: Dict): def from_dict(cls, d: Dict):
return ResolvedLib(d["name"], d["dirpath"], d["fullpath"], d["sha256"]) return ResolvedLib(
d["name"], d["dirpath"], d["fullpath"], d["last_modification"], d["size"]
)
class LibraryPath: class LibraryPath:
@ -374,8 +382,8 @@ def is_dso_cache_up_to_date(dsos: CacheDirContent, cache_file_path: str) -> bool
We keep what's in the cache through a JSON file stored at the root We keep what's in the cache through a JSON file stored at the root
of the cache_dir. We consider a dynamically shared object to be up of the cache_dir. We consider a dynamically shared object to be up
to date if its name, its full path and its content sha256 are to date if its name, its full path, its size and last modification
equivalent.""" timestamp are equivalent."""
log_info("Checking if the cache is up to date") log_info("Checking if the cache is up to date")
if os.path.isfile(cache_file_path): if os.path.isfile(cache_file_path):
with open(cache_file_path, "r", encoding="utf8") as f: with open(cache_file_path, "r", encoding="utf8") as f:

18
src/nixglhost_wrapper_test.py
View File

@ -9,18 +9,12 @@ class TestCacheSerializer(unittest.TestCase):
lp = LibraryPath( lp = LibraryPath(
glx=[ glx=[
ResolvedLib( ResolvedLib(
"dummyglx.so", "dummyglx.so", "/lib", "/lib/dummyglx.so", 1670260550.481498, 1612
"/lib",
"/lib/dummyglx.so",
"031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406",
) )
], ],
cuda=[ cuda=[
ResolvedLib( ResolvedLib(
"dummycuda.so", "dummycuda.so", "/lib", "/lib/dummycuda.so", 2670260550.481498, 2612
"/lib",
"/lib/dummycuda.so",
"031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407",
) )
], ],
generic=[ generic=[
@ -28,15 +22,13 @@ class TestCacheSerializer(unittest.TestCase):
"dummygeneric.so", "dummygeneric.so",
"/lib", "/lib",
"/lib/dummygeneric.so", "/lib/dummygeneric.so",
"031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408", 3670260550.481498,
3612,
) )
], ],
egl=[ egl=[
ResolvedLib( ResolvedLib(
"dummyegl.so", "dummyegl.so", "/lib", "/lib/dummyegl.so", 4670260550.481498, 4612
"/lib",
"/lib/dummyegl.so",
"031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409",
) )
], ],
path="/path/to/lib/dir", path="/path/to/lib/dir",

26
tests/fixtures/json_permut/1.json vendored
View File

@ -6,7 +6,8 @@
"name": "dummyglx.so", "name": "dummyglx.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummyglx.so", "fullpath": "/lib/dummyglx.so",
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406" "last_modification": 1670260550.481498,
"size": 1612
} }
], ],
"cuda": [ "cuda": [
@ -14,7 +15,8 @@
"name": "dummycuda.so", "name": "dummycuda.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummycuda.so", "fullpath": "/lib/dummycuda.so",
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407" "last_modification": 2670260550.481498,
"size": 2612
} }
], ],
"generic": [ "generic": [
@ -22,7 +24,8 @@
"name": "dummygeneric.so", "name": "dummygeneric.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummygeneric.so", "fullpath": "/lib/dummygeneric.so",
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408" "last_modification": 3670260550.481498,
"size": 3612
} }
], ],
"egl": [ "egl": [
@ -30,7 +33,8 @@
"name": "dummyegl.so", "name": "dummyegl.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummyegl.so", "fullpath": "/lib/dummyegl.so",
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409" "last_modification": 4670260550.481498,
"size": 4612
} }
], ],
"path": "/path/to/lib/dir" "path": "/path/to/lib/dir"
@ -41,7 +45,8 @@
"name": "dummyglx.so", "name": "dummyglx.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummyglx.so", "fullpath": "/lib/dummyglx.so",
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406" "last_modification": 5670260550.481498,
"size": 5612
} }
], ],
"cuda": [ "cuda": [
@ -49,7 +54,8 @@
"name": "dummycuda.so", "name": "dummycuda.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummycuda.so", "fullpath": "/lib/dummycuda.so",
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407" "last_modification": 6670260550.481498,
"size": 6612
} }
], ],
"generic": [ "generic": [
@ -57,7 +63,8 @@
"name": "dummygeneric.so", "name": "dummygeneric.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummygeneric.so", "fullpath": "/lib/dummygeneric.so",
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408" "last_modification": 7670260550.481498,
"size": 7612
} }
], ],
"egl": [ "egl": [
@ -65,11 +72,12 @@
"name": "dummyegl.so", "name": "dummyegl.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummyegl.so", "fullpath": "/lib/dummyegl.so",
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409" "last_modification": 8670260550.481498,
"size": 8612
} }
], ],
"path": "/path/to/lib/dir2" "path": "/path/to/lib/dir2"
} }
], ],
"version": 2 "version": 3
} }

26
tests/fixtures/json_permut/2.json vendored
View File

@ -6,7 +6,8 @@
"name": "dummyglx.so", "name": "dummyglx.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummyglx.so", "fullpath": "/lib/dummyglx.so",
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406" "last_modification": 5670260550.481498,
"size": 5612
} }
], ],
"cuda": [ "cuda": [
@ -14,7 +15,8 @@
"name": "dummycuda.so", "name": "dummycuda.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummycuda.so", "fullpath": "/lib/dummycuda.so",
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407" "last_modification": 6670260550.481498,
"size": 6612
} }
], ],
"generic": [ "generic": [
@ -22,7 +24,8 @@
"name": "dummygeneric.so", "name": "dummygeneric.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummygeneric.so", "fullpath": "/lib/dummygeneric.so",
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408" "last_modification": 7670260550.481498,
"size": 7612
} }
], ],
"egl": [ "egl": [
@ -30,7 +33,8 @@
"name": "dummyegl.so", "name": "dummyegl.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummyegl.so", "fullpath": "/lib/dummyegl.so",
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409" "last_modification": 8670260550.481498,
"size": 8612
} }
], ],
"path": "/path/to/lib/dir2" "path": "/path/to/lib/dir2"
@ -41,7 +45,8 @@
"name": "dummyglx.so", "name": "dummyglx.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummyglx.so", "fullpath": "/lib/dummyglx.so",
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406" "last_modification": 1670260550.481498,
"size": 1612
} }
], ],
"cuda": [ "cuda": [
@ -49,7 +54,8 @@
"name": "dummycuda.so", "name": "dummycuda.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummycuda.so", "fullpath": "/lib/dummycuda.so",
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407" "last_modification": 2670260550.481498,
"size": 2612
} }
], ],
"generic": [ "generic": [
@ -57,7 +63,8 @@
"name": "dummygeneric.so", "name": "dummygeneric.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummygeneric.so", "fullpath": "/lib/dummygeneric.so",
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408" "last_modification": 3670260550.481498,
"size": 3612
} }
], ],
"egl": [ "egl": [
@ -65,11 +72,12 @@
"name": "dummyegl.so", "name": "dummyegl.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummyegl.so", "fullpath": "/lib/dummyegl.so",
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409" "last_modification": 4670260550.481498,
"size": 4612
} }
], ],
"path": "/path/to/lib/dir" "path": "/path/to/lib/dir"
} }
], ],
"version": 2 "version": 3
} }

26
tests/fixtures/json_permut/not-equal.json vendored
View File

@ -6,7 +6,8 @@
"name": "dummyglx.so", "name": "dummyglx.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummyglx.so", "fullpath": "/lib/dummyglx.so",
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406" "last_modification": 2670260550.481498,
"size": 2612
} }
], ],
"cuda": [ "cuda": [
@ -14,7 +15,8 @@
"name": "dummycuda.so", "name": "dummycuda.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummycuda.so", "fullpath": "/lib/dummycuda.so",
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407" "last_modification": 2670260550.481498,
"size": 2612
} }
], ],
"generic": [ "generic": [
@ -22,7 +24,8 @@
"name": "dummygeneric.so", "name": "dummygeneric.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummygeneric.so", "fullpath": "/lib/dummygeneric.so",
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408" "last_modification": 3670260550.481498,
"size": 3612
} }
], ],
"egl": [ "egl": [
@ -30,7 +33,8 @@
"name": "dummyegl.so", "name": "dummyegl.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummyegl.so", "fullpath": "/lib/dummyegl.so",
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409" "last_modification": 4670260550.481498,
"size": 4612
} }
], ],
"path": "/path/to/lib/dir" "path": "/path/to/lib/dir"
@ -41,7 +45,8 @@
"name": "dummyglx.so", "name": "dummyglx.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummyglx.so", "fullpath": "/lib/dummyglx.so",
"sha256": "4444444444444444444444444444444444444444444444444444444444444444" "last_modification": 5670260550.481498,
"size": 5612
} }
], ],
"cuda": [ "cuda": [
@ -49,7 +54,8 @@
"name": "dummycuda.so", "name": "dummycuda.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummycuda.so", "fullpath": "/lib/dummycuda.so",
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407" "last_modification": 6670260550.481498,
"size": 6612
} }
], ],
"generic": [ "generic": [
@ -57,7 +63,8 @@
"name": "dummygeneric.so", "name": "dummygeneric.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummygeneric.so", "fullpath": "/lib/dummygeneric.so",
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408" "last_modification": 7670260550.481498,
"size": 7612
} }
], ],
"egl": [ "egl": [
@ -65,11 +72,12 @@
"name": "dummyegl.so", "name": "dummyegl.so",
"dirpath": "/lib", "dirpath": "/lib",
"fullpath": "/lib/dummyegl.so", "fullpath": "/lib/dummyegl.so",
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409" "last_modification": 8670260550.481498,
"size": 8612
} }
], ],
"path": "/path/to/lib/dir2" "path": "/path/to/lib/dir2"
} }
], ],
"version": 2 "version": 3
} }