Hot Cache: use the DSO last write time/size instead of content hash
After profiling a nixglhost hot run, it turns out that we were spending more than 98% of the run time reading and sha256-hashing files. Let's give up on content hashing the files and assume that using their name, size and last write time is good enough. On a hot run, we reduce the run time from about 3s to 0.3s on a nvme-powered ryzen 7 desktop. I guess this 10x speedup probably worth the little cache corectness we lose on the way.
This commit is contained in:
parent
61a5fcdae8
commit
3ff2f01812
|
|
@ -14,7 +14,7 @@ from glob import glob
|
|||
from typing import List, Literal, Dict, Tuple, TypedDict, TextIO, Optional
|
||||
|
||||
IN_NIX_STORE = False
|
||||
CACHE_VERSION = 2
|
||||
CACHE_VERSION = 3
|
||||
|
||||
|
||||
if IN_NIX_STORE:
|
||||
|
|
@ -30,47 +30,55 @@ class ResolvedLib:
|
|||
together with some metadata helping us to uniquely identify it."""
|
||||
|
||||
def __init__(
|
||||
self, name: str, dirpath: str, fullpath: str, sha256: Optional[str] = None
|
||||
self,
|
||||
name: str,
|
||||
dirpath: str,
|
||||
fullpath: str,
|
||||
last_modification: Optional[float] = None,
|
||||
size: Optional[int] = None,
|
||||
):
|
||||
self.name: str = name
|
||||
self.dirpath: str = dirpath
|
||||
self.fullpath: str = fullpath
|
||||
if sha256 is None:
|
||||
h = hashlib.sha256()
|
||||
with open(fullpath, "rb") as f:
|
||||
h.update(f.read())
|
||||
sha: str = h.hexdigest()
|
||||
if size is None or last_modification is None:
|
||||
stat = os.stat(fullpath)
|
||||
self.last_modification: float = stat.st_atime
|
||||
self.size: int = stat.st_size
|
||||
else:
|
||||
sha = sha256
|
||||
self.sha256: str = sha
|
||||
self.last_modification = last_modification
|
||||
self.size = size
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"ResolvedLib<{self.name}, {self.dirpath}, {self.fullpath}, {self.sha256}>"
|
||||
)
|
||||
return f"ResolvedLib<{self.name}, {self.dirpath}, {self.fullpath}, {self.last_modification}, {self.size}>"
|
||||
|
||||
def to_dict(self) -> Dict:
|
||||
return {
|
||||
"name": self.name,
|
||||
"dirpath": self.dirpath,
|
||||
"fullpath": self.fullpath,
|
||||
"sha256": self.sha256,
|
||||
"last_modification": self.last_modification,
|
||||
"size": self.size,
|
||||
}
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.name, self.dirpath, self.fullpath, self.sha256))
|
||||
return hash(
|
||||
(self.name, self.dirpath, self.fullpath, self.last_modification, self.size)
|
||||
)
|
||||
|
||||
def __eq__(self, o):
|
||||
return (
|
||||
self.name == o.name
|
||||
and self.fullpath == o.fullpath
|
||||
and self.sha256 == o.sha256
|
||||
and self.dirpath == o.dirpath
|
||||
and self.last_modification == o.last_modification
|
||||
and self.size == o.size
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict):
|
||||
return ResolvedLib(d["name"], d["dirpath"], d["fullpath"], d["sha256"])
|
||||
return ResolvedLib(
|
||||
d["name"], d["dirpath"], d["fullpath"], d["last_modification"], d["size"]
|
||||
)
|
||||
|
||||
|
||||
class LibraryPath:
|
||||
|
|
@ -374,8 +382,8 @@ def is_dso_cache_up_to_date(dsos: CacheDirContent, cache_file_path: str) -> bool
|
|||
|
||||
We keep what's in the cache through a JSON file stored at the root
|
||||
of the cache_dir. We consider a dynamically shared object to be up
|
||||
to date if its name, its full path and its content sha256 are
|
||||
equivalent."""
|
||||
to date if its name, its full path, its size and last modification
|
||||
timestamp are equivalent."""
|
||||
log_info("Checking if the cache is up to date")
|
||||
if os.path.isfile(cache_file_path):
|
||||
with open(cache_file_path, "r", encoding="utf8") as f:
|
||||
|
|
|
|||
|
|
@ -9,18 +9,12 @@ class TestCacheSerializer(unittest.TestCase):
|
|||
lp = LibraryPath(
|
||||
glx=[
|
||||
ResolvedLib(
|
||||
"dummyglx.so",
|
||||
"/lib",
|
||||
"/lib/dummyglx.so",
|
||||
"031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406",
|
||||
"dummyglx.so", "/lib", "/lib/dummyglx.so", 1670260550.481498, 1612
|
||||
)
|
||||
],
|
||||
cuda=[
|
||||
ResolvedLib(
|
||||
"dummycuda.so",
|
||||
"/lib",
|
||||
"/lib/dummycuda.so",
|
||||
"031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407",
|
||||
"dummycuda.so", "/lib", "/lib/dummycuda.so", 2670260550.481498, 2612
|
||||
)
|
||||
],
|
||||
generic=[
|
||||
|
|
@ -28,15 +22,13 @@ class TestCacheSerializer(unittest.TestCase):
|
|||
"dummygeneric.so",
|
||||
"/lib",
|
||||
"/lib/dummygeneric.so",
|
||||
"031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408",
|
||||
3670260550.481498,
|
||||
3612,
|
||||
)
|
||||
],
|
||||
egl=[
|
||||
ResolvedLib(
|
||||
"dummyegl.so",
|
||||
"/lib",
|
||||
"/lib/dummyegl.so",
|
||||
"031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409",
|
||||
"dummyegl.so", "/lib", "/lib/dummyegl.so", 4670260550.481498, 4612
|
||||
)
|
||||
],
|
||||
path="/path/to/lib/dir",
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@
|
|||
"name": "dummyglx.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummyglx.so",
|
||||
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406"
|
||||
"last_modification": 1670260550.481498,
|
||||
"size": 1612
|
||||
}
|
||||
],
|
||||
"cuda": [
|
||||
|
|
@ -14,7 +15,8 @@
|
|||
"name": "dummycuda.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummycuda.so",
|
||||
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407"
|
||||
"last_modification": 2670260550.481498,
|
||||
"size": 2612
|
||||
}
|
||||
],
|
||||
"generic": [
|
||||
|
|
@ -22,7 +24,8 @@
|
|||
"name": "dummygeneric.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummygeneric.so",
|
||||
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408"
|
||||
"last_modification": 3670260550.481498,
|
||||
"size": 3612
|
||||
}
|
||||
],
|
||||
"egl": [
|
||||
|
|
@ -30,7 +33,8 @@
|
|||
"name": "dummyegl.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummyegl.so",
|
||||
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409"
|
||||
"last_modification": 4670260550.481498,
|
||||
"size": 4612
|
||||
}
|
||||
],
|
||||
"path": "/path/to/lib/dir"
|
||||
|
|
@ -41,7 +45,8 @@
|
|||
"name": "dummyglx.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummyglx.so",
|
||||
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406"
|
||||
"last_modification": 5670260550.481498,
|
||||
"size": 5612
|
||||
}
|
||||
],
|
||||
"cuda": [
|
||||
|
|
@ -49,7 +54,8 @@
|
|||
"name": "dummycuda.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummycuda.so",
|
||||
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407"
|
||||
"last_modification": 6670260550.481498,
|
||||
"size": 6612
|
||||
}
|
||||
],
|
||||
"generic": [
|
||||
|
|
@ -57,7 +63,8 @@
|
|||
"name": "dummygeneric.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummygeneric.so",
|
||||
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408"
|
||||
"last_modification": 7670260550.481498,
|
||||
"size": 7612
|
||||
}
|
||||
],
|
||||
"egl": [
|
||||
|
|
@ -65,11 +72,12 @@
|
|||
"name": "dummyegl.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummyegl.so",
|
||||
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409"
|
||||
"last_modification": 8670260550.481498,
|
||||
"size": 8612
|
||||
}
|
||||
],
|
||||
"path": "/path/to/lib/dir2"
|
||||
}
|
||||
],
|
||||
"version": 2
|
||||
"version": 3
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@
|
|||
"name": "dummyglx.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummyglx.so",
|
||||
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406"
|
||||
"last_modification": 5670260550.481498,
|
||||
"size": 5612
|
||||
}
|
||||
],
|
||||
"cuda": [
|
||||
|
|
@ -14,7 +15,8 @@
|
|||
"name": "dummycuda.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummycuda.so",
|
||||
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407"
|
||||
"last_modification": 6670260550.481498,
|
||||
"size": 6612
|
||||
}
|
||||
],
|
||||
"generic": [
|
||||
|
|
@ -22,7 +24,8 @@
|
|||
"name": "dummygeneric.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummygeneric.so",
|
||||
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408"
|
||||
"last_modification": 7670260550.481498,
|
||||
"size": 7612
|
||||
}
|
||||
],
|
||||
"egl": [
|
||||
|
|
@ -30,7 +33,8 @@
|
|||
"name": "dummyegl.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummyegl.so",
|
||||
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409"
|
||||
"last_modification": 8670260550.481498,
|
||||
"size": 8612
|
||||
}
|
||||
],
|
||||
"path": "/path/to/lib/dir2"
|
||||
|
|
@ -41,7 +45,8 @@
|
|||
"name": "dummyglx.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummyglx.so",
|
||||
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406"
|
||||
"last_modification": 1670260550.481498,
|
||||
"size": 1612
|
||||
}
|
||||
],
|
||||
"cuda": [
|
||||
|
|
@ -49,7 +54,8 @@
|
|||
"name": "dummycuda.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummycuda.so",
|
||||
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407"
|
||||
"last_modification": 2670260550.481498,
|
||||
"size": 2612
|
||||
}
|
||||
],
|
||||
"generic": [
|
||||
|
|
@ -57,7 +63,8 @@
|
|||
"name": "dummygeneric.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummygeneric.so",
|
||||
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408"
|
||||
"last_modification": 3670260550.481498,
|
||||
"size": 3612
|
||||
}
|
||||
],
|
||||
"egl": [
|
||||
|
|
@ -65,11 +72,12 @@
|
|||
"name": "dummyegl.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummyegl.so",
|
||||
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409"
|
||||
"last_modification": 4670260550.481498,
|
||||
"size": 4612
|
||||
}
|
||||
],
|
||||
"path": "/path/to/lib/dir"
|
||||
}
|
||||
],
|
||||
"version": 2
|
||||
"version": 3
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@
|
|||
"name": "dummyglx.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummyglx.so",
|
||||
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406"
|
||||
"last_modification": 2670260550.481498,
|
||||
"size": 2612
|
||||
}
|
||||
],
|
||||
"cuda": [
|
||||
|
|
@ -14,7 +15,8 @@
|
|||
"name": "dummycuda.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummycuda.so",
|
||||
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407"
|
||||
"last_modification": 2670260550.481498,
|
||||
"size": 2612
|
||||
}
|
||||
],
|
||||
"generic": [
|
||||
|
|
@ -22,7 +24,8 @@
|
|||
"name": "dummygeneric.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummygeneric.so",
|
||||
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408"
|
||||
"last_modification": 3670260550.481498,
|
||||
"size": 3612
|
||||
}
|
||||
],
|
||||
"egl": [
|
||||
|
|
@ -30,7 +33,8 @@
|
|||
"name": "dummyegl.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummyegl.so",
|
||||
"sha256": "031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409"
|
||||
"last_modification": 4670260550.481498,
|
||||
"size": 4612
|
||||
}
|
||||
],
|
||||
"path": "/path/to/lib/dir"
|
||||
|
|
@ -41,7 +45,8 @@
|
|||
"name": "dummyglx.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummyglx.so",
|
||||
"sha256": "4444444444444444444444444444444444444444444444444444444444444444"
|
||||
"last_modification": 5670260550.481498,
|
||||
"size": 5612
|
||||
}
|
||||
],
|
||||
"cuda": [
|
||||
|
|
@ -49,7 +54,8 @@
|
|||
"name": "dummycuda.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummycuda.so",
|
||||
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9407"
|
||||
"last_modification": 6670260550.481498,
|
||||
"size": 6612
|
||||
}
|
||||
],
|
||||
"generic": [
|
||||
|
|
@ -57,7 +63,8 @@
|
|||
"name": "dummygeneric.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummygeneric.so",
|
||||
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9408"
|
||||
"last_modification": 7670260550.481498,
|
||||
"size": 7612
|
||||
}
|
||||
],
|
||||
"egl": [
|
||||
|
|
@ -65,11 +72,12 @@
|
|||
"name": "dummyegl.so",
|
||||
"dirpath": "/lib",
|
||||
"fullpath": "/lib/dummyegl.so",
|
||||
"sha256": "131edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9409"
|
||||
"last_modification": 8670260550.481498,
|
||||
"size": 8612
|
||||
}
|
||||
],
|
||||
"path": "/path/to/lib/dir2"
|
||||
}
|
||||
],
|
||||
"version": 2
|
||||
"version": 3
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue