578 lines
13 KiB
C
578 lines
13 KiB
C
/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
|
|
|
|
/***
|
|
This file is part of systemd.
|
|
|
|
Copyright 2012 Lennart Poettering
|
|
|
|
systemd is free software; you can redistribute it and/or modify it
|
|
under the terms of the GNU Lesser General Public License as published by
|
|
the Free Software Foundation; either version 2.1 of the License, or
|
|
(at your option) any later version.
|
|
|
|
systemd is distributed in the hope that it will be useful, but
|
|
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public License
|
|
along with systemd; If not, see <http://www.gnu.org/licenses/>.
|
|
***/
|
|
|
|
#include <errno.h>
|
|
#include <stdlib.h>
|
|
#include <sys/mman.h>
|
|
#include <string.h>
|
|
|
|
#include "hashmap.h"
|
|
#include "list.h"
|
|
#include "log.h"
|
|
#include "util.h"
|
|
#include "macro.h"
|
|
#include "mmap-cache.h"
|
|
|
|
typedef struct Window Window;
|
|
typedef struct Context Context;
|
|
typedef struct FileDescriptor FileDescriptor;
|
|
|
|
struct Window {
|
|
MMapCache *cache;
|
|
|
|
bool keep_always;
|
|
bool in_unused;
|
|
|
|
void *ptr;
|
|
uint64_t offset;
|
|
int prot;
|
|
size_t size;
|
|
|
|
FileDescriptor *fd;
|
|
|
|
LIST_FIELDS(Window, by_fd);
|
|
LIST_FIELDS(Window, unused);
|
|
|
|
LIST_HEAD(Context, contexts);
|
|
};
|
|
|
|
struct Context {
|
|
MMapCache *cache;
|
|
unsigned id;
|
|
Window *window;
|
|
|
|
LIST_FIELDS(Context, by_window);
|
|
};
|
|
|
|
struct FileDescriptor {
|
|
MMapCache *cache;
|
|
int fd;
|
|
LIST_HEAD(Window, windows);
|
|
};
|
|
|
|
struct MMapCache {
|
|
int n_ref;
|
|
|
|
Hashmap *fds;
|
|
Hashmap *contexts;
|
|
|
|
unsigned n_windows;
|
|
|
|
LIST_HEAD(Window, unused);
|
|
Window *last_unused;
|
|
};
|
|
|
|
#define WINDOWS_MIN 64
|
|
#define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
|
|
|
|
MMapCache* mmap_cache_new(void) {
|
|
MMapCache *m;
|
|
|
|
m = new0(MMapCache, 1);
|
|
if (!m)
|
|
return NULL;
|
|
|
|
m->n_ref = 1;
|
|
return m;
|
|
}
|
|
|
|
MMapCache* mmap_cache_ref(MMapCache *m) {
|
|
assert(m);
|
|
assert(m->n_ref > 0);
|
|
|
|
m->n_ref ++;
|
|
return m;
|
|
}
|
|
|
|
static void window_unlink(Window *w) {
|
|
Context *c;
|
|
|
|
assert(w);
|
|
|
|
if (w->ptr)
|
|
munmap(w->ptr, w->size);
|
|
|
|
if (w->fd)
|
|
LIST_REMOVE(Window, by_fd, w->fd->windows, w);
|
|
|
|
if (w->in_unused) {
|
|
if (w->cache->last_unused == w)
|
|
w->cache->last_unused = w->unused_prev;
|
|
|
|
LIST_REMOVE(Window, unused, w->cache->unused, w);
|
|
}
|
|
|
|
LIST_FOREACH(by_window, c, w->contexts) {
|
|
assert(c->window == w);
|
|
c->window = NULL;
|
|
}
|
|
}
|
|
|
|
static void window_free(Window *w) {
|
|
assert(w);
|
|
|
|
window_unlink(w);
|
|
w->cache->n_windows--;
|
|
free(w);
|
|
}
|
|
|
|
static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
|
|
assert(w);
|
|
assert(fd >= 0);
|
|
assert(size > 0);
|
|
|
|
return
|
|
w->fd &&
|
|
fd == w->fd->fd &&
|
|
prot == w->prot &&
|
|
offset >= w->offset &&
|
|
offset + size <= w->offset + w->size;
|
|
}
|
|
|
|
static Window *window_add(MMapCache *m) {
|
|
Window *w;
|
|
|
|
assert(m);
|
|
|
|
if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
|
|
|
|
/* Allocate a new window */
|
|
w = new0(Window, 1);
|
|
if (!w)
|
|
return NULL;
|
|
m->n_windows++;
|
|
} else {
|
|
|
|
/* Reuse an existing one */
|
|
w = m->last_unused;
|
|
window_unlink(w);
|
|
zero(*w);
|
|
}
|
|
|
|
w->cache = m;
|
|
return w;
|
|
}
|
|
|
|
static void context_detach_window(Context *c) {
|
|
Window *w;
|
|
|
|
assert(c);
|
|
|
|
if (!c->window)
|
|
return;
|
|
|
|
w = c->window;
|
|
c->window = NULL;
|
|
LIST_REMOVE(Context, by_window, w->contexts, c);
|
|
|
|
if (!w->contexts && !w->keep_always) {
|
|
/* Not used anymore? */
|
|
LIST_PREPEND(Window, unused, c->cache->unused, w);
|
|
if (!c->cache->last_unused)
|
|
c->cache->last_unused = w;
|
|
|
|
w->in_unused = true;
|
|
}
|
|
}
|
|
|
|
static void context_attach_window(Context *c, Window *w) {
|
|
assert(c);
|
|
assert(w);
|
|
|
|
if (c->window == w)
|
|
return;
|
|
|
|
context_detach_window(c);
|
|
|
|
if (w->in_unused) {
|
|
/* Used again? */
|
|
LIST_REMOVE(Window, unused, c->cache->unused, w);
|
|
if (c->cache->last_unused == w)
|
|
c->cache->last_unused = w->unused_prev;
|
|
|
|
w->in_unused = false;
|
|
}
|
|
|
|
c->window = w;
|
|
LIST_PREPEND(Context, by_window, w->contexts, c);
|
|
}
|
|
|
|
static Context *context_add(MMapCache *m, unsigned id) {
|
|
Context *c;
|
|
int r;
|
|
|
|
assert(m);
|
|
|
|
c = hashmap_get(m->contexts, UINT_TO_PTR(id + 1));
|
|
if (c)
|
|
return c;
|
|
|
|
r = hashmap_ensure_allocated(&m->contexts, trivial_hash_func, trivial_compare_func);
|
|
if (r < 0)
|
|
return NULL;
|
|
|
|
c = new0(Context, 1);
|
|
if (!c)
|
|
return NULL;
|
|
|
|
c->cache = m;
|
|
c->id = id;
|
|
|
|
r = hashmap_put(m->contexts, UINT_TO_PTR(id + 1), c);
|
|
if (r < 0) {
|
|
free(c);
|
|
return NULL;
|
|
}
|
|
|
|
return c;
|
|
}
|
|
|
|
static void context_free(Context *c) {
|
|
assert(c);
|
|
|
|
context_detach_window(c);
|
|
|
|
if (c->cache)
|
|
assert_se(hashmap_remove(c->cache->contexts, UINT_TO_PTR(c->id + 1)));
|
|
|
|
free(c);
|
|
}
|
|
|
|
static void fd_free(FileDescriptor *f) {
|
|
assert(f);
|
|
|
|
while (f->windows)
|
|
window_free(f->windows);
|
|
|
|
if (f->cache)
|
|
assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1)));
|
|
|
|
free(f);
|
|
}
|
|
|
|
static FileDescriptor* fd_add(MMapCache *m, int fd) {
|
|
FileDescriptor *f;
|
|
int r;
|
|
|
|
assert(m);
|
|
assert(fd >= 0);
|
|
|
|
f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
|
|
if (f)
|
|
return f;
|
|
|
|
r = hashmap_ensure_allocated(&m->fds, trivial_hash_func, trivial_compare_func);
|
|
if (r < 0)
|
|
return NULL;
|
|
|
|
f = new0(FileDescriptor, 1);
|
|
if (!f)
|
|
return NULL;
|
|
|
|
f->cache = m;
|
|
f->fd = fd;
|
|
|
|
r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f);
|
|
if (r < 0) {
|
|
free(f);
|
|
return NULL;
|
|
}
|
|
|
|
return f;
|
|
}
|
|
|
|
static void mmap_cache_free(MMapCache *m) {
|
|
Context *c;
|
|
FileDescriptor *f;
|
|
|
|
assert(m);
|
|
|
|
while ((c = hashmap_first(m->contexts)))
|
|
context_free(c);
|
|
|
|
while ((f = hashmap_first(m->fds)))
|
|
fd_free(f);
|
|
|
|
while (m->unused)
|
|
window_free(m->unused);
|
|
|
|
free(m);
|
|
}
|
|
|
|
MMapCache* mmap_cache_unref(MMapCache *m) {
|
|
assert(m);
|
|
assert(m->n_ref > 0);
|
|
|
|
m->n_ref --;
|
|
if (m->n_ref == 0)
|
|
mmap_cache_free(m);
|
|
|
|
return NULL;
|
|
}
|
|
|
|
static int make_room(MMapCache *m) {
|
|
assert(m);
|
|
|
|
if (!m->last_unused)
|
|
return 0;
|
|
|
|
window_free(m->last_unused);
|
|
return 1;
|
|
}
|
|
|
|
static int try_context(
|
|
MMapCache *m,
|
|
int fd,
|
|
int prot,
|
|
unsigned context,
|
|
bool keep_always,
|
|
uint64_t offset,
|
|
size_t size,
|
|
void **ret) {
|
|
|
|
Context *c;
|
|
|
|
assert(m);
|
|
assert(m->n_ref > 0);
|
|
assert(fd >= 0);
|
|
assert(size > 0);
|
|
assert(ret);
|
|
|
|
c = hashmap_get(m->contexts, UINT_TO_PTR(context+1));
|
|
if (!c)
|
|
return 0;
|
|
|
|
assert(c->id == context);
|
|
|
|
if (!c->window)
|
|
return 0;
|
|
|
|
if (!window_matches(c->window, fd, prot, offset, size)) {
|
|
|
|
/* Drop the reference to the window, since it's unnecessary now */
|
|
context_detach_window(c);
|
|
return 0;
|
|
}
|
|
|
|
c->window->keep_always = c->window->keep_always || keep_always;
|
|
|
|
*ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
|
|
return 1;
|
|
}
|
|
|
|
static int find_mmap(
|
|
MMapCache *m,
|
|
int fd,
|
|
int prot,
|
|
unsigned context,
|
|
bool keep_always,
|
|
uint64_t offset,
|
|
size_t size,
|
|
void **ret) {
|
|
|
|
FileDescriptor *f;
|
|
Window *w;
|
|
Context *c;
|
|
|
|
assert(m);
|
|
assert(m->n_ref > 0);
|
|
assert(fd >= 0);
|
|
assert(size > 0);
|
|
assert(ret);
|
|
|
|
f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
|
|
if (!f)
|
|
return 0;
|
|
|
|
assert(f->fd == fd);
|
|
|
|
LIST_FOREACH(by_fd, w, f->windows)
|
|
if (window_matches(w, fd, prot, offset, size))
|
|
break;
|
|
|
|
if (!w)
|
|
return 0;
|
|
|
|
c = context_add(m, context);
|
|
if (!c)
|
|
return -ENOMEM;
|
|
|
|
context_attach_window(c, w);
|
|
w->keep_always = w->keep_always || keep_always;
|
|
|
|
*ret = (uint8_t*) w->ptr + (offset - w->offset);
|
|
return 1;
|
|
}
|
|
|
|
static int add_mmap(
|
|
MMapCache *m,
|
|
int fd,
|
|
int prot,
|
|
unsigned context,
|
|
bool keep_always,
|
|
uint64_t offset,
|
|
size_t size,
|
|
struct stat *st,
|
|
void **ret) {
|
|
|
|
uint64_t woffset, wsize;
|
|
Context *c;
|
|
FileDescriptor *f;
|
|
Window *w;
|
|
void *d;
|
|
int r;
|
|
|
|
assert(m);
|
|
assert(m->n_ref > 0);
|
|
assert(fd >= 0);
|
|
assert(size > 0);
|
|
assert(ret);
|
|
|
|
woffset = offset & ~((uint64_t) page_size() - 1ULL);
|
|
wsize = size + (offset - woffset);
|
|
wsize = PAGE_ALIGN(wsize);
|
|
|
|
if (wsize < WINDOW_SIZE) {
|
|
uint64_t delta;
|
|
|
|
delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2);
|
|
|
|
if (delta > offset)
|
|
woffset = 0;
|
|
else
|
|
woffset -= delta;
|
|
|
|
wsize = WINDOW_SIZE;
|
|
}
|
|
|
|
if (st) {
|
|
/* Memory maps that are larger then the files
|
|
underneath have undefined behavior. Hence, clamp
|
|
things to the file size if we know it */
|
|
|
|
if (woffset >= (uint64_t) st->st_size)
|
|
return -EADDRNOTAVAIL;
|
|
|
|
if (woffset + wsize > (uint64_t) st->st_size)
|
|
wsize = PAGE_ALIGN(st->st_size - woffset);
|
|
}
|
|
|
|
for (;;) {
|
|
d = mmap(NULL, wsize, prot, MAP_SHARED, fd, woffset);
|
|
if (d != MAP_FAILED)
|
|
break;
|
|
if (errno != ENOMEM)
|
|
return -errno;
|
|
|
|
r = make_room(m);
|
|
if (r < 0)
|
|
return r;
|
|
if (r == 0)
|
|
return -ENOMEM;
|
|
}
|
|
|
|
c = context_add(m, context);
|
|
if (!c)
|
|
return -ENOMEM;
|
|
|
|
f = fd_add(m, fd);
|
|
if (!f)
|
|
return -ENOMEM;
|
|
|
|
w = window_add(m);
|
|
if (!w)
|
|
return -ENOMEM;
|
|
|
|
w->keep_always = keep_always;
|
|
w->ptr = d;
|
|
w->offset = woffset;
|
|
w->prot = prot;
|
|
w->size = wsize;
|
|
w->fd = f;
|
|
|
|
LIST_PREPEND(Window, by_fd, f->windows, w);
|
|
|
|
context_detach_window(c);
|
|
c->window = w;
|
|
LIST_PREPEND(Context, by_window, w->contexts, c);
|
|
|
|
*ret = (uint8_t*) w->ptr + (offset - w->offset);
|
|
return 1;
|
|
}
|
|
|
|
int mmap_cache_get(
|
|
MMapCache *m,
|
|
int fd,
|
|
int prot,
|
|
unsigned context,
|
|
bool keep_always,
|
|
uint64_t offset,
|
|
size_t size,
|
|
struct stat *st,
|
|
void **ret) {
|
|
|
|
int r;
|
|
|
|
assert(m);
|
|
assert(m->n_ref > 0);
|
|
assert(fd >= 0);
|
|
assert(size > 0);
|
|
assert(ret);
|
|
|
|
/* Check whether the current context is the right one already */
|
|
r = try_context(m, fd, prot, context, keep_always, offset, size, ret);
|
|
if (r != 0)
|
|
return r;
|
|
|
|
/* Search for a matching mmap */
|
|
r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret);
|
|
if (r != 0)
|
|
return r;
|
|
|
|
/* Create a new mmap */
|
|
return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret);
|
|
}
|
|
|
|
void mmap_cache_close_fd(MMapCache *m, int fd) {
|
|
FileDescriptor *f;
|
|
|
|
assert(m);
|
|
assert(fd >= 0);
|
|
|
|
f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
|
|
if (!f)
|
|
return;
|
|
|
|
fd_free(f);
|
|
}
|
|
|
|
void mmap_cache_close_context(MMapCache *m, unsigned context) {
|
|
Context *c;
|
|
|
|
assert(m);
|
|
|
|
c = hashmap_get(m->contexts, UINT_TO_PTR(context + 1));
|
|
if (!c)
|
|
return;
|
|
|
|
context_free(c);
|
|
}
|