97 #define DICT_LMDB_SUFFIX "lmdb"
102 #define SCOPY(buf, data, size) \
103 vstring_str(vstring_strncpy(buf ? buf : (buf = vstring_alloc(10)), data, size))
117 #define DICT_LMDB_SIZE_INCR 2
118 #define DICT_LMDB_SIZE_MAX SSIZE_T_MAX
120 #define DICT_LMDB_API_RETRY_LIMIT 2
121 #define DICT_LMDB_BULK_RETRY_LIMIT \
122 ((int) (2 * sizeof(size_t) * CHAR_BIT))
129 static const char *dict_lmdb_lookup(
DICT *dict,
const char *name)
131 DICT_LMDB *dict_lmdb = (DICT_LMDB *) dict;
134 const char *result = 0;
145 msg_panic(
"dict_lmdb_lookup: no DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL flag");
169 mdb_key.mv_data = (
void *) name;
170 mdb_key.mv_size = klen + 1;
171 status =
slmdb_get(&dict_lmdb->slmdb, &mdb_key, &mdb_value);
174 result = SCOPY(dict_lmdb->val_buf, mdb_value.mv_data,
176 }
else if (status != MDB_NOTFOUND) {
178 dict_lmdb->dict.type, dict_lmdb->dict.name,
179 mdb_strerror(status));
188 mdb_key.mv_data = (
void *) name;
189 mdb_key.mv_size = klen;
190 status =
slmdb_get(&dict_lmdb->slmdb, &mdb_key, &mdb_value);
193 result = SCOPY(dict_lmdb->val_buf, mdb_value.mv_data,
195 }
else if (status != MDB_NOTFOUND) {
197 dict_lmdb->dict.type, dict_lmdb->dict.name,
198 mdb_strerror(status));
214 static int dict_lmdb_update(
DICT *dict,
const char *name,
const char *value)
216 DICT_LMDB *dict_lmdb = (DICT_LMDB *) dict;
227 msg_panic(
"dict_lmdb_update: no DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL flag");
238 mdb_key.mv_data = (
void *) name;
239 mdb_value.mv_data = (
void *) value;
240 mdb_key.mv_size = strlen(name);
241 mdb_value.mv_size = strlen(value);
249 #ifdef LMDB_NO_TRAILING_NULL
274 status =
slmdb_put(&dict_lmdb->slmdb, &mdb_key, &mdb_value,
277 if (status == MDB_KEYEXIST) {
281 msg_warn(
"%s:%s: duplicate entry: \"%s\"",
282 dict_lmdb->dict.type, dict_lmdb->dict.name, name);
284 msg_fatal(
"%s:%s: duplicate entry: \"%s\"",
285 dict_lmdb->dict.type, dict_lmdb->dict.name, name);
288 dict_lmdb->dict.type, dict_lmdb->dict.name,
289 mdb_strerror(status));
305 static int dict_lmdb_delete(
DICT *dict,
const char *name)
307 DICT_LMDB *dict_lmdb = (DICT_LMDB *) dict;
319 msg_panic(
"dict_lmdb_delete: no DICT_FLAG_TRY1NULL | DICT_FLAG_TRY0NULL flag");
343 mdb_key.mv_data = (
void *) name;
344 mdb_key.mv_size = klen + 1;
345 status =
slmdb_del(&dict_lmdb->slmdb, &mdb_key);
347 if (status == MDB_NOTFOUND)
350 msg_fatal(
"error deleting from %s:%s: %s",
351 dict_lmdb->dict.type, dict_lmdb->dict.name,
352 mdb_strerror(status));
363 mdb_key.mv_data = (
void *) name;
364 mdb_key.mv_size = klen;
365 status =
slmdb_del(&dict_lmdb->slmdb, &mdb_key);
367 if (status == MDB_NOTFOUND)
370 msg_fatal(
"error deleting from %s:%s: %s",
371 dict_lmdb->dict.type, dict_lmdb->dict.name,
372 mdb_strerror(status));
390 static int dict_lmdb_sequence(
DICT *dict,
int function,
391 const char **key,
const char **value)
393 const char *myname =
"dict_lmdb_sequence";
394 DICT_LMDB *dict_lmdb = (DICT_LMDB *) dict;
413 msg_panic(
"%s: invalid function: %d", myname,
function);
434 *key = SCOPY(dict_lmdb->key_buf, mdb_key.mv_data, mdb_key.mv_size);
435 if (mdb_value.mv_data != 0 && mdb_value.mv_size > 0)
436 *value = SCOPY(dict_lmdb->val_buf, mdb_value.mv_data,
455 dict_lmdb->dict.type, dict_lmdb->dict.name,
456 mdb_strerror(status));
471 static void dict_lmdb_close(
DICT *dict)
473 DICT_LMDB *dict_lmdb = (DICT_LMDB *) dict;
476 if (dict_lmdb->key_buf)
478 if (dict_lmdb->val_buf)
487 static void dict_lmdb_longjmp(
void *context,
int val)
489 DICT_LMDB *dict_lmdb = (DICT_LMDB *) context;
496 static void dict_lmdb_notify(
void *context,
int error_code,...)
498 DICT_LMDB *dict_lmdb = (DICT_LMDB *) context;
501 va_start(ap, error_code);
502 switch (error_code) {
504 msg_info(
"database %s:%s: using size limit %lu during open",
505 dict_lmdb->dict.type, dict_lmdb->dict.name,
506 (
unsigned long) va_arg(ap,
size_t));
509 msg_info(
"database %s:%s: using size limit %lu after MDB_MAP_FULL",
510 dict_lmdb->dict.type, dict_lmdb->dict.name,
511 (
unsigned long) va_arg(ap,
size_t));
513 case MDB_MAP_RESIZED:
514 msg_info(
"database %s:%s: using size limit %lu after MDB_MAP_RESIZED",
515 dict_lmdb->dict.type, dict_lmdb->dict.name,
516 (
unsigned long) va_arg(ap,
size_t));
518 case MDB_READERS_FULL:
519 msg_info(
"database %s:%s: pausing after MDB_READERS_FULL",
520 dict_lmdb->dict.type, dict_lmdb->dict.name);
523 msg_warn(
"unknown MDB error code: %d", error_code);
531 static void dict_lmdb_assert(
void *context,
const char *text)
533 DICT_LMDB *dict_lmdb = (DICT_LMDB *) context;
536 dict_lmdb->dict.type, dict_lmdb->dict.name, text);
543 DICT_LMDB *dict_lmdb;
548 int mdb_flags, slmdb_flags, status;
554 #define DICT_LMDB_OPEN_RETURN(d) do { \
565 mdb_flags = MDB_NOSUBDIR | MDB_NOLOCK;
566 if (open_flags == O_RDONLY)
567 mdb_flags |= MDB_RDONLY;
602 #ifndef MDB_NOMEMINIT
603 if (dict_flags & DICT_FLAG_BULK_UPDATE)
604 mdb_flags |= MDB_WRITEMAP;
611 DICT_LMDB_SIZE_MAX)) != 0
612 || (status =
slmdb_open(&slmdb, mdb_path, open_flags, mdb_flags,
613 slmdb_flags)) != 0) {
616 "open database %s: %s", mdb_path, mdb_strerror(status));
617 DICT_LMDB_OPEN_RETURN(dict);
630 if (dict_flags & DICT_FLAG_BULK_UPDATE) {
632 msg_fatal(
"%s: lock dictionary: %m", mdb_path);
634 msg_fatal(
"%s: unlock dictionary: %m", mdb_path);
641 dict_lmdb->slmdb = slmdb;
642 dict_lmdb->dict.
lookup = dict_lmdb_lookup;
643 dict_lmdb->dict.update = dict_lmdb_update;
644 dict_lmdb->dict.delete = dict_lmdb_delete;
645 dict_lmdb->dict.sequence = dict_lmdb_sequence;
646 dict_lmdb->dict.close = dict_lmdb_close;
648 if (
fstat(db_fd, &st) < 0)
650 dict_lmdb->dict.lock_fd = dict_lmdb->dict.stat_fd = db_fd;
652 dict_lmdb->dict.mtime = st.st_mtime;
653 dict_lmdb->dict.owner.uid = st.st_uid;
654 dict_lmdb->dict.owner.status = (st.st_uid != 0);
656 dict_lmdb->key_buf = 0;
657 dict_lmdb->val_buf = 0;
664 &&
stat(path, &st) == 0
665 && st.st_mtime > dict_lmdb->dict.mtime
666 && st.st_mtime < time((time_t *) 0) - 100)
667 msg_warn(
"database %s is older than source file %s", mdb_path, path);
669 #define DICT_LMDB_IMPL_FLAGS (DICT_FLAG_FIXED | DICT_FLAG_MULTI_WRITER)
671 dict_lmdb->dict.flags = dict_flags | DICT_LMDB_IMPL_FLAGS;
677 if (dict_flags & DICT_FLAG_BULK_UPDATE)
693 msg_panic(
"dict_lmdb_open: slmdb_control: %m");
696 dict_lmdb_notify((
void *) dict_lmdb, MDB_SUCCESS,
699 DICT_LMDB_OPEN_RETURN(
DICT_DEBUG (&dict_lmdb->dict));
#define DICT_FLAG_DUP_IGNORE
#define dict_longjmp(dict, val)
#define MYFLOCK_OP_SHARED
#define DICT_SEQ_FUN_FIRST
NORETURN msg_panic(const char *fmt,...)
void(* SLMDB_NOTIFY_FN)(void *, int,...)
int slmdb_del(SLMDB *, MDB_val *)
#define DICT_SEQ_FUN_NEXT
#define MYFLOCK_OP_EXCLUSIVE
int slmdb_init(SLMDB *, size_t, int, size_t)
size_t dict_lmdb_map_size
#define CA_SLMDB_CTL_API_RETRY_LIMIT(v)
#define DICT_FLAG_FOLD_FIX
VSTRING * vstring_strcpy(VSTRING *vp, const char *src)
int slmdb_get(SLMDB *, MDB_val *, MDB_val *)
void dict_jmp_alloc(DICT *)
DICT * dict_lmdb_open(const char *, int, int)
#define DICT_FLAG_DUP_REPLACE
#define DICT_FLAG_TRY1NULL
void msg_warn(const char *fmt,...)
VSTRING * vstring_alloc(ssize_t len)
int myflock(int fd, int lock_style, int operation)
#define slmdb_curr_limit(slmdb)
char * lowercase(char *string)
const char *(* lookup)(struct DICT *, const char *)
#define DICT_FLAG_DUP_WARN
#define CA_SLMDB_CTL_LONGJMP_FN(v)
NORETURN msg_fatal(const char *fmt,...)
char * concatenate(const char *arg0,...)
#define CA_SLMDB_CTL_CB_CONTEXT(v)
#define CA_SLMDB_CTL_NOTIFY_FN(v)
VSTRING * vstring_free(VSTRING *vp)
int slmdb_cursor_get(SLMDB *, MDB_val *, MDB_val *, MDB_cursor_op)
#define CA_SLMDB_CTL_BULK_RETRY_LIMIT(v)
int slmdb_open(SLMDB *, const char *, int, int, int)
DICT * dict_alloc(const char *, const char *, ssize_t)
#define CA_SLMDB_CTL_ASSERT_FN(v)
#define MYFLOCK_STYLE_FCNTL
#define DICT_FLAG_BULK_UPDATE
int slmdb_control(SLMDB *, int,...)
#define DICT_FLAG_TRY0NULL
int slmdb_put(SLMDB *, MDB_val *, MDB_val *, int)
DICT * dict_surrogate(const char *dict_type, const char *dict_name, int open_flags, int dict_flags, const char *fmt,...)
void msg_info(const char *fmt,...)