diff --git a/CMakeLists.txt b/CMakeLists.txt index 80793ac5..79fcbcb6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,6 +102,15 @@ option(WITH_MAPSERVER "Enable (experimental) support for the mapserver library" option(WITH_RIAK "Use Riak as a cache backend" OFF) option(WITH_GDAL "Choose if GDAL raster support should be built in" ON) option(WITH_MAPCACHE_DETAIL "Build coverage analysis tool for SQLite caches" ON) +option(WITH_LMDB_UTILS "Build LMDB utilities for debugging" OFF) + +# Ensure WITH_LMDB_UTILS is only enabled if WITH_LMDB is enabled +if(NOT WITH_LMDB) + if(WITH_LMDB_UTILS) + message(WARNING "WITH_LMDB_UTILS is ON but WITH_LMDB is OFF. Disabling WITH_LMDB_UTILS.") + endif() + set(WITH_LMDB_UTILS OFF CACHE BOOL "Build LMDB utilities for debugging" FORCE) +endif() find_package(PNG) if(PNG_FOUND) @@ -376,6 +385,7 @@ status_optional_component("RIAK" "${USE_RIAK}" "${RIAK_LIBRARY}") status_optional_component("GDAL" "${USE_GDAL}" "${GDAL_LIBRARY}") message(STATUS " * Optional features") status_optional_feature("MAPCACHE_DETAIL" "${WITH_MAPCACHE_DETAIL}") +status_optional_feature("LMDB_UTILS" "${WITH_LMDB_UTILS}") INSTALL(TARGETS mapcache DESTINATION ${CMAKE_INSTALL_LIBDIR}) @@ -384,6 +394,8 @@ add_subdirectory(cgi) add_subdirectory(apache) add_subdirectory(nginx) +add_subdirectory(contrib/lmdb_utils) + if (WITH_MAPCACHE_DETAIL) add_subdirectory(contrib/mapcache_detail) endif (WITH_MAPCACHE_DETAIL) diff --git a/contrib/lmdb_utils/CMakeLists.txt b/contrib/lmdb_utils/CMakeLists.txt new file mode 100644 index 00000000..9a7d9de4 --- /dev/null +++ b/contrib/lmdb_utils/CMakeLists.txt @@ -0,0 +1,12 @@ +if(WITH_LMDB_UTILS) + find_package(LMDB REQUIRED) + include_directories(${LMDB_INCLUDE_DIR}) + add_executable(mapcache_lmdb_list ${CMAKE_CURRENT_SOURCE_DIR}/mapcache_lmdb_list.c) + target_include_directories(mapcache_lmdb_list PRIVATE ${APR_INCLUDE_DIR}) + target_link_libraries(mapcache_lmdb_list PRIVATE ${LMDB_LIBRARY} ${APR_LIBRARY}) + add_executable(mapcache_lmdb_get ${CMAKE_CURRENT_SOURCE_DIR}/mapcache_lmdb_get.c) + target_include_directories(mapcache_lmdb_get PRIVATE ${APR_INCLUDE_DIR}) + target_link_libraries(mapcache_lmdb_get PRIVATE ${LMDB_LIBRARY} ${APR_LIBRARY}) + + INSTALL(TARGETS mapcache_lmdb_list mapcache_lmdb_get RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif(WITH_LMDB_UTILS) diff --git a/contrib/lmdb_utils/mapcache_lmdb_get.c b/contrib/lmdb_utils/mapcache_lmdb_get.c new file mode 100644 index 00000000..6674e463 --- /dev/null +++ b/contrib/lmdb_utils/mapcache_lmdb_get.c @@ -0,0 +1,221 @@ +/****************************************************************************** + * + * Project: MapCache + * Purpose: Extract raw value of a single key from a LMDB database + * Author: Maris Nartiss + * + ****************************************************************************** + * Copyright (c) 2025 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include "lmdb.h" + +void fail(const char *msg, int rc) { + fprintf(stderr, "%s: %s\n", msg, mdb_strerror(rc)); + exit(EXIT_FAILURE); +} + +static const apr_getopt_option_t options[] = { + { "dbpath", 'd', TRUE, "Path to the LMDB database directory" }, + { "key", 'k', TRUE, "Key to retrieve" }, + { "output", 'o', TRUE, "Output file (default: output.bin)" }, + { "help", 'h', FALSE, "Show help" }, + { NULL, 0, 0, NULL } +}; + +void usage(const char *progname) { + int i = 0; + printf("usage: %s options\n", progname); + while (options[i].name) { + if (options[i].optch < 256) { + if (options[i].has_arg == TRUE) { + printf("-%c|--%s [value]: %s\n", options[i].optch, options[i].name, options[i].description); + } else { + printf("-%c|--%s: %s\n", options[i].optch, options[i].name, options[i].description); + } + } else { + if (options[i].has_arg == TRUE) { + printf(" --%s [value]: %s\n", options[i].name, options[i].description); + } else { + printf(" --%s: %s\n", options[i].name, options[i].description); + } + } + i++; + } + exit(EXIT_FAILURE); +} + +int main(int argc, const char * const *argv) { + const char *db_path = NULL; + const char *key_str = NULL; + const char *out_filename = "output.bin"; + int rc; + MDB_env *env; + MDB_txn *txn; + MDB_dbi dbi; + MDB_val key, data; + apr_pool_t *pool = NULL; + apr_getopt_t *opt; + int optch; + const char *optarg; + + apr_initialize(); + apr_pool_create(&pool, NULL); + apr_getopt_init(&opt, pool, argc, argv); + + while ((rc = apr_getopt_long(opt, options, &optch, &optarg)) == APR_SUCCESS) { + switch (optch) { + case 'h': + usage(argv[0]); + break; + case 'd': + db_path = optarg; + break; + case 'k': + key_str = optarg; + break; + case 'o': + out_filename = optarg; + break; + } + } + + if (rc != APR_EOF) { + fprintf(stderr, "Error: Invalid option.\n"); + usage(argv[0]); + } + + if (!db_path || !key_str) { + fprintf(stderr, "Error: --dbpath and --key are required.\n"); + usage(argv[0]); + } + + // Create and open the environment + rc = mdb_env_create(&env); + if (rc) fail("mdb_env_create", rc); + + rc = mdb_env_open(env, db_path, MDB_RDONLY, 0664); + if (rc) { + mdb_env_close(env); + fail("mdb_env_open", rc); + } + + // Begin a read-only transaction + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) { + mdb_env_close(env); + fail("mdb_txn_begin", rc); + } + + // Open the database + rc = mdb_dbi_open(txn, NULL, 0, &dbi); + if (rc) { + mdb_txn_abort(txn); + mdb_env_close(env); + fail("mdb_dbi_open", rc); + } + + // Prepare the key, which is a null-terminated string in the DB + key.mv_size = strlen(key_str) + 1; + key.mv_data = (void *)key_str; + + // Retrieve the data + rc = mdb_get(txn, dbi, &key, &data); + + if (rc == MDB_SUCCESS) { + apr_time_t timestamp; + char time_str[APR_RFC822_DATE_LEN]; + FILE *fp; + fp = fopen(out_filename, "wb"); + if (!fp) { + mdb_txn_abort(txn); + mdb_env_close(env); + perror("Unable to open output file"); + return EXIT_FAILURE; + } else { + // Check for special blank tile encoding ('#' marker) + if (data.mv_size > 1 && ((char *)data.mv_data)[0] == '#') { + if (data.mv_size == 5 + sizeof(apr_time_t)) { + const unsigned char *color = (unsigned char *)data.mv_data + 1; + printf("Key found. Blank tile, color #%02x%02x%02x%02x. " + "Writing description to \"%s\"\n", + color[0], color[1], color[2], color[3], out_filename); + fprintf(fp, "Blank tile, RGBA: #%02x%02x%02x%02x\n", + color[0], color[1], color[2], color[3]); + } else { + printf("Key found. Blank tile marker found, but data " + "size is unexpected (%zu bytes). Writing as is.\n", + data.mv_size); + fwrite(data.mv_data, 1, data.mv_size, fp); + } + + // Extract timestamp from the end of the data + memcpy(×tamp, (char *)data.mv_data + 5, sizeof(apr_time_t)); + + // Convert to human-readable format + apr_rfc822_date(time_str, timestamp); + + printf("Timestamp: %s\n", time_str); + } else if (data.mv_size >= sizeof(apr_time_t)) { + // Regular tile data, strip the trailing timestamp + size_t image_size = data.mv_size - sizeof(apr_time_t); + + // Extract timestamp from the end of the data + memcpy(×tamp, (char *)data.mv_data + image_size, sizeof(apr_time_t)); + + // Convert to human-readable format + apr_rfc822_date(time_str, timestamp); + + printf("Key found. Writing %zu image bytes to \"%s\"\n", + image_size, out_filename); + printf("Timestamp: %s\n", time_str); + fwrite(data.mv_data, 1, image_size, fp); + } else { + // Data is smaller than a timestamp, write as is + printf("Key found. Data size (%zu) is smaller than a " + "timestamp, writing as is to \"%s\"\n", + data.mv_size, out_filename); + fwrite(data.mv_data, 1, data.mv_size, fp); + } + fclose(fp); + } + } else if (rc == MDB_NOTFOUND) { + fprintf(stderr, "Key '%s' not found.\n", key_str); + } else { + fail("mdb_get", rc); + } + + // Clean up + mdb_txn_abort(txn); + mdb_env_close(env); + + apr_pool_destroy(pool); + apr_terminate(); + + return (rc == MDB_SUCCESS) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/contrib/lmdb_utils/mapcache_lmdb_list.c b/contrib/lmdb_utils/mapcache_lmdb_list.c new file mode 100644 index 00000000..fde4adaa --- /dev/null +++ b/contrib/lmdb_utils/mapcache_lmdb_list.c @@ -0,0 +1,228 @@ +/****************************************************************************** + * + * Project: MapCache + * Purpose: List all keys in a LMDB database + * Author: Maris Nartiss + * + ****************************************************************************** + * Copyright (c) 2025 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "lmdb.h" + +void fail(const char *msg, int rc) { + fprintf(stderr, "%s: %s\n", msg, mdb_strerror(rc)); + exit(EXIT_FAILURE); +} + +static const apr_getopt_option_t options[] = { + { "dbpath", 'd', TRUE, "Path to the LMDB database directory" }, + { "summary", 's', FALSE, "Print only the total number of keys" }, + { "extended", 'e', FALSE, "Show extended info (timestamp, size)" }, + { "help", 'h', FALSE, "Show help" }, + { NULL, 0, 0, NULL } +}; + +void usage(const char *progname) { + int i = 0; + printf("usage: %s options\n", progname); + while (options[i].name) { + if (options[i].optch < 256) { + if (options[i].has_arg == TRUE) { + printf("-%c|--%s [value]: %s\n", options[i].optch, options[i].name, options[i].description); + } else { + printf("-%c|--%s: %s\n", options[i].optch, options[i].name, options[i].description); + } + } else { + if (options[i].has_arg == TRUE) { + printf(" --%s [value]: %s\n", options[i].name, options[i].description); + } else { + printf(" --%s: %s\n", options[i].name, options[i].description); + } + } + i++; + } + exit(EXIT_FAILURE); +} + +int main(int argc, const char * const *argv) { + const char *db_path = NULL; + int rc; + MDB_env *env; + MDB_txn *txn; + MDB_cursor *cursor = NULL; + MDB_val key; + MDB_dbi dbi; + MDB_val value; + int summary_flag = 0; + int extended_flag = 0; + MDB_stat stat; + apr_pool_t *pool = NULL; + apr_getopt_t *opt; + int optch; + const char *optarg; + + apr_initialize(); + apr_pool_create(&pool, NULL); + apr_getopt_init(&opt, pool, argc, argv); + + while ((rc = apr_getopt_long(opt, options, &optch, &optarg)) == APR_SUCCESS) { + switch (optch) { + case 'h': + usage(argv[0]); + break; + case 'd': + db_path = optarg; + break; + case 's': + summary_flag = 1; + break; + case 'e': + extended_flag = 1; + break; + } + } + + if (rc != APR_EOF) { + fprintf(stderr, "Error: Invalid option.\n"); + usage(argv[0]); + } + + if (!db_path) { + fprintf(stderr, "Error: --dbpath is required.\n"); + usage(argv[0]); + } + + // Create and open the environment + rc = mdb_env_create(&env); + if (rc) fail("mdb_env_create", rc); + + rc = mdb_env_open(env, db_path, MDB_RDONLY, 0664); + if (rc) { + mdb_env_close(env); + fail("mdb_env_open", rc); + } + + // Begin a read-only transaction + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) { + mdb_env_close(env); + fail("mdb_txn_begin", rc); + } + + // Open the database + rc = mdb_dbi_open(txn, NULL, 0, &dbi); + if (rc) { + mdb_txn_abort(txn); + mdb_env_close(env); + fail("mdb_dbi_open", rc); + } + + if (summary_flag) { + rc = mdb_stat(txn, dbi, &stat); + if (rc) { + mdb_txn_abort(txn); + mdb_env_close(env); + fail("mdb_stat", rc); + } + printf("Total keys found: %zu\n", stat.ms_entries); + } else { + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + mdb_txn_abort(txn); + mdb_env_close(env); + fail("mdb_cursor_open", rc); + } + + // Iterate over keys + while ((rc = mdb_cursor_get(cursor, &key, &value, MDB_NEXT)) == 0) { + // Print the key itself, without the null terminator + fwrite(key.mv_data, 1, key.mv_size - 1, stdout); + + if (extended_flag) { + apr_time_t timestamp; + size_t data_size; + + // Check for special blank tile encoding ('#' marker) + if (value.mv_size > 1 && ((char *)value.mv_data)[0] == '#') { + if (value.mv_size == 5 + sizeof(apr_time_t)) { + memcpy(×tamp, (char *)value.mv_data + 5, sizeof(apr_time_t)); + data_size = 4; // RGBA color + } else { + // Unexpected size, cannot parse + timestamp = 0; + data_size = 0; + } + } else if (value.mv_size >= sizeof(apr_time_t)) { + // Regular tile data + data_size = value.mv_size - sizeof(apr_time_t); + memcpy(×tamp, (char *)value.mv_data + data_size, sizeof(apr_time_t)); + } else { + // Data is too small, cannot parse + timestamp = 0; + data_size = value.mv_size; + } + + if (timestamp > 0) { + apr_time_exp_t exploded_time; + char iso_time_str[30]; + apr_time_exp_gmt(&exploded_time, timestamp); + apr_snprintf(iso_time_str, sizeof(iso_time_str), + "%d-%02d-%02dT%02d:%02d:%02dZ", + exploded_time.tm_year + 1900, + exploded_time.tm_mon + 1, + exploded_time.tm_mday, + exploded_time.tm_hour, + exploded_time.tm_min, + exploded_time.tm_sec); + printf(",%s,%zu", iso_time_str, data_size); + } else { + printf(",,%zu", data_size); // Print size even if timestamp is unknown + } + } + printf("\n"); + } + + if (rc != MDB_NOTFOUND) { + mdb_txn_abort(txn); + mdb_env_close(env); + fail("mdb_cursor_get", rc); + } + mdb_cursor_close(cursor); + } + + // Clean up + mdb_txn_abort(txn); + mdb_env_close(env); + + apr_pool_destroy(pool); + apr_terminate(); + + return EXIT_SUCCESS; +}