diff --git a/Makefile.am b/Makefile.am index ff9e0cdd..9bf0e172 100644 --- a/Makefile.am +++ b/Makefile.am @@ -7,7 +7,7 @@ ACLOCAL_AMFLAGS = -I m4 AM_CPPFLAGS = $(FT2_CFLAGS) $(PTHREAD_CFLAGS) $(MAPNIK_INCLUDES) $(BOOST_CPPFLAGS) $(ICU_CPPFLAGS) $(CAIRO_CFLAGS) $(CAIROMM_CFLAGS) -DSYSTEM_LIBINIPARSER=@SYSTEM_LIBINIPARSER@ -STORE_SOURCES = src/store.c src/store_file.c src/store_file_utils.c src/store_memcached.c src/store_rados.c src/store_ro_http_proxy.c src/store_ro_composite.c src/store_null.c +STORE_SOURCES = src/store.c src/store_file.c src/store_file_utils.c src/store_memcached.c src/store_couchbase.c src/store_rados.c src/store_ro_http_proxy.c src/store_ro_composite.c src/store_null.c STORE_LDFLAGS = $(LIBMEMCACHED_LDFLAGS) $(LIBRADOS_LDFLAGS) $(LIBCURL) $(CAIRO_LIBS) bin_PROGRAMS = renderd render_expired render_list render_speedtest render_old @@ -45,10 +45,10 @@ test: gen_tile_test ./gen_tile_test all-local: - $(APXS) -c $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_null.c + $(APXS) -c $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_couchbase.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_null.c install-mod_tile: mkdir -p $(DESTDIR)`$(APXS) -q LIBEXECDIR` - $(APXS) -S LIBEXECDIR=$(DESTDIR)`$(APXS) -q LIBEXECDIR` -c -i $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_null.c + $(APXS) -S LIBEXECDIR=$(DESTDIR)`$(APXS) -q LIBEXECDIR` -c -i $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_couchbase.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_null.c diff --git a/configure.ac b/configure.ac index f1d05901..f5aa0a90 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ fi # Checks for libraries. # Checks for header files. -AC_CHECK_HEADERS([arpa/inet.h fcntl.h limits.h netdb.h netinet/in.h stdint.h stdlib.h string.h sys/socket.h sys/time.h syslog.h unistd.h utime.h paths.h sys/cdefs.h sys/loadavg.h]) +AC_CHECK_HEADERS([arpa/inet.h fcntl.h limits.h netdb.h netinet/in.h stdint.h stdlib.h string.h sys/socket.h sys/time.h syslog.h unistd.h utime.h paths.h sys/cdefs.h sys/loadavg.h openssl/md5.h]) # Checks for typedefs, structures, and compiler characteristics. AC_C_INLINE diff --git a/extra/Makefile b/extra/Makefile index a20aea88..7ba5930d 100644 --- a/extra/Makefile +++ b/extra/Makefile @@ -1,2 +1,4 @@ -meta2tile: meta2tile.c - gcc -O3 -std=gnu99 -I../includes -I/usr/include meta2tile.c -o meta2tile -lm +COUCHBASE_SRCS=../src/store_couchbase.c ../src/store_memcached.c + +meta2tile: meta2tile.c $(COUCHBASE_SRCS) + gcc -O3 -std=gnu99 -I../includes -I/usr/include $(COUCHBASE_SRCS) meta2tile.c -o meta2tile -lm -lmemcached -lssl -Wall diff --git a/extra/meta2tile.c b/extra/meta2tile.c index 8562f029..00e4fad6 100644 --- a/extra/meta2tile.c +++ b/extra/meta2tile.c @@ -18,17 +18,43 @@ #include #include #include +#include +#include #include "store_file.h" +#include "store_couchbase.h" #include "metatile.h" #define MIN(x,y) ((x)<(y)?(x):(y)) #define META_MAGIC "META" +#define THREAD_USLEEP 1 + +enum Status +{ + THREAD_RUNNING = 0, + THREAD_FREE, + THREAD_COMPUTE +}; + +struct ThreadInfo { + volatile int id; + volatile enum Status status; + volatile char isFinished; + pthread_t pid; + char path[PATH_MAX]; + pthread_mutex_t mutex; + struct storage_backend * store; +}; + +int threadCount = 1; static int verbose = 0; -static int num_render = 0; +static long int num_render = 0; static struct timeval start, end; +char store_conf[255]; +int force=0; + const char *source; const char *target; @@ -41,9 +67,37 @@ int mode = MODE_GLOB; int zoom[MAXZOOM+1]; float bbox[4] = {-180.0, -90.0, 180.0, 90.0}; +void log_message(int log_lvl, const char *format, ...) { + va_list ap; + char *msg = malloc(1000*sizeof(char)); + + va_start(ap, format); + + if (msg) { + vsnprintf(msg, 1000, format, ap); + switch (log_lvl) { + case STORE_LOGLVL_DEBUG: + fprintf(stderr, "debug: %s\n", msg); + break; + case STORE_LOGLVL_INFO: + fprintf(stderr, "info: %s\n", msg); + break; + case STORE_LOGLVL_WARNING: + fprintf(stderr, "WARNING: %s\n", msg); + break; + case STORE_LOGLVL_ERR: + fprintf(stderr, "ERROR: %s\n", msg); + break; + } + free(msg); + fflush(stderr); + } + va_end(ap); +} + int path_to_xyz(const char *path, int *px, int *py, int *pz) { - int i, n, hash[5], x, y, z; + int i, hash[5], x, y, z; char copy[PATH_MAX]; strcpy(copy, path); char *slash = rindex(copy, '/'); @@ -103,13 +157,14 @@ double tiley2lat(int y, int z) return 180.0 / M_PI * atan(0.5 * (exp(n) - exp(-n))); } -int expand_meta(const char *name) +int expand_meta(const char *name, struct storage_backend * store) { int fd; - char header[4096]; int x, y, z; - size_t pos; + size_t pos = 0; void *buf; + char path[PATH_MAX]; + struct stat_info tile_stat; if (path_to_xyz(name, &x, &y, &z)) return -1; @@ -160,68 +215,81 @@ int expand_meta(const char *name) return -5; } - char path[PATH_MAX]; - sprintf(path, "%s/%d", target, z); - if (mkdir(path, 0755) && (errno != EEXIST)) + if (store == NULL) { - fprintf(stderr, "cannot create directory %s: %s\n", path, strerror(errno)); - close(fd); - return -1; + sprintf(path, "%s/%d", target, z); + if (mkdir(path, 0755) && (errno != EEXIST)) + { + fprintf(stderr, "cannot create directory %s: %s\n", path, strerror(errno)); + close(fd); + return -1; + } } - for (int meta = 0; meta < METATILE*METATILE; meta++) - { - int tx = x + (meta / METATILE); - int ty = y + (meta % METATILE); - int output; - - if (ty==y) - { - sprintf(path, "%s/%d/%d", target, z, tx); - if (mkdir(path, 0755) && (errno != EEXIST)) - { - fprintf(stderr, "cannot create directory %s: %s\n", path, strerror(errno)); - close(fd); + if (store != NULL) { + if (!force) tile_stat = store->tile_stat(store, target, "options", x, y, z); + if (force || (tile_stat.size < 0) || (tile_stat.expired)) { + if (store->metatile_write(store, target, "options", x, y, z, buf, st.st_size) == -1) { + close(fd); + fprintf(stderr, "Failed to write data to couchbase %s/%d/%d/%d.png\n", target, x, y, z); return -1; } + if (verbose) printf("Produced metatile: %s/%d/%d/%d.meta\n", target, x, y, z); } - - sprintf(path, "%s/%d/%d/%d.png", target, z, tx, ty); - output = open(path, O_WRONLY | O_TRUNC | O_CREAT, 0666); - if (output == -1) + } else { + for (int meta = 0; meta < METATILE*METATILE; meta++) { - fprintf(stderr, "cannot open %s for writing: %s\n", path, strerror(errno)); - close(fd); - return -1; - } + int tx = x + (meta / METATILE); + int ty = y + (meta % METATILE); + int output; - pos = 0; - while (pos < m->index[meta].size) - { - size_t len = m->index[meta].size - pos; - int written = write(output, buf + pos + m->index[meta].offset, len); - if (written < 0) + if (ty==y) { - fprintf(stderr, "Failed to write data to file %s. Reason: %s\n", path, strerror(errno)); - close(fd); - return -7; - } - else if (written > 0) + sprintf(path, "%s/%d/%d", target, z, tx); + if (mkdir(path, 0755) && (errno != EEXIST)) + { + fprintf(stderr, "cannot create directory %s: %s\n", path, strerror(errno)); + close(fd); + return -1; + } + } + + sprintf(path, "%s/%d/%d/%d.png", target, z, tx, ty); + output = open(path, O_WRONLY | O_TRUNC | O_CREAT, 0666); + if (output == -1) { - pos += written; - } - else + fprintf(stderr, "cannot open %s for writing: %s\n", path, strerror(errno)); + close(fd); + return -1; + } + + pos = 0; + while (pos < m->index[meta].size) { - break; + size_t len = m->index[meta].size - pos; + int written = write(output, buf + pos + m->index[meta].offset, len); + + if (written < 0) + { + fprintf(stderr, "Failed to write data to file %s. Reason: %s\n", path, strerror(errno)); + close(fd); + return -7; + } + else if (written > 0) + { + pos += written; + } + else + { + break; + } } + close(output); + if (verbose) printf("Produced tile: %s\n", path); } - close(output); - if (verbose) printf("Produced tile: %s\n", path); } - munmap(buf, st.st_size); close(fd); - num_render++; return pos; } @@ -239,7 +307,50 @@ void display_rate(struct timeval start, struct timeval end, int num) fflush(NULL); } -static void descend(const char *search, int zoomdone) +static void *thread(void *pArg) +{ + struct ThreadInfo *threadInfo = (struct ThreadInfo *) pArg; +// printf("Started thread %d\n", threadInfo->id); + + pthread_mutex_lock(&threadInfo->mutex); + + threadInfo->status = THREAD_FREE; +// threadInfo->store = init_storage_couchbase(store_conf); + + pthread_mutex_unlock(&threadInfo->mutex); + + while(1) + { + + pthread_mutex_lock(&threadInfo->mutex); + if ((threadInfo->status == THREAD_FREE) && (threadInfo->isFinished == 1)) + { + pthread_mutex_unlock(&threadInfo->mutex); + break; + } + + if (threadInfo->status == THREAD_COMPUTE) + { + pthread_mutex_unlock(&threadInfo->mutex); + + expand_meta(threadInfo->path, threadInfo->store); + + pthread_mutex_lock(&threadInfo->mutex); + threadInfo->status = THREAD_FREE; + pthread_mutex_unlock(&threadInfo->mutex); + } + else + { + pthread_mutex_unlock(&threadInfo->mutex); + usleep(THREAD_USLEEP); + } + } + +// printf("Finished thread %d\n", threadInfo->id); + return NULL; +} + +static void descend(const char *search, int zoomdone, struct ThreadInfo *pThreadInfo) { DIR *tiles = opendir(search); struct dirent *entry; @@ -284,11 +395,36 @@ static void descend(const char *search, int zoomdone) if (stat(path, &b)) continue; if (S_ISDIR(b.st_mode)) { - descend(path, zoomdone || this_is_zoom); + descend(path, zoomdone || this_is_zoom, pThreadInfo); continue; } p = strrchr(path, '.'); - if (p && !strcmp(p, ".meta")) expand_meta(path); + if (p && !strcmp(p, ".meta")) { + while(1) + { + char isFind = 0; + for (int j = 0; j < threadCount; ++j) + { + pthread_mutex_lock(&pThreadInfo[j].mutex); + if (pThreadInfo[j].status == THREAD_FREE) + { + //printf("buff = %s", buff); + strcpy(pThreadInfo[j].path, path); + pThreadInfo[j].status = THREAD_COMPUTE; + isFind = 1; + pthread_mutex_unlock(&pThreadInfo[j].mutex); + break; + } + pthread_mutex_unlock(&pThreadInfo[j].mutex); + } + + if (isFind == 1) + break; + else + usleep(THREAD_USLEEP); + } + num_render++; + } } closedir(tiles); } @@ -296,7 +432,7 @@ static void descend(const char *search, int zoomdone) void usage() { - fprintf(stderr, "Usage: m2t [-m mode] [-b bbox] [-z zoom] sourcedir targetdir\n"); + fprintf(stderr, "Usage: m2t [-m mode] [-b bbox] [-z zoom] [-f force] [-t threads] [-c \"couchbase:{memcached://localhost:11211,memcached://localhost:11212}\"] sourcedir targetdir\n"); fprintf(stderr, "Convert .meta files found in source dir to .png in target dir,\n"); fprintf(stderr, "using the standard \"hash\" type directory (5-level) for meta\n"); fprintf(stderr, "tiles and the z/x/y.png structure (3-level) for output.\n"); @@ -344,13 +480,16 @@ int main(int argc, char **argv) { {"verbose", 0, 0, 'v'}, {"help", 0, 0, 'h'}, + {"couchbase", 1, 0, 'c'}, {"bbox", 1, 0, 'b'}, {"mode", 1, 0, 'm'}, {"zoom", 1, 0, 'z'}, + {"threads", 1, 0, 't'}, + {"force", 0, 0, 'f'}, {0, 0, 0, 0} }; - c = getopt_long(argc, argv, "vhb:m:z:", long_options, &option_index); + c = getopt_long(argc, argv, "vhb:m:z:c:t:f", long_options, &option_index); if (c == -1) break; @@ -377,6 +516,19 @@ int main(int argc, char **argv) return -1; } break; + case 'c': + sprintf(store_conf, "%s", optarg); + if (store_conf == NULL) { + return -1; + } + break; + case 't': + threadCount=atoi(optarg); + if (threadCount <= 0) { + fprintf(stderr, "Invalid number of threads, must be at least 1\n"); + return 1; + } + break; case 'm': if (!strcmp(optarg, "glob")) { @@ -394,12 +546,36 @@ int main(int argc, char **argv) return -1; } break; + case 'f': /* -f, --force */ + force=1; + break; default: fprintf(stderr, "unhandled char '%c'\n", c); break; } } + struct ThreadInfo *pThreadInfo = (struct ThreadInfo *)malloc(sizeof(struct ThreadInfo) * threadCount); + + if (!pThreadInfo) { + fprintf(stderr, "Failed to allocate memory\n"); + return -1; + } + + for (int i = 0; i < threadCount; ++i) + { + pThreadInfo[i].isFinished = 0; + pThreadInfo[i].id = i; + pThreadInfo[i].store = init_storage_couchbase(store_conf); + pThreadInfo[i].status = THREAD_COMPUTE; + pthread_mutex_init(&pThreadInfo[i].mutex, NULL); + if (pthread_create(&pThreadInfo[i].pid, NULL, thread, (void *)&pThreadInfo[i]) != 0) + { + fprintf(stderr, "Failed to start thread\n"); + _exit(-1); + } + } + if (!zoomset) for (int i=0; i<=MAXZOOM; i++) zoom[i]=1; if (optind >= argc-1) @@ -415,7 +591,17 @@ int main(int argc, char **argv) gettimeofday(&start, NULL); - descend(source, 0); + descend(source, 0, pThreadInfo); + + for (int j = 0; j < threadCount; ++j) { + pthread_mutex_lock(&pThreadInfo[j].mutex); + pThreadInfo[j].isFinished = 1; + pthread_mutex_unlock(&pThreadInfo[j].mutex); + } + + for (int i = 0; i < threadCount; ++i) { + pthread_join(pThreadInfo[i].pid, NULL); + } gettimeofday(&end, NULL); printf("\nTotal for all tiles converted\n"); @@ -424,5 +610,13 @@ int main(int argc, char **argv) printf("Total tiles converted: "); display_rate(start, end, num_render * METATILE * METATILE); + for (int i=0; iclose_storage(pThreadInfo[i].store); + } + } + + free(pThreadInfo); + return 0; } diff --git a/includes/daemon.h b/includes/daemon.h index 85fa6476..7c588226 100644 --- a/includes/daemon.h +++ b/includes/daemon.h @@ -18,6 +18,7 @@ extern "C" { typedef struct { char *socketname; + char *pidfile; char *iphostname; int ipport; int num_threads; diff --git a/includes/request_queue.h b/includes/request_queue.h index f8d5e797..a192c8c7 100644 --- a/includes/request_queue.h +++ b/includes/request_queue.h @@ -71,6 +71,8 @@ enum protoCmd request_queue_add_request(struct request_queue * queue, struct ite void request_queue_remove_request(struct request_queue * queue, struct item * request, int render_time); void request_queue_clear_requests_by_fd(struct request_queue * queue, int fd); +enum protoCmd request_queue_reenqueue_request(struct request_queue * queue, struct item *item); + int request_queue_no_requests_queued(struct request_queue * queue, enum protoCmd); void request_queue_copy_stats(struct request_queue * queue, stats_struct * stats); diff --git a/includes/store_couchbase.h b/includes/store_couchbase.h new file mode 100644 index 00000000..3abfd742 --- /dev/null +++ b/includes/store_couchbase.h @@ -0,0 +1,15 @@ +#ifndef STORECOUCHBASE_H +#define STORECOUCHBASE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "store.h" + + struct storage_backend * init_storage_couchbase(const char * connection_string); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/mod_tile.spec b/mod_tile.spec new file mode 100644 index 00000000..94c60fe9 --- /dev/null +++ b/mod_tile.spec @@ -0,0 +1,121 @@ +# +# spec file for package apache2-mod_tile +# +# Copyright (c) 2013 SUSE LINUX Products GmbH, Nuernberg, Germany. +# +# All modifications and additions to the file contributed by third parties +# remain the property of their copyright owners, unless otherwise agreed +# upon. The license for this file, and modifications and additions to the +# file, is the same license as for the pristine package itself (unless the +# license for the pristine package is not an Open Source License, in which +# case the license is the MIT License). An "Open Source License" is a +# license that conforms to the Open Source Definition (Version 1.9) +# published by the Open Source Initiative. + +# Please submit bugfixes or comments via http://bugs.opensuse.org/ +# + +#%define debug_package %{nil} +%define METATILE 8 +%define apxs /usr/sbin/apxs +BuildRequires: httpd-devel + +Name: mod_tile +Version: 0.5 +Release: 20140123%{?dist} +Requires: httpd +Summary: Apache module for map tile handling +License: GPL-2.0+ +Group: Productivity/Networking/Web/Servers +Url: http://wiki.openstreetmap.org/wiki/Mod_tile +Source: mod_tile-%{version}.tar.bz2 +BuildRoot: %{_tmppath}/%{name}-%{version}-build +BuildRequires: autoconf +BuildRequires: automake +BuildRequires: gcc-c++ +BuildRequires: libtool +BuildRequires: make +#BuildRequires: pkgconfig +BuildRequires: boost-devel +BuildRequires: freetype-devel +BuildRequires: libicu-devel >= 4.2 +BuildRequires: proj-devel +BuildRequires: mapnik-devel >= 2.2.0-6 +BuildRequires: openssl-devel + +%description +Mod tile is a system to serve raster tiles for example to use within a slippy map. +It provides a dynamic combination of efficient caching and on the fly rendering. +Due to its dynamic rendering, only a small fraction of overall tiles need to be +kept on disk, reducing the resources required. At the same time, its caching +strategy allows for a high performance serving and can support several thousand +requests per second. + +Mod_tile was originally written for serving the tiles of the main OSM map (Mapnik +layer), but since is being used on a variety of different servers providing maps +ontop of OpenStreetMap data. + +%package -n renderd +Summary: Render daemon for Apache2 map tile module +Group: Productivity/Networking/Web/Servers + +%description -n renderd +Default rendering daemon for mod_tile. + +%prep +%setup -q -n mod_tile +sed -i 's/#define METATILE (8)/#define METATILE (%{METATILE})/' includes/render_config.h + +%build +export CPPFLAGS="-I/usr/include/agg2" +./autogen.sh +%configure \ + --with-apxs="%{apxs}" +%__make %{?_smp_flags} -j8 +cd extra && make + +%post -n renderd -p /sbin/ldconfig +%postun -n renderd -p /sbin/ldconfig + +%install +%makeinstall +make DESTDIR=%{buildroot} install-mod_tile +cp -a extra/meta2tile %{buildroot}/%{_bindir} + +%clean +%{?buildroot:%__rm -rf "%{buildroot}"} + +%files -n renderd +%defattr(-,root,root) +%doc COPYING +#%{_bindir}/convert_meta +%{_bindir}/render* +%{_bindir}/meta2tile +%config %{_sysconfdir}/renderd.conf +%{_libdir}/libiniparser* +%exclude %{_libdir}/libiniparser.so +%exclude %{_libdir}/libiniparser.a +%{_mandir}/man1/render_expired.1.gz +%{_mandir}/man1/render_list.1.gz +%{_mandir}/man1/render_old.1.gz +%{_mandir}/man1/render_speedtest.1.gz +%{_mandir}/man8/renderd.8.gz + +%files +%defattr(-,root,root) +%doc COPYING +%{_libdir}/httpd/modules/mod_tile.so + +%changelog +* Thu Jan 23 2014 kay.diam@gmail.com +- Bump version +* Fri Dec 27 2013 kay.diam@gmail.com +- Bump version and added METATILE const +* Mon Sep 09 2013 kay.diam@gmail.com +- Adaptation for CentOS 6.4 +* Wed Feb 20 2013 opensuse@dstoecker.de +- update to revision 29268 +* Thu Jan 17 2013 BSipos@rkf-eng.com +- Added ICU package version requirement to match mapnik requirement. +* Thu Jul 26 2012 opensuse@dstoecker.de +- initial version diff --git a/renderd.conf b/renderd.conf index b8c5fa87..ec3ae6f6 100644 --- a/renderd.conf +++ b/renderd.conf @@ -1,6 +1,7 @@ [renderd] ;socketname=/var/run/renderd/renderd.sock num_threads=4 +pidfile=/var/run/renderd/renderd.pid tile_dir=/var/lib/mod_tile stats_file=/var/run/renderd/renderd.stats diff --git a/src/daemon.c b/src/daemon.c index 44bbea86..af8e34fd 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -40,15 +40,20 @@ static pthread_t *render_threads; static pthread_t *slave_threads; static struct sigaction sigPipeAction; +static struct sigaction sigTermAction; static pthread_t stats_thread; #endif static int exit_pipe_fd; +static volatile sig_atomic_t stopFlag = 0; static renderd_config config; int noSlaveRenders; +static int check_slaves = 0; +static int live_slaves = 0; +static pthread_mutex_t live_slaves_mu = PTHREAD_MUTEX_INITIALIZER; static const char *cmdStr(enum protoCmd c) { @@ -65,8 +70,34 @@ static const char *cmdStr(enum protoCmd c) } } +void term(int signum) +{ + stopFlag = 1; +} +static int update_live_slaves(int n) { + int res = 0; + int rc = pthread_mutex_lock(&live_slaves_mu); + if (rc) { + syslog(LOG_ERR, "pthread_mutex_lock failed"); + } + live_slaves += n; + res = live_slaves; + + if (!rc) { + rc = pthread_mutex_unlock(&live_slaves_mu); + if (rc) { + syslog(LOG_ERR, "pthread_mutex_unlock failed"); + } + } + + if (n != 0) { + syslog(LOG_DEBUG, "update_live_slaves: %d (%d)", n, res); + } + + return res; +} void send_response(struct item *item, enum protoCmd rsp, int render_time) { struct protocol *req = &item->req; @@ -79,9 +110,9 @@ void send_response(struct item *item, enum protoCmd rsp, int render_time) { if ((item->fd != FD_INVALID) && ((req->cmd == cmdRender) || (req->cmd == cmdRenderPrio) || (req->cmd == cmdRenderBulk))) { req->cmd = rsp; //fprintf(stderr, "Sending message %s to %d\n", cmdStr(rsp), item->fd); - + send_cmd(req, item->fd); - + } prev = item; item = item->duplicates; @@ -96,9 +127,9 @@ enum protoCmd rx_request(struct protocol *req, int fd) // Upgrade version 1 and 2 to version 3 if (req->ver == 1) { strcpy(req->xmlname, "default"); - } + } if (req->ver < 3) { - strcpy(req->mimetype,"image/png"); + strcpy(req->mimetype,"image/png"); strcpy(req->options,""); } else if (req->ver != 3) { syslog(LOG_ERR, "Bad protocol version %d", req->ver); @@ -136,6 +167,11 @@ enum protoCmd rx_request(struct protocol *req, int fd) item->my = item->req.y; #endif + if (check_slaves && update_live_slaves(0) == 0) { + syslog(LOG_ERR, "There are no live slaves"); + return cmdNotDone; + } + return request_queue_add_request(render_request_queue, item); } @@ -165,7 +201,7 @@ void process_loop(int listen_fd) exit_pipe_fd = pipefds[1]; exit_pipe_read = pipefds[0]; - while (1) { + while (!stopFlag) { struct sockaddr_un in_addr; socklen_t in_addrlen = sizeof(in_addr); fd_set rd; @@ -407,6 +443,9 @@ int server_socket_init(renderd_config *sConfig) { struct sockaddr_in6 addrI; mode_t old; int fd; +#ifdef USE_SO_REUSEADDR + int optval; +#endif if (sConfig->ipport > 0) { syslog(LOG_INFO, "Initialising TCP/IP server socket on %s:%i", @@ -416,6 +455,12 @@ int server_socket_init(renderd_config *sConfig) { fprintf(stderr, "failed to create IP socket\n"); exit(2); } + +#ifdef USE_SO_REUSEADDR + optval = 1; + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); +#endif + bzero(&addrI, sizeof(addrI)); addrI.sin6_family = AF_INET6; addrI.sin6_addr = in6addr_any; @@ -463,6 +508,15 @@ int server_socket_init(renderd_config *sConfig) { } +static void reenqueue_item(struct request_queue * queue, struct item *item) { + if (update_live_slaves(0) == 0) { + send_response(item, cmdNotDone, -1); + } + else { + request_queue_reenqueue_request(queue, item); + } +} + /** * This function is used as a the start function for the slave renderer thread. * It pulls a request from the central queue of requests and dispatches it to @@ -476,7 +530,12 @@ void *slave_thread(void * arg) { renderd_config * sConfig = (renderd_config *) arg; int pfd = FD_INVALID; - int retry; + int retry, reenqueue = 0; + // Thread status: + // 0 - unknown + // 1 - ok + // 2 - fail + int thread_status = 0; size_t ret_size; struct protocol * resp; @@ -489,6 +548,10 @@ void *slave_thread(void * arg) { while (1) { if (pfd == FD_INVALID) { + if (thread_status == 1) { + update_live_slaves(-1); // Kick thread until reconnect + thread_status = 2; + } pfd = client_socket_init(sConfig); if (pfd == FD_INVALID) { if (sConfig->ipport > 0) { @@ -505,6 +568,11 @@ void *slave_thread(void * arg) { } } + if (thread_status != 1) { + update_live_slaves(1); + thread_status = 1; + } + enum protoCmd ret; struct item *item = request_queue_fetch_request(render_request_queue); if (item) { @@ -536,6 +604,8 @@ void *slave_thread(void * arg) { free(resp); free(req_slave); close(pfd); + reenqueue_item(render_request_queue, item); + update_live_slaves(-1); return NULL; } @@ -544,9 +614,8 @@ void *slave_thread(void * arg) { pfd = client_socket_init(sConfig); if (pfd == FD_INVALID) { syslog(LOG_ERR, - "Failed to re-connect to render slave, dropping request"); - ret = cmdNotDone; - send_response(item, ret, -1); + "Failed to re-connect to render slave"); + reenqueue_item(render_request_queue, item); break; } } while (retry--); @@ -556,18 +625,24 @@ void *slave_thread(void * arg) { ret_size = 0; retry = 10; + reenqueue = 0; while ((ret_size < sizeof(struct protocol)) && (retry > 0)) { - ret_size = recv(pfd, resp + ret_size, (sizeof(struct protocol) + ret_size += recv(pfd, resp + ret_size, (sizeof(struct protocol) - ret_size), 0); if ((errno == EPIPE) || ret_size == 0) { close(pfd); pfd = FD_INVALID; ret_size = 0; syslog(LOG_ERR, "Pipe to render slave closed"); + reenqueue_item(render_request_queue, item); + reenqueue = 1; break; } retry--; } + if (reenqueue) { + continue; + } if (ret_size < sizeof(struct protocol)) { if (sConfig->ipport > 0) { syslog( LOG_ERR, @@ -606,6 +681,7 @@ void *slave_thread(void * arg) { sleep(1); // TODO: Use an event to indicate there are new requests } } + free(resp); free(req_slave); return NULL; @@ -614,7 +690,7 @@ void *slave_thread(void * arg) { #ifndef MAIN_ALREADY_DEFINED int main(int argc, char **argv) { - int fd, i, j, k; + int fd, i, j, k, n; int c; int foreground=0; @@ -821,6 +897,9 @@ int main(int argc, char **argv) sprintf(buffer, "%s:socketname", name); config_slaves[render_sec].socketname = iniparser_getstring(ini, buffer, (char *) RENDER_SOCKET); + sprintf(buffer, "%s:pidfile", name); + config_slaves[render_sec].pidfile = iniparser_getstring(ini, + buffer, (char *) PIDFILE); sprintf(buffer, "%s:iphostname", name); config_slaves[render_sec].iphostname = iniparser_getstring(ini, buffer, ""); @@ -838,6 +917,7 @@ int main(int argc, char **argv) if (render_sec == active_slave) { config.socketname = config_slaves[render_sec].socketname; + config.pidfile = config_slaves[render_sec].pidfile; config.iphostname = config_slaves[render_sec].iphostname; config.ipport = config_slaves[render_sec].ipport; config.num_threads = config_slaves[render_sec].num_threads; @@ -861,6 +941,7 @@ int main(int argc, char **argv) } else { syslog(LOG_INFO, "config renderd: unix socketname=%s\n", config.socketname); } + syslog(LOG_INFO, "config renderd: pidfile=%s\n", config.pidfile); syslog(LOG_INFO, "config renderd: num_threads=%d\n", config.num_threads); if (active_slave == 0) { syslog(LOG_INFO, "config renderd: num_slaves=%d\n", noSlaveRenders); @@ -917,6 +998,19 @@ int main(int argc, char **argv) exit(6); } + memset(&sigTermAction, 0, sizeof(struct sigaction)); + sigTermAction.sa_handler = term; + if (sigaction(SIGTERM, &sigTermAction, NULL) < 0) { + fprintf(stderr, "failed to register signal handler\n"); + close(fd); + exit(6); + } + if (sigaction(SIGINT, &sigTermAction, NULL) < 0) { + fprintf(stderr, "failed to register signal handler\n"); + close(fd); + exit(6); + } + render_init(config.mapnik_plugins_dir, config.mapnik_font_dir, config.mapnik_font_dir_recurse); /* unless the command line said to run in foreground mode, fork and detach from terminal */ @@ -927,10 +1021,14 @@ int main(int argc, char **argv) fprintf(stderr, "can't daemonize: %s\n", strerror(errno)); } /* write pid file */ - FILE *pidfile = fopen(PIDFILE, "w"); + FILE *pidfile = fopen(config.pidfile, "w"); if (pidfile) { (void) fprintf(pidfile, "%d\n", getpid()); (void) fclose(pidfile); + } else { + syslog(LOG_ERR, "Failed to open %s pidfile: %s", config.pidfile, strerror(errno)); + close(fd); + exit(1); } } @@ -953,26 +1051,38 @@ int main(int argc, char **argv) } if (active_slave == 0) { + if (noSlaveRenders > 0) { + check_slaves = 1; + } //Only the master renderd opens connections to its slaves k = 0; slave_threads = (pthread_t *) malloc(sizeof(pthread_t) * noSlaveRenders); - for (i = 1; i < MAX_SLAVES; i++) { - for (j = 0; j < config_slaves[i].num_threads; j++) { - if (pthread_create(&slave_threads[k++], NULL, slave_thread, - (void *) &config_slaves[i])) { - fprintf(stderr, "error spawning render thread\n"); - close(fd); - exit(7); + j = 0; + do { + n = 0; + for (i = 1; i < MAX_SLAVES; i++) { + if (j < config_slaves[i].num_threads) { + if (pthread_create(&slave_threads[k++], NULL, slave_thread, + (void *) &config_slaves[i])) { + fprintf(stderr, "error spawning render thread\n"); + close(fd); + exit(7); + } + ++n; } } - } + ++j; + } while (n > 0); } process_loop(fd); unlink(config.socketname); close(fd); + + syslog(LOG_DEBUG, "Goodbye!"); + return 0; } #endif diff --git a/src/metatile.cpp b/src/metatile.cpp index c2c435a5..a0b0e041 100644 --- a/src/metatile.cpp +++ b/src/metatile.cpp @@ -116,7 +116,6 @@ void metaTile::save(struct storage_backend * store) { free(metatilebuffer); } - void metaTile::expire_tiles(int sock, char * host, char * uri) { if (sock < 0) { return; diff --git a/src/render_list.c b/src/render_list.c index bc0f032b..8a2cae5e 100644 --- a/src/render_list.c +++ b/src/render_list.c @@ -39,9 +39,32 @@ static int minZoom = 0; static int maxZoom = MAX_ZOOM; static int verbose = 0; static int maxLoad = MAX_LOAD_OLD; +static float bbox[4] = {-180.0, -90.0, 180.0, 90.0}; int work_complete; +int long2tilex(double lon, int z) +{ + return (int)(floor((lon + 180.0) / 360.0 * pow(2.0, z))); +} + +int lat2tiley(double lat, int z) +{ + return (int)(floor((1.0 - log( tan(lat * M_PI/180.0) + 1.0 / cos(lat * M_PI/180.0)) / M_PI) / 2.0 * pow(2.0, z))); +} + +int handle_bbox(char *arg) +{ + char *token = strtok(arg, ","); + int bbi = 0; + while(token && bbi<4) + { + bbox[bbi++] = atof(token); + token = strtok(NULL, ","); + } + return (bbi==4 && token==NULL); +} + void display_rate(struct timeval start, struct timeval end, int num) { int d_s, d_us; @@ -71,6 +94,7 @@ int main(int argc, char **argv) int all=0; int numThreads = 1; int force=0; + int bboxset=0; struct storage_backend * store; struct stat_info s; @@ -91,11 +115,12 @@ int main(int argc, char **argv) {"verbose", 0, 0, 'v'}, {"force", 0, 0, 'f'}, {"all", 0, 0, 'a'}, + {"bbox", 1, 0, 'b'}, {"help", 0, 0, 'h'}, {0, 0, 0, 0} }; - c = getopt_long(argc, argv, "hvaz:Z:x:X:y:Y:s:m:t:n:l:f", long_options, &option_index); + c = getopt_long(argc, argv, "hvab:z:Z:x:X:y:Y:s:m:t:n:l:f", long_options, &option_index); if (c == -1) break; @@ -103,6 +128,14 @@ int main(int argc, char **argv) case 'a': /* -a, --all */ all=1; break; + case 'b': /* -b, --bbox */ + if (!handle_bbox(optarg)) + { + fprintf(stderr, "invalid bbox argument - must be of the form east,south,west,north\n"); + return -1; + } + bboxset = 1; + break; case 's': /* -s, --socket */ free(spath); spath = strdup(optarg); @@ -158,6 +191,7 @@ int main(int argc, char **argv) case 'h': /* -h, --help */ fprintf(stderr, "Usage: render_list [OPTION] ...\n"); fprintf(stderr, " -a, --all render all tiles in given zoom level range instead of reading from STDIN\n"); + fprintf(stderr, " -b, --bbox render all tiles in given bbox (min Longitude, min Latitude, max Longitude, max Latitude) range instead of reading from STDIN\n"); fprintf(stderr, " -f, --force render tiles even if they seem current\n"); fprintf(stderr, " -m, --map=MAP render tiles in this map (defaults to '" XMLCONFIG_DEFAULT "')\n"); fprintf(stderr, " -l, --max-load=LOAD sleep if load is this high (defaults to %d)\n", MAX_LOAD_OLD); @@ -249,6 +283,44 @@ int main(int argc, char **argv) } } } + } else if (bboxset) { + int x, y, z, mask = METATILE - 1; + printf("Rendering bbox (%.6f, %.6f) to (%.6f, %.6f) tiles from zoom %d to zoom %d\n", bbox[0], bbox[1], bbox[2], bbox[3], minZoom, maxZoom); + for (z=minZoom; z <= maxZoom; z++) { + minX = long2tilex(bbox[0], z) & ~mask; + minY = lat2tiley(bbox[1], z) & ~mask; + maxX = long2tilex(bbox[2], z) & ~mask; + maxY = lat2tiley(bbox[3], z) & ~mask; + if (minX < 0 || minY < 0 || maxX < -1 || maxY < -1) { + fprintf(stderr, "Invalid range, x and y values must be >= 0\n"); + fprintf(stderr, "Zoom %d from (%d, %d) to (%d, %d)\n", z, minX, minY, maxX, maxY); + fprintf(stderr, "Exit from zoom %d\n", z); + continue; + } + if (minX > maxX) { + // swap minX and maxX + minX = minX ^ maxX; + maxX = minX ^ maxX; + minX = minX ^ maxX; + } + if (minY > maxY) { + // swap minY and maxY + minY = minY ^ maxY; + maxY = minY ^ maxY; + minY = minY ^ maxY; + } + printf("Rendering bbox tiles for zoom %d from (%d, %d) to (%d, %d)\n", z, minX, minY, maxX, maxY); + for (x=minX; x <= maxX; x+=METATILE) { + for (y=minY; y <= maxY; y+=METATILE) { + if (!force) s = store->tile_stat(store, mapname, "", x, y, z); + if (force || (s.size < 0) || (s.expired)) { + enqueue(mapname, x, y, z); + num_render++; + } + num_all++; + } + } + } } else { while(!feof(stdin)) { int n = fscanf(stdin, "%d %d %d", &x, &y, &z); @@ -301,7 +373,9 @@ int main(int argc, char **argv) } store->close_storage(store); - free(store); + if (store != NULL) { + free(store); + } finish_workers(); free(spath); diff --git a/src/request_queue.c b/src/request_queue.c index 035afb9d..737f0714 100644 --- a/src/request_queue.c +++ b/src/request_queue.c @@ -360,6 +360,12 @@ int request_queue_no_requests_queued(struct request_queue * queue, enum protoCmd return noReq; } +enum protoCmd request_queue_reenqueue_request(struct request_queue * queue, struct item *item) { + syslog(LOG_WARNING, "Reenqueue item"); + request_queue_remove_request(queue, item, 0); + return request_queue_add_request(queue, item); +} + void request_queue_copy_stats(struct request_queue * queue, stats_struct * stats) { pthread_mutex_lock(&(queue->qLock)); memcpy(stats, &(queue->stats), sizeof(stats_struct)); diff --git a/src/store.c b/src/store.c index f9d511c7..f8afa4fd 100644 --- a/src/store.c +++ b/src/store.c @@ -18,6 +18,7 @@ #include "store.h" #include "store_file.h" #include "store_memcached.h" +#include "store_couchbase.h" #include "store_rados.h" #include "store_ro_http_proxy.h" #include "store_ro_composite.h" @@ -93,6 +94,11 @@ struct storage_backend * init_storage_backend(const char * options) { store = init_storage_memcached(options); return store; } + if (strstr(options,"couchbase:{") == options) { + log_message(STORE_LOGLVL_DEBUG, "init_storage_backend: initialising couchbase storage backend at: %s", options); + store = init_storage_couchbase(options); + return store; + } if (strstr(options,"ro_http_proxy://") == options) { log_message(STORE_LOGLVL_DEBUG, "init_storage_backend: initialising ro_http_proxy storage backend at: %s", options); store = init_storage_ro_http_proxy(options); diff --git a/src/store_couchbase.c b/src/store_couchbase.c new file mode 100644 index 00000000..2c07a86c --- /dev/null +++ b/src/store_couchbase.c @@ -0,0 +1,660 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_OPENSSL_MD5_H +#include +#endif + +#define COUCHBASE_WRITE_RETRIES 3 + +#ifdef HAVE_LIBMEMCACHED +#include +#endif + +#include "store.h" +#include "store_memcached.h" +#include "metatile.h" +#include "render_config.h" +#include "protocol.h" + +#if defined(HAVE_LIBMEMCACHED) && defined(HAVE_OPENSSL_MD5_H) + +struct couchbase_ctx { + struct storage_backend * hashes; + struct storage_backend * tiles; +}; + +struct metahash_layout { + int count; // METATILE ^ 2 + unsigned char hash_entry[][MD5_DIGEST_LENGTH]; // md5 entries +}; + +static char * md5_to_ascii(const unsigned char hash[MD5_DIGEST_LENGTH]) { + const char *hex = "0123456789abcdef"; + char *r, result[MD5_DIGEST_LENGTH * 2 + 1]; + int i; + + for (i = 0, r = result; i < MD5_DIGEST_LENGTH; i++) { + *r++ = hex[hash[i] >> 4]; + *r++ = hex[hash[i] & 0xF]; + } + *r = '\0'; + + return strndup(result, MD5_DIGEST_LENGTH*2); +} + +static bool is_md5_in_metahash(const unsigned char *hash, struct metahash_layout *mh) { + int i; + for (i=0; i < mh->count; i++) { + if (memcmp(mh->hash_entry[i], hash, MD5_DIGEST_LENGTH) == 0) { + return true; + } + } + return false; +} + +static void md5_bin(const unsigned char *buf, int length, unsigned char *result) { + MD5_CTX my_md5; + + MD5_Init(&my_md5); + MD5_Update(&my_md5, buf, (unsigned int)length); + MD5_Final(result, &my_md5); +} + +static struct metahash_layout * meta_to_hashes(int x, int y, const char *buf) { + int metahash_len = sizeof(struct metahash_layout) + METATILE*METATILE*sizeof(unsigned char)*MD5_DIGEST_LENGTH; + struct metahash_layout *mh = (struct metahash_layout *)malloc(metahash_len); + struct meta_layout *m = (struct meta_layout *)(buf); + + if (mh == NULL) { + return NULL; + } + + mh->count = METATILE * METATILE; + + int tile_index; + for (tile_index = 0; tile_index < mh->count; tile_index++) { + const size_t tile_offset = m->index[tile_index].offset; + const size_t tile_size = m->index[tile_index].size; + md5_bin((unsigned char *)buf+tile_offset, tile_size, mh->hash_entry[tile_index]); + } + + return mh; +} + +static char * couchbase_xyz_to_storagekey(const char *xmlconfig, int x, int y, int z, char * key) { + int mask; + + mask = METATILE - 1; + x &= ~mask; + y &= ~mask; + + snprintf(key, PATH_MAX - 1, "%s/%d/%d/%d", xmlconfig, x, y, z); + + return key; +} + +static int couchbase_tile_read(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z, char *buf, size_t sz, int * compressed, char * log_msg) { + struct couchbase_ctx * ctx = (struct couchbase_ctx *)(store->storage_ctx); + char meta_path[PATH_MAX]; + uint32_t flags; + size_t len; + size_t md5_len; + memcached_return_t rc; + char * buf_raw; + char * md5; + char * md5_raw; + + int mask = METATILE - 1; + size_t tile_index = (x & mask) * METATILE + (y & mask); + int metahash_len = sizeof(struct metahash_layout) + METATILE*METATILE*sizeof(unsigned char)*MD5_DIGEST_LENGTH; + struct metahash_layout *mh = (struct metahash_layout *)malloc(metahash_len); + + couchbase_xyz_to_storagekey(xmlconfig, x, y, z, meta_path); + + if (mh == NULL) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_read: failed to allocate memory for metahash: %s", meta_path); + return -2; + } + + md5_raw = memcached_get(ctx->hashes->storage_ctx, meta_path, strlen(meta_path), &md5_len, &flags, &rc); + if (rc != MEMCACHED_SUCCESS) { + if (rc != MEMCACHED_NOTFOUND) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_read: failed to read meta %s from couchbase %s", meta_path, memcached_last_error_message(ctx->hashes->storage_ctx)); + } + free(mh); + return -1; + } + if (md5_len != (metahash_len + sizeof(struct stat_info))) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_read: %s meta size %d doesn't equal %d", meta_path, md5_len, metahash_len + sizeof(struct stat_info)); + free(md5_raw); + free(mh); + return -1; + } + + memcpy(mh, md5_raw + sizeof(struct stat_info), metahash_len); + + if (mh->count != METATILE*METATILE) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_read: %s meta count %d doesn't equal %d", meta_path, mh->count, METATILE*METATILE); + free(md5_raw); + free(mh); + return -1; + } + + md5 = md5_to_ascii(mh->hash_entry[tile_index]); + buf_raw = memcached_get(ctx->tiles->storage_ctx, md5, MD5_DIGEST_LENGTH*2, &len, &flags, &rc); + if (rc != MEMCACHED_SUCCESS) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_read: failed to read tile %s (%s) from couchbase %s", meta_path, md5, memcached_last_error_message(ctx->tiles->storage_ctx)); + free(md5_raw); + free(md5); + free(mh); + return -1; + } + + *compressed = 0; + + memcpy(buf, buf_raw, len); + free(md5_raw); + free(md5); + free(buf_raw); + free(mh); + return len; +} + +static struct stat_info couchbase_tile_stat(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z) { + struct couchbase_ctx * ctx = (struct couchbase_ctx *)(store->storage_ctx); + struct stat_info tile_stat; + char meta_path[PATH_MAX]; + size_t md5_len; + uint32_t flags; + memcached_return_t rc; + char * md5_raw; + int metahash_len = sizeof(struct metahash_layout) + METATILE*METATILE*sizeof(unsigned char)*MD5_DIGEST_LENGTH; + struct metahash_layout *mh = (struct metahash_layout *)malloc(metahash_len); + + couchbase_xyz_to_storagekey(xmlconfig, x, y, z, meta_path); + + tile_stat.size = -1; + tile_stat.expired = 0; + tile_stat.mtime = 0; + tile_stat.atime = 0; + tile_stat.ctime = 0; + + if (mh == NULL) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_stat: failed to allocate memory for metahash: %s", meta_path); + free(mh); + return tile_stat; + } + + md5_raw = memcached_get(ctx->hashes->storage_ctx, meta_path, strlen(meta_path), &md5_len, &flags, &rc); + + if (rc != MEMCACHED_SUCCESS) { + if (rc != MEMCACHED_NOTFOUND) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_stat: failed to get meta stat %s from couchbase %s", meta_path, memcached_last_error_message(ctx->hashes->storage_ctx)); + } + free(mh); + return tile_stat; + } + + if (md5_len != (metahash_len + sizeof(struct stat_info))) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_stat: invalid %s meta stat size from couchbase %s, %d != %d", meta_path, memcached_last_error_message(ctx->hashes->storage_ctx), md5_len, metahash_len+sizeof(struct stat_info)); + free(mh); + free(md5_raw); + return tile_stat; + } + + memcpy(mh,md5_raw+sizeof(struct stat_info),metahash_len); + + if (mh->count != METATILE*METATILE) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_stat: %s meta count %d doesn't equal %d", meta_path, mh->count, METATILE*METATILE); + free(mh); + free(md5_raw); + return tile_stat; + } + + memcpy(&tile_stat,md5_raw, sizeof(struct stat_info)); + + free(mh); + free(md5_raw); + return tile_stat; +} + + +static char * couchbase_tile_storage_id(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z, char * string) { + snprintf(string,PATH_MAX - 1, "couchbase:///%s/%d/%d/%d", xmlconfig, x, y, z); + return string; +} + +static int couchbase_metatile_write(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z, const char *buf, int sz) { + struct couchbase_ctx * ctx = (struct couchbase_ctx *)(store->storage_ctx); + struct meta_layout *m = (struct meta_layout *)(buf); + struct metahash_layout *mh; + struct metahash_layout *mh_dedup; + int mh_dedup_len = sizeof(struct metahash_layout); + int mh_dedup_item_len = sizeof(unsigned char)*MD5_DIGEST_LENGTH; + int metahash_len = sizeof(struct metahash_layout) + METATILE*METATILE*sizeof(unsigned char)*MD5_DIGEST_LENGTH; + char meta_path[PATH_MAX]; + unsigned int header_len = sizeof(struct meta_layout) + METATILE*METATILE*sizeof(struct entry); +// char tmp[PATH_MAX]; + struct stat_info tile_stat; + int sz2 = metahash_len + sizeof(tile_stat); + char * buf2 = malloc(sz2); + char * md5; + char * md5_check; + size_t md5_check_len; + uint32_t flags; + memcached_return_t rc; + + struct metahash_layout *mh_old; + struct metahash_layout *mh_old_dedup; + char * md5_old_raw; + size_t md5_old_len; + bool delete_old = false; + + if (buf2 == NULL) { + return -2; + } + + mh = meta_to_hashes(x,y,buf); + mh_dedup = (struct metahash_layout *)malloc(mh_dedup_len); + mh_old_dedup = (struct metahash_layout *)malloc(mh_dedup_len); + + if (mh == NULL || mh_dedup == NULL || mh_old_dedup == NULL) { + free(buf2); + if (mh) free(mh); + if (mh_dedup) free(mh_dedup); + if (mh_old_dedup) free(mh_old_dedup); + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_write: failed to allocate memory for metahash: %s", meta_path); + return -2; + } + + mh_dedup->count = 0; + mh_old_dedup->count = 0; + + tile_stat.expired = 0; + tile_stat.size = sz-header_len; + tile_stat.mtime = time(NULL); + tile_stat.atime = tile_stat.mtime; + tile_stat.ctime = tile_stat.mtime; + + memcpy(buf2, &tile_stat, sizeof(tile_stat)); + memcpy(buf2 + sizeof(tile_stat), mh, metahash_len); + +// log_message(STORE_LOGLVL_DEBUG, "Trying to create and write a metatile to %s", couchbase_tile_storage_id(store, xmlconfig, x, y, z, tmp)); + + snprintf(meta_path,PATH_MAX - 1, "%s/%d/%d/%d", xmlconfig, x, y, z); + + // metahash old get START + md5_old_raw = memcached_get(ctx->hashes->storage_ctx, meta_path, strlen(meta_path), &md5_old_len, &flags, &rc); + if (rc != MEMCACHED_SUCCESS) { + if (rc != MEMCACHED_NOTFOUND) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_write: failed to read old meta %s from couchbase %s", meta_path, memcached_last_error_message(ctx->hashes->storage_ctx)); + } + } else if (md5_old_len != (metahash_len + sizeof(struct stat_info))) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_write: %s old meta size %d doesn't equal %d", meta_path, md5_old_len, metahash_len + sizeof(struct stat_info)); + } else if (md5_old_raw != NULL) { + mh_old = (struct metahash_layout *)malloc(metahash_len); + if (mh_old == NULL) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_write: failed to allocate memory for mh_old: %s", meta_path); + } else { + memcpy(mh_old, md5_old_raw + sizeof(struct stat_info), metahash_len); + if (mh_old->count != METATILE*METATILE) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_tile_write: %s old meta count %d doesn't equal %d", meta_path, mh_old->count, METATILE*METATILE); + free(mh_old); + } else { + delete_old = true; + } + } + } + if (md5_old_raw) free(md5_old_raw); + //metahash old gen END + + int counter = 0; + do { + if (counter > 0) sleep(1); + rc = memcached_set(ctx->hashes->storage_ctx, meta_path, strlen(meta_path), buf2, metahash_len+sizeof(tile_stat), (time_t)0, (uint32_t)0); + counter++; + } while (rc != MEMCACHED_SUCCESS && counter < COUCHBASE_WRITE_RETRIES); + if (rc != MEMCACHED_SUCCESS || counter > 1) { + if (rc != MEMCACHED_SUCCESS) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_write: failed to write meta %s to couchbase %s in %d iterations", meta_path, memcached_last_error_message(ctx->hashes->storage_ctx), counter); + free(mh); + free(mh_dedup); + free(mh_old_dedup); + if (delete_old) free(mh_old); + free(buf2); + return -1; + } else { + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_write: successfully wrote meta %s to couchbase %s in %d iterations", meta_path, memcached_last_error_message(ctx->hashes->storage_ctx), counter); + } + } + +// log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_write: write meta %s to couchbase %s", meta_path, memcached_last_error_message(ctx->hashes->storage_ctx)); + + int tile_index; + for (tile_index = 0; tile_index < mh->count; tile_index++) { + struct metahash_layout *mh_tmp; + + if (delete_old && (mh_old_dedup->count == 0 || !is_md5_in_metahash(mh_old->hash_entry[tile_index],mh_old_dedup))) { + // update dedup array for old metahash data + mh_old_dedup->count++; + mh_tmp = (struct metahash_layout *)realloc(mh_old_dedup,mh_dedup_len+mh_dedup_item_len*mh_old_dedup->count); + if (mh_tmp == NULL) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_write: failed to reallocate memory for mh_old_dedup"); + free(mh); + free(mh_dedup); + free(mh_old_dedup); + free(mh_old); + free(buf2); + return -2; + } else { + mh_old_dedup = mh_tmp; + memcpy(mh_old_dedup->hash_entry[mh_old_dedup->count-1], mh_old->hash_entry[tile_index], MD5_DIGEST_LENGTH); + } + } + + if (mh_dedup->count > 0 && is_md5_in_metahash(mh->hash_entry[tile_index],mh_dedup)) { + continue; + } + + mh_dedup->count++; + mh_tmp = (struct metahash_layout *)realloc(mh_dedup,mh_dedup_len+mh_dedup_item_len*mh_dedup->count); + if (mh_tmp == NULL) { + // update dedup array for metahash data + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_write: failed to reallocate memory for mh_dedup"); + free(mh); + free(mh_dedup); + free(mh_old_dedup); + if (delete_old) free(mh_old); + free(buf2); + return -2; + } else { + mh_dedup = mh_tmp; + memcpy(mh_dedup->hash_entry[mh_dedup->count-1], mh->hash_entry[tile_index], MD5_DIGEST_LENGTH); + } + + counter = 0; + int tx = x + (tile_index / METATILE); + int ty = y + (tile_index % METATILE); + md5 = md5_to_ascii(mh->hash_entry[tile_index]); + + md5_check = memcached_get(ctx->tiles->storage_ctx, md5, MD5_DIGEST_LENGTH*2, &md5_check_len, &flags, &rc); + + // check if tile exists in couchbase and qeuals to new one + if (rc != MEMCACHED_SUCCESS || m->index[tile_index].size != md5_check_len || memcmp(buf+m->index[tile_index].offset, md5_check, md5_check_len) != 0) { + do { + if (counter > 0) sleep(1); + rc = memcached_set(ctx->tiles->storage_ctx, md5, MD5_DIGEST_LENGTH*2, buf+m->index[tile_index].offset, m->index[tile_index].size, (time_t)0, (uint32_t)0); + counter++; + } while (rc != MEMCACHED_SUCCESS && counter < COUCHBASE_WRITE_RETRIES); + if (rc != MEMCACHED_SUCCESS || counter > 1) { + if (rc != MEMCACHED_SUCCESS) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_write: failed to write tile %d/%d/%d to couchbase %s in %d iterations", tx, ty, z, memcached_last_error_message(ctx->tiles->storage_ctx), counter); + } else { + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_write: successfully wrote tile %d/%d/%d to couchbase %s in %d iterations", tx, ty, z, memcached_last_error_message(ctx->tiles->storage_ctx), counter); + } + } + } + if (md5_check) free(md5_check); + free(md5); + } + + if (delete_old) { + // delete old tiles + for (tile_index = 0; tile_index < mh_old_dedup->count; tile_index++) { + if (!is_md5_in_metahash(mh_old_dedup->hash_entry[tile_index],mh_dedup)) { + md5 = md5_to_ascii(mh_old_dedup->hash_entry[tile_index]); + rc = memcached_delete(ctx->tiles->storage_ctx, md5, MD5_DIGEST_LENGTH*2, 0); + if (rc != MEMCACHED_SUCCESS && rc != MEMCACHED_NOTFOUND) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_write: failed to delete old tile %s while updating %s in couchbase %s", md5, meta_path, memcached_last_error_message(ctx->tiles->storage_ctx)); + } + free(md5); + } + } + free(mh_old); + } + + free(mh); + free(mh_dedup); + free(mh_old_dedup); + free(buf2); + + memcached_flush_buffers(ctx->hashes->storage_ctx); + memcached_flush_buffers(ctx->tiles->storage_ctx); + return sz; +} + +static int couchbase_metatile_delete(struct storage_backend * store, const char *xmlconfig, int x, int y, int z) { + struct couchbase_ctx * ctx = (struct couchbase_ctx *)(store->storage_ctx); + char meta_path[PATH_MAX]; + int metahash_len = sizeof(struct metahash_layout) + METATILE*METATILE*sizeof(unsigned char)*MD5_DIGEST_LENGTH; + struct metahash_layout *mh = (struct metahash_layout *)malloc(metahash_len); + int mh_dedup_len = sizeof(struct metahash_layout); + int mh_dedup_item_len = sizeof(unsigned char)*MD5_DIGEST_LENGTH; + struct metahash_layout *mh_dedup = (struct metahash_layout *)malloc(mh_dedup_len); + char * md5; + char * md5_raw; + size_t md5_raw_len; + uint32_t flags; + memcached_return_t rc; + + couchbase_xyz_to_storagekey(xmlconfig, x, y, z, meta_path); + + if (mh == NULL || mh_dedup == NULL) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_delete: failed to allocate memory for metahash: %s", meta_path); + if (mh) free(mh); + if (mh_dedup) free(mh_dedup); + return -2; + } + + mh_dedup->count = 0; + + md5_raw = memcached_get(ctx->hashes->storage_ctx, meta_path, strlen(meta_path), &md5_raw_len, &flags, &rc); + + if (rc != MEMCACHED_SUCCESS) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_delete: failed to read meta %s from couchbase %s", meta_path, memcached_last_error_message(ctx->hashes->storage_ctx)); + free(mh); + free(mh_dedup); + return -1; + } + if (md5_raw_len != (metahash_len + sizeof(struct stat_info))) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_delete: %s meta size %d doesn't equal %d", meta_path, md5_raw_len, metahash_len + sizeof(struct stat_info)); + free(md5_raw); + free(mh); + free(mh_dedup); + return -1; + } + + memcpy(mh, md5_raw + sizeof(struct stat_info), metahash_len); + + if (mh->count != METATILE*METATILE) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_delete: %s meta count %d doesn't equal %d", meta_path, mh->count, METATILE*METATILE); + free(md5_raw); + free(mh); + free(mh_dedup); + return -1; + } + + int tile_index; + for (tile_index = 0; tile_index < mh->count; tile_index++) { + if (mh_dedup->count > 0 && is_md5_in_metahash(mh->hash_entry[tile_index],mh_dedup)) { + continue; + } + + struct metahash_layout *mh_tmp; + mh_dedup->count++; + mh_tmp = (struct metahash_layout *)realloc(mh_dedup,mh_dedup_len+mh_dedup_item_len*mh_dedup->count); + if (mh_tmp == NULL) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_delete: failed to reallocate memory for mh_dedup"); + free(mh); + free(mh_dedup); + return -2; + } else { + mh_dedup = mh_tmp; + memcpy(mh_dedup->hash_entry[mh_dedup->count-1], mh->hash_entry[tile_index], MD5_DIGEST_LENGTH); + } + + int tx = x + (tile_index / METATILE); + int ty = y + (tile_index % METATILE); + md5 = md5_to_ascii(mh->hash_entry[tile_index]); + rc = memcached_delete(ctx->tiles->storage_ctx, md5, MD5_DIGEST_LENGTH*2, 0); + if (rc != MEMCACHED_SUCCESS && rc != MEMCACHED_NOTFOUND) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_delete: failed to delete tile %d/%d/%d from couchbase %s", tx, ty, z, memcached_last_error_message(ctx->tiles->storage_ctx)); + free(md5_raw); + free(md5); + free(mh); + free(mh_dedup); + return -1; + } + free(md5); + } + + rc = memcached_delete(ctx->hashes->storage_ctx, meta_path, strlen(meta_path), 0); + + if (rc != MEMCACHED_SUCCESS && rc != MEMCACHED_NOTFOUND) { + log_message(STORE_LOGLVL_DEBUG,"couchbase_metatile_delete: failed to delete meta %s from couchbase %s", meta_path, memcached_last_error_message(ctx->hashes->storage_ctx)); + free(md5_raw); + free(mh); + free(mh_dedup); + return -1; + } + + free(md5_raw); + free(mh); + free(mh_dedup); + + return 0; +} + +static int couchbase_metatile_expire(struct storage_backend * store, const char *xmlconfig, int x, int y, int z) { + struct couchbase_ctx * ctx = (struct couchbase_ctx *)(store->storage_ctx); + char meta_path[PATH_MAX]; + char * buf; + size_t len; + uint32_t flags; + uint64_t cas; + memcached_return_t rc; + + couchbase_xyz_to_storagekey(xmlconfig, x, y, z, meta_path); + buf = memcached_get(ctx->hashes->storage_ctx, meta_path, strlen(meta_path), &len, &flags, &rc); + if (rc != MEMCACHED_SUCCESS) { + return -1; + } + + //cas = memcached_result_cas(&rc); + + ((struct stat_info *)buf)->expired = 1; + + rc = memcached_cas(ctx->hashes->storage_ctx, meta_path, strlen(meta_path), buf, len, 0, flags, cas); + + if (rc != MEMCACHED_SUCCESS) { + free(buf); + return -1; + } + + free(buf); + return 0; +} + +static int couchbase_close_storage(struct storage_backend * store) { + struct couchbase_ctx * ctx = (struct couchbase_ctx *)(store->storage_ctx); + + ctx->hashes->close_storage(ctx->hashes); + ctx->tiles->close_storage(ctx->tiles); + + free(ctx); + free(store); + return 0; +} +#endif //Have memcached / openssl_md5 + +struct storage_backend * init_storage_couchbase(const char * connection_string) { + +#if !defined(HAVE_LIBMEMCACHED) || !defined(HAVE_OPENSSL_MD5_H) + log_message(STORE_LOGLVL_ERR,"init_storage_couchbase: Support for memcached/openssl has not been compiled into this program"); + return NULL; +#else + struct storage_backend * store = malloc(sizeof(struct storage_backend)); + struct couchbase_ctx * ctx = malloc(sizeof(struct couchbase_ctx)); + char * connection_string_hashes; + char * connection_string_tiles; + int len; + + log_message(STORE_LOGLVL_DEBUG,"init_storage_couchbase: initialising couchbase storage backend for %s", connection_string); + + if (!store || !ctx) { + log_message(STORE_LOGLVL_ERR,"init_storage_couchbase: failed to allocate memory for context"); + if (store) free(store); + if (ctx) free(ctx); + return NULL; + } + + connection_string_tiles = strstr(connection_string,","); + if (connection_string_tiles == NULL) { + log_message(STORE_LOGLVL_ERR,"init_storage_couchbase: failed to parse configuration string"); + free(ctx); + free(store); + return NULL; + } + + len = strlen(connection_string) - strlen("couchbase:{") - strlen(connection_string_tiles); + connection_string_hashes = malloc(len + 1); + memcpy(connection_string_hashes,connection_string + strlen("couchbase:{"), len); + connection_string_hashes[len] = 0; + connection_string_tiles = strdup(connection_string_tiles + 1); + connection_string_tiles[strlen(connection_string_tiles) - 1] = 0; + + log_message(STORE_LOGLVL_DEBUG,"init_storage_couchbase: Hashes memcached storage backend: %s", connection_string_hashes); + log_message(STORE_LOGLVL_DEBUG,"init_storage_couchbase: Tiles memcached storage backend: %s", connection_string_tiles); + + if (strstr(connection_string_hashes,"memcached://") == NULL || strstr(connection_string_tiles,"memcached://") == NULL) { + log_message(STORE_LOGLVL_ERR,"init_storage_couchbase: failed to parse configuration string"); + free(connection_string_hashes); + free(connection_string_tiles); + free(ctx); + free(store); + return NULL; + } + + ctx->hashes = init_storage_memcached(connection_string_hashes); + if (ctx->hashes == NULL) { + log_message(STORE_LOGLVL_ERR,"init_storage_couchbase: failed to initialise hashes storage backend"); + free(connection_string_hashes); + free(connection_string_tiles); + free(ctx); + free(store); + return NULL; + } + + ctx->tiles = init_storage_memcached(connection_string_tiles); + if (ctx->tiles == NULL) { + log_message(STORE_LOGLVL_ERR,"init_storage_couchbase: failed to initialise tiles storage backend"); + ctx->hashes->close_storage(ctx->hashes); + free(connection_string_hashes); + free(connection_string_tiles); + free(ctx); + free(store); + return NULL; + } + + store->storage_ctx = ctx; + + store->tile_read = &couchbase_tile_read; + store->tile_stat = &couchbase_tile_stat; + store->metatile_write = &couchbase_metatile_write; + store->metatile_delete = &couchbase_metatile_delete; + store->metatile_expire = &couchbase_metatile_expire; + store->tile_storage_id = &couchbase_tile_storage_id; + store->close_storage = &couchbase_close_storage; + + free(connection_string_hashes); + free(connection_string_tiles); + + return store; +#endif +} diff --git a/src/store_memcached.c b/src/store_memcached.c index 9b79783b..c1783a21 100644 --- a/src/store_memcached.c +++ b/src/store_memcached.c @@ -15,6 +15,8 @@ #include #include +#define USE_BINARY_PROTOCOL + #ifdef HAVE_LIBMEMCACHED #include #endif @@ -38,7 +40,7 @@ static char * memcached_xyz_to_storagekey(const char *xmlconfig, int x, int y, i return key; } -static int memcached_tile_read(struct storage_backend * store, const char *xmlconfig, int x, int y, int z, char *buf, size_t sz, int * compressed, char * log_msg) { +static int memcached_tile_read(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z, char *buf, size_t sz, int * compressed, char * log_msg) { char meta_path[PATH_MAX]; int meta_offset; @@ -97,7 +99,7 @@ static int memcached_tile_read(struct storage_backend * store, const char *xmlco return tile_size; } -static struct stat_info memcached_tile_stat(struct storage_backend * store, const char *xmlconfig, int x, int y, int z) { +static struct stat_info memcached_tile_stat(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z) { struct stat_info tile_stat; char meta_path[PATH_MAX]; unsigned int header_len = sizeof(struct meta_layout) + METATILE*METATILE*sizeof(struct entry); @@ -134,13 +136,13 @@ static struct stat_info memcached_tile_stat(struct storage_backend * store, cons } -static char * memcached_tile_storage_id(struct storage_backend * store, const char *xmlconfig, int x, int y, int z, char * string) { +static char * memcached_tile_storage_id(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z, char * string) { snprintf(string,PATH_MAX - 1, "memcached:///%s/%d/%d/%d.meta", xmlconfig, x, y, z); return string; } -static int memcached_metatile_write(struct storage_backend * store, const char *xmlconfig, int x, int y, int z, const char *buf, int sz) { +static int memcached_metatile_write(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z, const char *buf, int sz) { char meta_path[PATH_MAX]; char tmp[PATH_MAX]; struct stat_info tile_stat; @@ -161,7 +163,7 @@ static int memcached_metatile_write(struct storage_backend * store, const char * memcpy(buf2, &tile_stat, sizeof(tile_stat)); memcpy(buf2 + sizeof(tile_stat), buf, sz); - log_message(STORE_LOGLVL_DEBUG, "Trying to create and write a metatile to %s\n", memcached_tile_storage_id(store, xmlconfig, x, y, z, tmp)); + log_message(STORE_LOGLVL_DEBUG, "Trying to create and write a metatile to %s\n", memcached_tile_storage_id(store, xmlconfig, options, x, y, z, tmp)); snprintf(meta_path,PATH_MAX - 1, "%s/%d/%d/%d.meta", xmlconfig, x, y, z); @@ -235,15 +237,33 @@ struct storage_backend * init_storage_memcached(const char * connection_string) #else struct storage_backend * store = malloc(sizeof(struct storage_backend)); memcached_st * ctx; - char * connection_str = "--server=localhost"; + char * connection_str; + int len; if (store == NULL) { log_message(STORE_LOGLVL_ERR,"init_storage_memcached: Failed to allocate memory for storage backend"); return NULL; } - ctx = memcached(connection_str, strlen(connection_str)); + +#ifdef USE_BINARY_PROTOCOL + len = strlen("--server= --binary-protocol") + strlen(connection_string) - strlen("memcached://"); +#else + len = strlen("--server=") + strlen(connection_string) - strlen("memcached://"); +#endif + connection_str = malloc(len + 1); + memcpy(connection_str,"--server=", strlen("--server=")); + memcpy(connection_str+strlen("--server="),connection_string + strlen("memcached://"), len-strlen("--server=")); +#ifdef USE_BINARY_PROTOCOL + memcpy(connection_str+len-strlen(" --binary-protocol")," --binary-protocol", strlen(" --binary-protocol")); +#endif + connection_str[len] = 0; + + log_message(STORE_LOGLVL_DEBUG,"init_storage_memcached: initialization string: %s", connection_str); + + ctx = memcached(connection_str, len); if (ctx == NULL) { - log_message(STORE_LOGLVL_ERR,"init_storage_memcached: Failed to create memcached ctx"); + log_message(STORE_LOGLVL_ERR,"init_storage_memcached: Failed to create memcached ctx: %s", connection_str); + free(connection_str); free(store); return NULL; } @@ -257,6 +277,7 @@ struct storage_backend * init_storage_memcached(const char * connection_string) store->tile_storage_id = &memcached_tile_storage_id; store->close_storage = &memcached_close_storage; + free(connection_str); return store; #endif } diff --git a/src/store_rados.c b/src/store_rados.c index d63ebe66..e18463d8 100644 --- a/src/store_rados.c +++ b/src/store_rados.c @@ -308,6 +308,7 @@ struct storage_backend * init_storage_rados(const char * connection_string) { int i; if (ctx == NULL) { + free(store); return NULL; } @@ -323,6 +324,7 @@ struct storage_backend * init_storage_rados(const char * connection_string) { log_message(STORE_LOGLVL_ERR,"init_storage_rados: cannot create a cluster handle: %s", strerror(-err)); free(ctx); free(store); + free(conf); return NULL; } @@ -331,6 +333,7 @@ struct storage_backend * init_storage_rados(const char * connection_string) { log_message(STORE_LOGLVL_ERR,"init_storage_rados: failed to read rados config file %s: %s", conf, strerror(-err)); free(ctx); free(store); + free(conf); return NULL; } pthread_mutex_lock(&qLock); @@ -340,6 +343,7 @@ struct storage_backend * init_storage_rados(const char * connection_string) { log_message(STORE_LOGLVL_ERR,"init_storage_rados: failed to connect to rados cluster: %s", strerror(-err)); free(ctx); free(store); + free(conf); return NULL; } @@ -349,6 +353,7 @@ struct storage_backend * init_storage_rados(const char * connection_string) { rados_shutdown(ctx->cluster); free(ctx); free(store); + free(conf); return NULL; } @@ -360,6 +365,7 @@ struct storage_backend * init_storage_rados(const char * connection_string) { rados_shutdown(ctx->cluster); free(ctx); free(store); + free(conf); return NULL; } diff --git a/src/store_ro_composite.c b/src/store_ro_composite.c index 563e445a..67da9447 100644 --- a/src/store_ro_composite.c +++ b/src/store_ro_composite.c @@ -220,6 +220,8 @@ struct storage_backend * init_storage_ro_composite(const char * connection_strin ctx->store_primary = init_storage_backend(tmp + 1); if (ctx->store_primary == NULL) { log_message(STORE_LOGLVL_ERR,"init_storage_ro_composite: failed to initialise primary storage backend"); + free(connection_string_primary); + free(connection_string_secondary); free(ctx); free(store); return NULL; @@ -232,6 +234,8 @@ struct storage_backend * init_storage_ro_composite(const char * connection_strin if (ctx->store_secondary == NULL) { log_message(STORE_LOGLVL_ERR,"init_storage_ro_composite: failed to initialise secondary storage backend"); ctx->store_primary->close_storage(ctx->store_primary); + free(connection_string_primary); + free(connection_string_secondary); free(ctx); free(store); return NULL; @@ -249,6 +253,9 @@ struct storage_backend * init_storage_ro_composite(const char * connection_strin store->tile_storage_id = &ro_composite_tile_storage_id; store->close_storage = &ro_composite_close_storage; + free(connection_string_primary); + free(connection_string_secondary); + return store; #endif }