From b63c30fcde02636aa71d02358eb505f747830966 Mon Sep 17 00:00:00 2001 From: Ryan Graham Date: Thu, 8 Aug 2019 17:09:25 -0700 Subject: [PATCH 1/3] c: optional code for fadvise/readahead More performance tuning that theoretically helps but doesn't seem to move the needle. --- main.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/main.c b/main.c index b63a46d..6b8079d 100644 --- a/main.c +++ b/main.c @@ -14,6 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ +#if __linux__ +// allow readahead(2) +#define _GNU_SOURCE +#endif + #include #include #include @@ -328,6 +333,37 @@ static int_fast32_t scan_all_slow(const unsigned char *buf, const unsigned char return count; } +static void fd_readahead(int fd, off_t offset, size_t len) { +#ifndef DO_READAHEAD + return; +#endif +#if __linux__ + if (0 > readahead(fd, offset, len)) { + perror("readahead"); + } +#elif __APPLE__ + struct radvisory radvise = { .ra_offset = offset, .ra_count = len }; + if (0 > fcntl(fd, F_RDADVISE, &radvise)) { + perror("inner radvise"); + } +#endif + return; +} + +static void fd_advise(int fd) { +#ifndef DO_ADVISE + return; +#endif +#if __linux__ + // same advise as would give mmap + posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); + posix_fadvise(fd, 0, 0, POSIX_FADV_WILLNEED); +#elif __APPLE__ + // enable readahead on this file + fcntl(fd, F_RDAHEAD, 1); +#endif +} + int main(int argc, const char *argv[]) { // 256k sounds nice. // 64*4k pages or 128*2k pages or 512*512b fs blocks @@ -336,7 +372,7 @@ int main(int argc, const char *argv[]) { const size_t MAX_BUF = 64*4096; unsigned char buf[MAX_BUF]; int_fast32_t remainder = 0; - INST(size_t total_read = 0); + size_t total_read = 0; ssize_t nread = 0; INST(int scans = 0); int_fast32_t max_scan = 0; @@ -344,9 +380,17 @@ int main(int argc, const char *argv[]) { if (argc > 1) { fd = open(argv[1], O_RDONLY); } + + // tell the OS to expect sequential reads + fd_advise(fd); + assert(fd >= 0); while ((nread = read(fd, buf+remainder, MAX_BUF-remainder)) > 0) { - INST(total_read += nread); + total_read += nread; + // tell kernel to queue up the next read while we process the current one + if (nread == MAX_BUF-remainder) { + fd_readahead(fd, total_read, MAX_BUF); + } max_scan = nread + remainder; if (max_scan < 41) { remainder = max_scan; From a0d365d07261e59b741087d50def6a39fff15d3f Mon Sep 17 00:00:00 2001 From: Ryan Graham Date: Thu, 8 Aug 2019 17:17:02 -0700 Subject: [PATCH 2/3] c: optionally force buffer to be page aligned In theory this makes certain operations more efficient. --- main.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/main.c b/main.c index 6b8079d..a08b9c9 100644 --- a/main.c +++ b/main.c @@ -27,6 +27,9 @@ limitations under the License. #include #include #include +#if DO_PAGE_ALIGNED +#include +#endif #ifndef DO_INST #define DO_INST 0 @@ -365,12 +368,22 @@ static void fd_advise(int fd) { } int main(int argc, const char *argv[]) { +#if DO_PAGE_ALIGNED + int page_size = getpagesize(); + const size_t MAX_BUF = page_size * 64; + unsigned char *buf; + if (0 > posix_memalign((void**)&buf, page_size, MAX_BUF)) { + perror("memalign"); + return 20; + } +#else // 256k sounds nice. // 64*4k pages or 128*2k pages or 512*512b fs blocks // small enough for stack allocation // small enough to fit in cache on modern CPU const size_t MAX_BUF = 64*4096; unsigned char buf[MAX_BUF]; +#endif int_fast32_t remainder = 0; size_t total_read = 0; ssize_t nread = 0; From 1eceed024f0aab361f2ad6bc9e359ee651e47889 Mon Sep 17 00:00:00 2001 From: Ryan Graham Date: Wed, 7 Aug 2019 15:32:57 -0700 Subject: [PATCH 3/3] c: option to use mmap when given a file name mmap() is supposed to be fast, right? Turns out we skip so aggressively that the overhead of the extra book keeping and round-tripping with the kernel that comes with mmap actually ends up costing us more than any gains. This is somewhat surprising given that mmap() ends up being a slight performance boost for ripgrep when dealing with large input files and that's the specific case I tested with. What I suspect is happening is that we are skipping so aggressively that the OS hasn't been able to read the next page for us by the time we finish with the current one. --- Makefile | 7 +++++++ main.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/Makefile b/Makefile index 939ac44..248c993 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,9 @@ scan-c: main.c scan-c-fast-simd: main.c $(CC) $(SIMD_CFLAGS) -O3 -DNDEBUG -o $@ $< +scan-c-fast-mmap: main.c + $(CC) $(FAST_CFLAGS) -DUSE_MMAP=1 -o $@ $< + scan-c-counters: main.c $(CC) $(SIMD_CFLAGS) -O3 -DDO_INST=1 -o $@ $< @@ -47,6 +50,10 @@ time-simd: scan-c-fast-simd time ./scan-c-fast-simd < raw.tar > c.txt diff -q c.txt prev.txt || diff c.txt prev.txt | wc -l +time-mmap: scan-c-fast-mmap + time ./scan-c-fast-mmap $(SAMPLE) > c.txt + diff -q c.txt prev.txt || diff c.txt prev.txt | wc -l + time: scan-c-fast time ./scan-c-fast < raw.tar > c.txt diff -q c.txt prev.txt || diff c.txt prev.txt | wc -l diff --git a/main.c b/main.c index a08b9c9..f61c52e 100644 --- a/main.c +++ b/main.c @@ -30,6 +30,8 @@ limitations under the License. #if DO_PAGE_ALIGNED #include #endif +#include +#include #ifndef DO_INST #define DO_INST 0 @@ -336,6 +338,42 @@ static int_fast32_t scan_all_slow(const unsigned char *buf, const unsigned char return count; } +#if USE_MMAP +static int mmap_scan(int fd) { + struct stat st; + int32_t remainder = 0; + + if (0 > fstat(fd, &st)) { + perror("fstat failed"); + return 1; + } +// MAP_POPULATE is only available on Linux, but it is also practically required +// on Linux to make mmap() even half as fast as read() for our usage. +#if ! __linux__ +#define MAP_POPULATE 0 +#endif + const unsigned char * buf = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE|MAP_POPULATE, fd, 0); + if (buf == MAP_FAILED) { + perror("mmap failed"); + return 2; + } + // madvise helps when the file isn't already cached, but that's about it + if (0 > madvise((void*)buf, st.st_size, MADV_SEQUENTIAL|MADV_WILLNEED)) { + perror("madvise failed"); + return 3; + } + remainder = scan_slice_fast(buf, buf+st.st_size); + if (remainder >= 40) { + scan_all_slow((buf+ st.st_size) - remainder, buf+st.st_size); + } + if (munmap((void*)buf, st.st_size) != 0) { + return 4; + } + close(fd); + return 0; +} +#endif + static void fd_readahead(int fd, off_t offset, size_t len) { #ifndef DO_READAHEAD return; @@ -392,6 +430,13 @@ int main(int argc, const char *argv[]) { int fd = 0; if (argc > 1) { fd = open(argv[1], O_RDONLY); + if (fd < 0) { + perror("open failed"); + return fd; + } +#if USE_MMAP + return mmap_scan(fd); +#endif } // tell the OS to expect sequential reads