[packages/rsync] track rsync-fadvise.patch locally (in this repo)
glen
glen at pld-linux.org
Mon Dec 3 23:30:49 CET 2018
commit 9cb7906fb8c9c2e334ac71ede1ac2a8ad21fde0f
Author: Elan Ruusamäe <glen at pld-linux.org>
Date: Tue Dec 4 00:26:12 2018 +0200
track rsync-fadvise.patch locally (in this repo)
rsync-fadvise.patch | 568 ++++++++++++++++++++++++++++++++++++++++++++++++++++
rsync.spec | 15 +-
2 files changed, 575 insertions(+), 8 deletions(-)
---
diff --git a/rsync.spec b/rsync.spec
index cbf6e1d..fdf9453 100644
--- a/rsync.spec
+++ b/rsync.spec
@@ -27,16 +27,15 @@ License: GPL v3+
Group: Networking/Utilities
Source0: https://rsync.samba.org/ftp/rsync/%{name}-%{version}.tar.gz
# Source0-md5: 1581a588fde9d89f6bc6201e8129afaf
-Source1: https://rsync.samba.org/ftp/rsync/rsync-patches-%{version}.tar.gz
+Source1: https://rsync.samba.org/ftp/rsync/%{name}-patches-%{version}.tar.gz
# Source1-md5: 753fc37ffc277571c69135e8bc5fae9d
Source2: %{name}.inet
Source3: %{name}.init
Source4: %{name}.sysconfig
Source5: %{name}d.logrotate
Patch0: %{name}-config.patch
-Patch1: https://tobi.oetiker.ch/patches/rsync-3.1.2-1-fadvise.patch
-# Patch1-md5: b055468e7cbc5d829aa943f07916d458
-Patch2: rsync-noatime.patch
+Patch1: %{name}-fadvise.patch
+Patch2: %{name}-noatime.patch
URL: https://rsync.samba.org/
BuildRequires: acl-devel
BuildRequires: autoconf >= 2.59
@@ -216,10 +215,10 @@ cat << 'EOF' > $RPM_BUILD_ROOT/etc/env.d/RSYNC_PASSWORD
#RSYNC_PASSWORD=
EOF
-install %{SOURCE2} $RPM_BUILD_ROOT/etc/sysconfig/rc-inetd/rsyncd
-install %{SOURCE3} $RPM_BUILD_ROOT/etc/rc.d/init.d/rsyncd
-install %{SOURCE4} $RPM_BUILD_ROOT/etc/sysconfig/rsyncd
-install %{SOURCE5} $RPM_BUILD_ROOT/etc/logrotate.d/rsyncd
+install -p %{SOURCE3} $RPM_BUILD_ROOT/etc/rc.d/init.d/rsyncd
+cp -p %{SOURCE2} $RPM_BUILD_ROOT/etc/sysconfig/rc-inetd/rsyncd
+cp -p %{SOURCE4} $RPM_BUILD_ROOT/etc/sysconfig/rsyncd
+cp -p %{SOURCE5} $RPM_BUILD_ROOT/etc/logrotate.d/rsyncd
%clean
rm -rf $RPM_BUILD_ROOT
diff --git a/rsync-fadvise.patch b/rsync-fadvise.patch
new file mode 100644
index 0000000..24e1f9b
--- /dev/null
+++ b/rsync-fadvise.patch
@@ -0,0 +1,568 @@
+diff -ru rsync-3.1.2.orig/checksum.c rsync-3.1.2/checksum.c
+--- rsync-3.1.2.orig/checksum.c 2015-08-08 22:47:03.000000000 +0300
++++ rsync-3.1.2/checksum.c 2016-10-24 15:38:28.002415712 +0300
+@@ -24,6 +24,9 @@
+ extern int checksum_seed;
+ extern int protocol_version;
+ extern int proper_seed_order;
++#ifdef WITH_DROP_CACHE
++#define close(fd) fadv_close(fd)
++#endif
+
+ /*
+ a simple 32 bit checksum that can be upadted from either end
+diff -ru rsync-3.1.2.orig/cleanup.c rsync-3.1.2/cleanup.c
+--- rsync-3.1.2.orig/cleanup.c 2015-08-08 22:47:03.000000000 +0300
++++ rsync-3.1.2/cleanup.c 2016-10-24 15:38:28.002415712 +0300
+@@ -53,7 +53,11 @@
+ int fd;
+ int ret;
+ STRUCT_STAT st;
+-
++#endif
++#ifdef WITH_DROP_CACHE
++ fadv_close_all();
++#endif
++#ifdef SHUTDOWN_ALL_SOCKETS
+ max_fd = sysconf(_SC_OPEN_MAX) - 1;
+ for (fd = max_fd; fd >= 0; fd--) {
+ if ((ret = do_fstat(fd, &st)) == 0) {
+diff -ru rsync-3.1.2.orig/config.h.in rsync-3.1.2/config.h.in
+--- rsync-3.1.2.orig/config.h.in 2015-12-21 22:20:53.000000000 +0200
++++ rsync-3.1.2/config.h.in 2016-10-24 15:38:28.006415712 +0300
+@@ -275,6 +275,9 @@
+ /* Define to 1 if you have the <memory.h> header file. */
+ #undef HAVE_MEMORY_H
+
++/* Define to 1 if you have the `mincore' function. */
++#undef HAVE_MINCORE
++
+ /* Define to 1 if you have the `mkfifo' function. */
+ #undef HAVE_MKFIFO
+
+@@ -287,6 +290,9 @@
+ /* Define to 1 if the system has the type `mode_t'. */
+ #undef HAVE_MODE_T
+
++/* Define to 1 if you have the `mmap' function. */
++#undef HAVE_MMAP
++
+ /* Define to 1 if you have the `mtrace' function. */
+ #undef HAVE_MTRACE
+
+@@ -329,6 +335,9 @@
+ /* true if you have posix ACLs */
+ #undef HAVE_POSIX_ACLS
+
++/* Define to 1 if you have the `posix_fadvise64' function. */
++#undef HAVE_POSIX_FADVISE64
++
+ /* Define to 1 if you have the `posix_fallocate' function. */
+ #undef HAVE_POSIX_FALLOCATE
+
+diff -ru rsync-3.1.2.orig/configure.ac rsync-3.1.2/configure.ac
+--- rsync-3.1.2.orig/configure.ac 2015-12-21 22:00:49.000000000 +0200
++++ rsync-3.1.2/configure.ac 2016-10-24 15:38:28.006415712 +0300
+@@ -598,6 +598,7 @@
+ setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \
+ seteuid strerror putenv iconv_open locale_charset nl_langinfo getxattr \
+ extattr_get_link sigaction sigprocmask setattrlist getgrouplist \
++ mmap mincore posix_fadvise64 \
+ initgroups utimensat posix_fallocate attropen setvbuf usleep)
+
+ dnl cygwin iconv.h defines iconv_open as libiconv_open
+diff -ru rsync-3.1.2.orig/configure.sh rsync-3.1.2/configure.sh
+--- rsync-3.1.2.orig/configure.sh 2015-12-21 22:20:53.000000000 +0200
++++ rsync-3.1.2/configure.sh 2016-10-24 15:38:28.006415712 +0300
+@@ -7692,6 +7692,7 @@
+ setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \
+ seteuid strerror putenv iconv_open locale_charset nl_langinfo getxattr \
+ extattr_get_link sigaction sigprocmask setattrlist getgrouplist \
++ mmap mincore posix_fadvise64 \
+ initgroups utimensat posix_fallocate attropen setvbuf usleep
+ do :
+ as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
+diff -ru rsync-3.1.2.orig/fileio.c rsync-3.1.2/fileio.c
+--- rsync-3.1.2.orig/fileio.c 2015-08-08 22:47:03.000000000 +0300
++++ rsync-3.1.2/fileio.c 2016-10-24 15:38:28.006415712 +0300
+@@ -51,7 +51,7 @@
+ ret = -1;
+ else {
+ do {
+- ret = write(f, "", 1);
++ ret = fadv_write(f, "", 1);
+ } while (ret < 0 && errno == EINTR);
+
+ ret = ret <= 0 ? -1 : 0;
+@@ -81,7 +81,7 @@
+ do_lseek(f, sparse_seek, SEEK_CUR);
+ sparse_seek = l2;
+
+- while ((ret = write(f, buf + l1, len - (l1+l2))) <= 0) {
++ while ((ret = fadv_write(f, buf + l1, len - (l1+l2))) <= 0) {
+ if (ret < 0 && errno == EINTR)
+ continue;
+ sparse_seek = 0;
+@@ -107,7 +107,7 @@
+ char *bp = wf_writeBuf;
+
+ while (wf_writeBufCnt > 0) {
+- if ((ret = write(f, bp, wf_writeBufCnt)) < 0) {
++ if ((ret = fadv_write(f, bp, wf_writeBufCnt)) < 0) {
+ if (errno == EINTR)
+ continue;
+ return ret;
+@@ -254,7 +254,7 @@
+ map->p_len = window_size;
+
+ while (read_size > 0) {
+- int32 nread = read(map->fd, map->p + read_offset, read_size);
++ int32 nread = fadv_read(map->fd, map->p + read_offset, read_size);
+ if (nread <= 0) {
+ if (!map->status)
+ map->status = nread ? errno : ENODATA;
+diff -ru rsync-3.1.2.orig/generator.c rsync-3.1.2/generator.c
+--- rsync-3.1.2.orig/generator.c 2015-12-05 21:10:24.000000000 +0200
++++ rsync-3.1.2/generator.c 2016-10-24 15:38:28.006415712 +0300
+@@ -111,6 +111,10 @@
+ static int need_retouch_dir_perms;
+ static const char *solo_file = NULL;
+
++#ifdef WITH_DROP_CACHE
++#define close(fd) fadv_close(fd)
++#endif
++
+ enum nonregtype {
+ TYPE_DIR, TYPE_SPECIAL, TYPE_DEVICE, TYPE_SYMLINK
+ };
+diff -ru rsync-3.1.2.orig/options.c rsync-3.1.2/options.c
+--- rsync-3.1.2.orig/options.c 2015-12-19 00:46:28.000000000 +0200
++++ rsync-3.1.2/options.c 2016-10-24 15:38:28.006415712 +0300
+@@ -62,6 +62,9 @@
+ int preserve_gid = 0;
+ int preserve_times = 0;
+ int update_only = 0;
++#ifdef WITH_DROP_CACHE
++int drop_cache = 0;
++#endif
+ int cvs_exclude = 0;
+ int dry_run = 0;
+ int do_xfers = 1;
+@@ -680,6 +683,9 @@
+ rprintf(F," --backup-dir=DIR make backups into hierarchy based in DIR\n");
+ rprintf(F," --suffix=SUFFIX set backup suffix (default %s w/o --backup-dir)\n",BACKUP_SUFFIX);
+ rprintf(F," -u, --update skip files that are newer on the receiver\n");
++#ifdef WITH_DROP_CACHE
++ rprintf(F," --drop-cache do not cache rsync files (POSIX_FADV_DONTNEED)\n");
++#endif
+ rprintf(F," --inplace update destination files in-place (SEE MAN PAGE)\n");
+ rprintf(F," --append append data onto shorter files\n");
+ rprintf(F," --append-verify like --append, but with old data in file checksum\n");
+@@ -914,6 +920,9 @@
+ {"no-one-file-system",0, POPT_ARG_VAL, &one_file_system, 0, 0, 0 },
+ {"no-x", 0, POPT_ARG_VAL, &one_file_system, 0, 0, 0 },
+ {"update", 'u', POPT_ARG_NONE, &update_only, 0, 0, 0 },
++#ifdef WITH_DROP_CACHE
++ {"drop-cache", 0, POPT_ARG_NONE, &drop_cache, 0, 0, 0 },
++#endif
+ {"existing", 0, POPT_ARG_NONE, &ignore_non_existing, 0, 0, 0 },
+ {"ignore-non-existing",0,POPT_ARG_NONE, &ignore_non_existing, 0, 0, 0 },
+ {"ignore-existing", 0, POPT_ARG_NONE, &ignore_existing, 0, 0, 0 },
+@@ -1065,6 +1074,9 @@
+ rprintf(F," --log-file=FILE override the \"log file\" setting\n");
+ rprintf(F," --log-file-format=FMT override the \"log format\" setting\n");
+ rprintf(F," --sockopts=OPTIONS specify custom TCP options\n");
++#ifdef WITH_DROP_CACHE
++ rprintf(F," --drop-cache do not cache rsync files (POSIX_FADV_DONTNEED)\n");
++#endif
+ rprintf(F," -v, --verbose increase verbosity\n");
+ rprintf(F," -4, --ipv4 prefer IPv4\n");
+ rprintf(F," -6, --ipv6 prefer IPv6\n");
+@@ -1089,6 +1101,9 @@
+ {"log-file", 0, POPT_ARG_STRING, &logfile_name, 0, 0, 0 },
+ {"log-file-format", 0, POPT_ARG_STRING, &logfile_format, 0, 0, 0 },
+ {"port", 0, POPT_ARG_INT, &rsync_port, 0, 0, 0 },
++#ifdef WITH_DROP_CACHE
++ {"drop-cache", 0, POPT_ARG_NONE, &drop_cache, 0, 0, 0 },
++#endif
+ {"sockopts", 0, POPT_ARG_STRING, &sockopts, 0, 0, 0 },
+ {"protocol", 0, POPT_ARG_INT, &protocol_version, 0, 0, 0 },
+ {"server", 0, POPT_ARG_NONE, &am_server, 0, 0, 0 },
+@@ -2376,6 +2391,11 @@
+ if (!am_sender)
+ args[ac++] = "--sender";
+
++#ifdef WITH_DROP_CACHE
++ if (drop_cache)
++ args[ac++] = "--drop-cache";
++#endif
++
+ x = 1;
+ argstr[0] = '-';
+
+diff -ru rsync-3.1.2.orig/proto.h rsync-3.1.2/proto.h
+--- rsync-3.1.2.orig/proto.h 2015-12-21 22:22:53.000000000 +0200
++++ rsync-3.1.2/proto.h 2016-10-24 15:38:28.010415712 +0300
+@@ -348,6 +348,10 @@
+ uid_t recv_user_name(int f, uid_t uid);
+ gid_t recv_group_name(int f, gid_t gid, uint16 *flags_ptr);
+ void recv_id_list(int f, struct file_list *flist);
++ssize_t fadv_write(int fd, const void *buf, size_t count);
++ssize_t fadv_read(int fd, void *buf, size_t count);
++void fadv_close_all(void);
++int fadv_close(int fd);
+ void parse_name_map(char *map, BOOL usernames);
+ const char *getallgroups(uid_t uid, item_list *gid_list);
+ void set_nonblocking(int fd);
+diff -ru rsync-3.1.2.orig/receiver.c rsync-3.1.2/receiver.c
+--- rsync-3.1.2.orig/receiver.c 2015-09-07 20:07:17.000000000 +0300
++++ rsync-3.1.2/receiver.c 2016-10-24 15:38:28.010415712 +0300
+@@ -62,6 +62,10 @@
+ extern struct file_list *cur_flist, *first_flist, *dir_flist;
+ extern filter_rule_list daemon_filter_list;
+
++#ifdef WITH_DROP_CACHE
++#define close(fd) fadv_close(fd)
++#endif
++
+ static struct bitbag *delayed_bits = NULL;
+ static int phase = 0, redoing = 0;
+ static flist_ndx_list batch_redo_list;
+diff -ru rsync-3.1.2.orig/rsync.1 rsync-3.1.2/rsync.1
+--- rsync-3.1.2.orig/rsync.1 2015-12-21 22:22:41.000000000 +0200
++++ rsync-3.1.2/rsync.1 2016-10-24 15:38:28.010415712 +0300
+@@ -453,6 +453,7 @@
+ \-\-super receiver attempts super\-user activities
+ \-\-fake\-super store/recover privileged attrs using xattrs
+ \-S, \-\-sparse handle sparse files efficiently
++ \-\-drop\-cache drop cache continuosly using fadvise
+ \-\-preallocate allocate dest files before writing
+ \-n, \-\-dry\-run perform a trial run with no changes made
+ \-W, \-\-whole\-file copy files whole (w/o delta\-xfer algorithm)
+@@ -1426,6 +1427,13 @@
+ up less space on the destination. Conflicts with \fB\-\-inplace\fP because it\(cq\&s
+ not possible to overwrite data in a sparse fashion.
+ .IP
++.IP "\fB\-\-drop\-cache\fP"
++Stop rsync from filling up the file system cache with the files it copies\&. Without this
++option other processes, that had been crunching along happily on your system, will suddenly
++become slow as they find their data being outsed from the cache. The \fB\-\-drop\-cache\fP function
++uses posix_fadvise64 and mincore todo its work\&. It will only get compiled if configure can find posix_fadvise64 and mincore\&.
++Rsync will tries only to drop data from cache that has not been cached before.
++.IP
+ .IP "\fB\-\-preallocate\fP"
+ This tells the receiver to allocate each destination
+ file to its eventual size before writing data to the file. Rsync will only use
+diff -ru rsync-3.1.2.orig/rsync.h rsync-3.1.2/rsync.h
+--- rsync-3.1.2.orig/rsync.h 2015-08-08 22:47:03.000000000 +0300
++++ rsync-3.1.2/rsync.h 2016-10-24 15:38:28.010415712 +0300
+@@ -1291,3 +1291,13 @@
+ #ifdef MAINTAINER_MODE
+ const char *get_panic_action(void);
+ #endif
++
++#if defined HAVE_POSIX_FADVISE64 && defined HAVE_MINCORE && defined HAVE_MMAP
++#define WITH_DROP_CACHE 1
++#include <sys/mman.h>
++int fadv_close(int fd);
++void fadv_close_all(void);
++#endif
++
++ssize_t fadv_write(int fd, const void *buf, size_t count);
++ssize_t fadv_read(int fd, void *buf, size_t count);
+diff -ru rsync-3.1.2.orig/rsync.yo rsync-3.1.2/rsync.yo
+--- rsync-3.1.2.orig/rsync.yo 2015-12-21 22:00:49.000000000 +0200
++++ rsync-3.1.2/rsync.yo 2016-10-24 15:38:28.010415712 +0300
+@@ -1244,6 +1244,17 @@
+ up less space on the destination. Conflicts with bf(--inplace) because it's
+ not possible to overwrite data in a sparse fashion.
+
++dit(bf(--drop-cache)) Stop rsync from disturbing the file system cache with
++the data from the files it copies. Without this option other processes, that
++had been crunching along happily using cached data, will suddenly become
++slow as they find their favorite data blocks data being evicted from the
++cache by the files read and written by rsync. Since rsync has to wait until
++the data is written to disk, before it can drop the cache, this option will
++slow rsync down considerably, especially with small files and short copy
++jobs. The bf(--drop-cache) function uses posix_fadvise64 and mincore todo
++its work. It will only get compiled if configure can find posix_fadvise64
++and mincore.
++
+ dit(bf(--preallocate)) This tells the receiver to allocate each destination
+ file to its eventual size before writing data to the file. Rsync will only use
+ the real filesystem-level preallocation support provided by Linux's
+diff -ru rsync-3.1.2.orig/sender.c rsync-3.1.2/sender.c
+--- rsync-3.1.2.orig/sender.c 2015-09-07 20:07:17.000000000 +0300
++++ rsync-3.1.2/sender.c 2016-10-24 15:38:28.010415712 +0300
+@@ -46,6 +46,9 @@
+ extern int file_old_total;
+ extern struct stats stats;
+ extern struct file_list *cur_flist, *first_flist, *dir_flist;
++#ifdef WITH_DROP_CACHE
++#define close(fd) fadv_close(fd)
++#endif
+
+ BOOL extra_flist_sending_enabled;
+
+diff -ru rsync-3.1.2.orig/t_unsafe.c rsync-3.1.2/t_unsafe.c
+--- rsync-3.1.2.orig/t_unsafe.c 2015-08-08 22:47:03.000000000 +0300
++++ rsync-3.1.2/t_unsafe.c 2016-10-24 15:38:28.010415712 +0300
+@@ -24,6 +24,7 @@
+ #include "rsync.h"
+
+ int dry_run = 0;
++int drop_cache = 0;
+ int am_root = 0;
+ int am_sender = 1;
+ int read_only = 0;
+diff -ru rsync-3.1.2.orig/util.c rsync-3.1.2/util.c
+--- rsync-3.1.2.orig/util.c 2015-12-21 20:54:02.000000000 +0200
++++ rsync-3.1.2/util.c 2016-10-24 15:38:28.014415712 +0300
+@@ -37,6 +37,10 @@
+ extern unsigned int module_dirlen;
+ extern char *partial_dir;
+ extern filter_rule_list daemon_filter_list;
++#ifdef WITH_DROP_CACHE
++#include <sys/mman.h>
++extern int drop_cache;
++#endif
+
+ int sanitize_paths = 0;
+
+@@ -44,6 +48,218 @@
+ unsigned int curr_dir_len;
+ int curr_dir_depth; /* This is only set for a sanitizing daemon. */
+
++#ifdef WITH_DROP_CACHE
++#define FADV_BUFFER_SIZE 1024*1024*16
++
++static struct stat fadv_fd_stat[1024];
++static off_t fadv_fd_pos[1024];
++static unsigned char *fadv_core_ptr[1024];
++static int fadv_max_fd = 0;
++static int fadv_close_ring_tail = 0;
++static int fadv_close_ring_head = 0;
++static int fadv_close_ring_size = 0;
++static int fadv_close_ring[1024];
++static int fadv_close_buffer_size = 0;
++static size_t fadv_pagesize;
++
++static void fadv_fd_init_func(void)
++{
++ static int fadv_fd_init = 0;
++ if (fadv_fd_init == 0){
++ int i;
++ fadv_fd_init = 1;
++ fadv_pagesize = getpagesize();
++ if (fadv_max_fd == 0){
++ fadv_max_fd = sysconf(_SC_OPEN_MAX) - 20;
++ if (fadv_max_fd < 0)
++ fadv_max_fd = 1;
++ if (fadv_max_fd > 1000)
++ fadv_max_fd = 1000;
++ }
++ for (i=0;i<fadv_max_fd;i++){
++ fadv_fd_pos[i] = 0;
++ fadv_fd_stat[i].st_dev = 0;
++ fadv_fd_stat[i].st_ino = 0;
++ fadv_fd_stat[i].st_size = 0;
++ fadv_core_ptr[i] = NULL;
++ }
++ }
++}
++
++static void fadv_get_core(int fd)
++{
++ struct stat stat;
++ void *pa;
++ size_t pi;
++ fstat(fd,&stat);
++ if ( fadv_fd_stat[fd].st_dev == stat.st_dev && fadv_fd_stat[fd].st_ino == stat.st_ino ) {
++ return;
++ }
++ fadv_fd_stat[fd].st_dev = stat.st_dev;
++ fadv_fd_stat[fd].st_ino = stat.st_ino;
++ fadv_fd_stat[fd].st_size = stat.st_size;
++
++ if (fadv_core_ptr[fd]!=NULL){
++ free (fadv_core_ptr[fd]);
++ }
++
++ pa = mmap((void *)0, stat.st_size, PROT_READ, MAP_SHARED, fd, 0);
++ if (MAP_FAILED == pa) {
++ perror("mmap");
++ } else {
++ fadv_core_ptr[fd] = calloc(1, (stat.st_size+fadv_pagesize)/fadv_pagesize);
++ if ( fadv_core_ptr[fd] == NULL ){
++ perror("calloc");
++ } else {
++ if ( mincore(pa, stat.st_size, (fadv_core_ptr[fd])) != 0){
++ perror("mincore");
++ free(fadv_core_ptr[fd]);
++ fadv_core_ptr[fd]=(unsigned char*)0;
++ } else if (DEBUG_GTE(IO, 4)) {
++// } else {
++ rprintf(FINFO, "fadv_get_core(fd=%d): ", fd);
++ for (pi = 0; pi <= stat.st_size/fadv_pagesize; pi++) {
++ if ((fadv_core_ptr[fd])[pi]&1) {
++ rprintf(FINFO,"%lu ", (unsigned long)pi);
++ }
++ }
++ rprintf(FINFO,"\n");
++ }
++ munmap(pa, stat.st_size);
++ }
++ }
++}
++
++static void fadv_drop(int fd, int sync)
++{
++ /* trail 1 MB behind in dropping. we do this to make
++ sure that the same block or stripe does not have
++ to be written twice */
++ off_t pos = lseek(fd,0,SEEK_CUR) - 1024*1024;
++ if (fd > fadv_max_fd){
++ return;
++ }
++ if ( fadv_fd_pos[fd] < pos - FADV_BUFFER_SIZE ) {
++ if (sync) {
++ /* if the file is not flushed to disk before calling fadvise,
++ then the Cache will not be freed and the advise gets ignored
++ this does give a severe hit on performance. If only there
++ was a way to mark cache so that it gets release once the data
++ is written to disk. */
++ fdatasync(fd);
++ }
++ if (fadv_core_ptr[fd] != NULL) {
++ size_t pi;
++ if (pos < fadv_fd_stat[fd].st_size){
++ for (pi = fadv_fd_pos[fd]/fadv_pagesize; pi <= pos/fadv_pagesize; pi++) {
++ if (! (fadv_core_ptr[fd][pi]&1)) {
++ posix_fadvise64(fd, pi*fadv_pagesize, fadv_pagesize, POSIX_FADV_DONTNEED);
++ }
++ }
++ } else {
++ posix_fadvise64(fd, fadv_fd_stat[fd].st_size, pos-fadv_fd_stat[fd].st_size, POSIX_FADV_DONTNEED);
++ }
++ } else {
++ posix_fadvise64(fd, 0, pos, POSIX_FADV_DONTNEED);
++ }
++ fadv_fd_pos[fd] = pos;
++ }
++}
++
++#endif
++
++ssize_t fadv_write(int fd, const void *buf, size_t count)
++{
++ int ret = write(fd, buf, count);
++#ifdef WITH_DROP_CACHE
++ if (drop_cache) {
++ fadv_drop(fd,1);
++ }
++#endif
++ return ret;
++}
++
++ssize_t fadv_read(int fd, void *buf, size_t count)
++{
++ int ret;
++#ifdef WITH_DROP_CACHE
++ if (drop_cache) {
++ fadv_fd_init_func();
++ fadv_get_core(fd);
++ }
++#endif
++ ret = read(fd, buf, count);
++#ifdef WITH_DROP_CACHE
++ if (drop_cache) {
++ fadv_drop(fd,0);
++ }
++#endif
++ return ret;
++}
++
++#ifdef WITH_DROP_CACHE
++void fadv_close_all(void)
++{
++ /* printf ("%i\n",fadv_close_ring_size); */
++ while (fadv_close_ring_size > 0){
++ fdatasync(fadv_close_ring[fadv_close_ring_tail]);
++ if (fadv_core_ptr[fadv_close_ring[fadv_close_ring_tail]]){
++ size_t pi;
++ for (pi = 0; pi <= fadv_fd_stat[fadv_close_ring[fadv_close_ring_tail]].st_size/fadv_pagesize; pi++) {
++ if (!(fadv_core_ptr[fadv_close_ring[fadv_close_ring_tail]][pi]&1)) {
++ posix_fadvise64(fadv_close_ring[fadv_close_ring_tail], pi*fadv_pagesize, fadv_pagesize, POSIX_FADV_DONTNEED);
++ }
++ }
++ /* if the file has grown, drop the rest */
++ //posix_fadvise64(fadv_close_ring[fadv_close_ring_tail], fadv_fd_stat[fadv_close_ring[fadv_close_ring_tail]].st_size,0, POSIX_FADV_DONTNEED);
++
++ free(fadv_core_ptr[fadv_close_ring[fadv_close_ring_tail]]);
++ fadv_core_ptr[fadv_close_ring[fadv_close_ring_tail]] = NULL;
++ fadv_fd_stat[fadv_close_ring[fadv_close_ring_tail]].st_size = 0;
++ fadv_fd_stat[fadv_close_ring[fadv_close_ring_tail]].st_ino = 0;
++ fadv_fd_stat[fadv_close_ring[fadv_close_ring_tail]].st_dev = 0;
++ } else {
++ posix_fadvise64(fadv_close_ring[fadv_close_ring_tail], 0, 0,POSIX_FADV_DONTNEED);
++ }
++ fadv_close_ring_size--;
++ close(fadv_close_ring[fadv_close_ring_tail]);
++ fadv_close_ring_tail = (fadv_close_ring_tail + 1) % fadv_max_fd;
++ fadv_close_buffer_size = 0;
++ }
++}
++
++int fadv_close(int fd)
++{
++ if (drop_cache) {
++ /* if the file is not flushed to disk before calling fadvise,
++ then the Cache will not be freed and the advise gets ignored
++ this does give a severe hit on performance. So instead of doing
++ it right away, we save us a copy of the filehandle and do it
++ some time before we are out of filehandles. This speeds
++ up operation for small files massively. It is directly
++ related to the number of spare file handles you have. */
++ int newfd = dup(fd);
++ off_t pos = lseek(fd,0,SEEK_CUR);
++ fadv_fd_init_func();
++ fadv_core_ptr[newfd] = fadv_core_ptr[fd];
++ fadv_fd_stat[newfd].st_size = fadv_fd_stat[fd].st_size ;
++ fadv_core_ptr[fd] = NULL;
++ fadv_close_buffer_size += pos - fadv_fd_pos[fd];
++ fadv_close_ring[fadv_close_ring_head] = newfd;
++ fadv_close_ring_head = (fadv_close_ring_head + 1) % fadv_max_fd;
++ fadv_close_ring_size ++;
++ if (fadv_close_ring_size == fadv_max_fd || fadv_close_buffer_size > 1024*1024 ){
++ /* it seems fastest to drop things 'in groups' */
++ fadv_close_all();
++ }
++ };
++ return close(fd);
++}
++
++
++#define close(fd) fadv_close(fd)
++#endif
++
+ /* Set a fd into nonblocking mode. */
+ void set_nonblocking(int fd)
+ {
+@@ -273,7 +489,7 @@
+
+ total_written = 0;
+ while (len > 0) {
+- int written = write(desc, ptr, len);
++ int written = fadv_write(desc, ptr, len);
+ if (written < 0) {
+ if (errno == EINTR)
+ continue;
+@@ -305,7 +521,7 @@
+ return len;
+
+ do {
+- n_chars = read(desc, ptr, len);
++ n_chars = fadv_read(desc, ptr, len);
+ } while (n_chars < 0 && errno == EINTR);
+
+ return n_chars;
================================================================
---- gitweb:
http://git.pld-linux.org/gitweb.cgi/packages/rsync.git/commitdiff/b913b1dddc2fad3dad9451c40a1346c435ab1207
More information about the pld-cvs-commit
mailing list