read_ahead

Igor Sysoev is at rambler-co.ru
Mon Sep 14 18:13:18 MSD 2009


aio sendfile в nginx'е использует флаг SF_NODISKIO и sendfile() не
блокируется на диске, а сообщает об отсутствии данных в памяти,
после чего nginx инициирует асинхронную подгрузку данных, читая
только один байт. При этом ядро FreeBSD подгружает в память первые
128K файла, однако при последующих чтениях файл подгружается частями
только по 16K.

К письму прилагается два патча 

1) для FreeBSD 7, реализующий fcntl(F_RDAHEAD, размер):

patch -d /usr/src < patch.rdahead
после чего пересобираем и устанавливаем ядро и перегружаемся.

После загрузки обновляем fcntl.h:

cd /usr/src/include/
make obj all install

2) для nginx-0.8.15 - patch.read_ahead 

Настройки:

sendfile    on;
aio         sendfile;
read_ahead  256k;
tcp_nopush  on;

sysctl vfs.read_max=16  # 256K

Для Линукса read_ahead с любым размером >0 вызывает просто

posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)


-- 
Игорь Сысоев
http://sysoev.ru
-------------- next part --------------
--- sys/sys/fcntl.h	2009-06-02 19:05:17.000000000 +0400
+++ sys/sys/fcntl.h	2009-09-12 20:29:34.000000000 +0400
@@ -118,6 +118,10 @@
 #if __BSD_VISIBLE
 /* Attempt to bypass buffer cache */
 #define O_DIRECT	0x00010000
+#ifdef _KERNEL
+/* Read ahead */
+#define O_RDAHEAD	0x00020000
+#endif
 #endif
 
 /*
@@ -187,6 +191,7 @@
 #define	F_SETLK		12		/* set record locking information */
 #define	F_SETLKW	13		/* F_SETLK; wait if blocked */
 #define	F_SETLK_REMOTE	14		/* debugging support for remote locks */
+#define	F_RDAHEAD	15		/* read ahead */
 
 /* file descriptor flags (F_GETFD, F_SETFD) */
 #define	FD_CLOEXEC	1		/* close-on-exec flag */
--- sys/kern/vfs_vnops.c	2009-06-02 19:05:00.000000000 +0400
+++ sys/kern/vfs_vnops.c	2009-09-12 20:24:00.000000000 +0400
@@ -305,6 +305,9 @@
 sequential_heuristic(struct uio *uio, struct file *fp)
 {
 
+	if (fp->f_flag & O_RDAHEAD)
+		return(fp->f_seqcount << IO_SEQSHIFT);
+
 	if ((uio->uio_offset == 0 && fp->f_seqcount > 0) ||
 	    uio->uio_offset == fp->f_nextoff) {
 		/*
--- sys/kern/kern_descrip.c	2009-08-28 18:50:11.000000000 +0400
+++ sys/kern/kern_descrip.c	2009-09-12 20:23:36.000000000 +0400
@@ -411,6 +411,7 @@
 	u_int newmin;
 	int error, flg, tmp;
 	int vfslocked;
+	uint64_t bsize;
 
 	vfslocked = 0;
 	error = 0;
@@ -694,6 +695,31 @@
 		vfslocked = 0;
 		fdrop(fp, td);
 		break;
+
+	case F_RDAHEAD:
+		FILEDESC_SLOCK(fdp);
+		if ((fp = fdtofp(fd, fdp)) == NULL) {
+			FILEDESC_SUNLOCK(fdp);
+			error = EBADF;
+			break;
+		}
+		if (fp->f_type != DTYPE_VNODE) {
+			FILEDESC_SUNLOCK(fdp);
+			error = EBADF;
+			break;
+		}
+		FILE_LOCK(fp);
+		if (arg) {
+			bsize = fp->f_vnode->v_mount->mnt_stat.f_iosize;
+			fp->f_seqcount = (arg + bsize - 1) / bsize;
+			fp->f_flag |= O_RDAHEAD;
+		} else {
+			fp->f_flag &= ~O_RDAHEAD;
+		}
+		FILE_UNLOCK(fp);
+		FILEDESC_SUNLOCK(fdp);
+		break;
+
 	default:
 		error = EINVAL;
 		break;
-------------- next part --------------
Index: src/http/ngx_http_core_module.c
===================================================================
--- src/http/ngx_http_core_module.c	(revision 2440)
+++ src/http/ngx_http_core_module.c	(working copy)
@@ -408,6 +408,13 @@
 
 #endif
 
+    { ngx_string("read_ahead"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_size_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_core_loc_conf_t, read_ahead),
+      NULL },
+
     { ngx_string("directio"),
       NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
       ngx_http_core_directio,
@@ -2955,6 +2962,7 @@
 #if (NGX_HAVE_FILE_AIO)
     lcf->aio = NGX_CONF_UNSET;
 #endif
+    lcf->read_ahead = NGX_CONF_UNSET_SIZE;
     lcf->directio = NGX_CONF_UNSET;
     lcf->directio_alignment = NGX_CONF_UNSET;
     lcf->tcp_nopush = NGX_CONF_UNSET;
@@ -3156,6 +3164,7 @@
 #if (NGX_HAVE_FILE_AIO)
     ngx_conf_merge_value(conf->aio, prev->aio, 0);
 #endif
+    ngx_conf_merge_size_value(conf->read_ahead, prev->read_ahead, 0);
     ngx_conf_merge_off_value(conf->directio, prev->directio,
                               NGX_MAX_OFF_T_VALUE);
     ngx_conf_merge_off_value(conf->directio_alignment, prev->directio_alignment,
Index: src/http/ngx_http_file_cache.c
===================================================================
--- src/http/ngx_http_file_cache.c	(revision 2440)
+++ src/http/ngx_http_file_cache.c	(working copy)
@@ -281,6 +281,7 @@
     of.min_uses = clcf->open_file_cache_min_uses;
     of.events = clcf->open_file_cache_events;
     of.directio = NGX_OPEN_FILE_DIRECTIO_OFF;
+    of.read_ahead = clcf->read_ahead;
 
     if (ngx_open_cached_file(clcf->open_file_cache, &c->file.name, &of, r->pool)
         != NGX_OK)
Index: src/http/ngx_http_core_module.h
===================================================================
--- src/http/ngx_http_core_module.h	(revision 2440)
+++ src/http/ngx_http_core_module.h	(working copy)
@@ -332,6 +332,7 @@
     size_t        limit_rate;              /* limit_rate */
     size_t        limit_rate_after;        /* limit_rate_after */
     size_t        sendfile_max_chunk;      /* sendfile_max_chunk */
+    size_t        read_ahead;              /* read_ahead */
 
     ngx_msec_t    client_body_timeout;     /* client_body_timeout */
     ngx_msec_t    send_timeout;            /* send_timeout */
Index: src/http/ngx_http_script.c
===================================================================
--- src/http/ngx_http_script.c	(revision 2440)
+++ src/http/ngx_http_script.c	(working copy)
@@ -1407,6 +1407,7 @@
 
     ngx_memzero(&of, sizeof(ngx_open_file_info_t));
 
+    of.read_ahead = clcf->read_ahead;
     of.directio = clcf->directio;
     of.valid = clcf->open_file_cache_valid;
     of.min_uses = clcf->open_file_cache_min_uses;
Index: src/http/modules/ngx_http_gzip_static_module.c
===================================================================
--- src/http/modules/ngx_http_gzip_static_module.c	(revision 2440)
+++ src/http/modules/ngx_http_gzip_static_module.c	(working copy)
@@ -120,6 +120,7 @@
 
     ngx_memzero(&of, sizeof(ngx_open_file_info_t));
 
+    of.read_ahead = clcf->read_ahead;
     of.directio = clcf->directio;
     of.valid = clcf->open_file_cache_valid;
     of.min_uses = clcf->open_file_cache_min_uses;
Index: src/http/modules/ngx_http_index_module.c
===================================================================
--- src/http/modules/ngx_http_index_module.c	(revision 2440)
+++ src/http/modules/ngx_http_index_module.c	(working copy)
@@ -205,6 +205,7 @@
 
         ngx_memzero(&of, sizeof(ngx_open_file_info_t));
 
+        of.read_ahead = clcf->read_ahead;
         of.directio = clcf->directio;
         of.valid = clcf->open_file_cache_valid;
         of.min_uses = clcf->open_file_cache_min_uses;
Index: src/http/modules/ngx_http_static_module.c
===================================================================
--- src/http/modules/ngx_http_static_module.c	(revision 2440)
+++ src/http/modules/ngx_http_static_module.c	(working copy)
@@ -91,6 +91,7 @@
 
     ngx_memzero(&of, sizeof(ngx_open_file_info_t));
 
+    of.read_ahead = clcf->read_ahead;
     of.directio = clcf->directio;
     of.valid = clcf->open_file_cache_valid;
     of.min_uses = clcf->open_file_cache_min_uses;
Index: src/http/modules/ngx_http_flv_module.c
===================================================================
--- src/http/modules/ngx_http_flv_module.c	(revision 2440)
+++ src/http/modules/ngx_http_flv_module.c	(working copy)
@@ -106,6 +106,7 @@
 
     ngx_memzero(&of, sizeof(ngx_open_file_info_t));
 
+    of.read_ahead = clcf->read_ahead;
     of.directio = clcf->directio;
     of.valid = clcf->open_file_cache_valid;
     of.min_uses = clcf->open_file_cache_min_uses;
Index: src/http/modules/perl/nginx.xs
===================================================================
--- src/http/modules/perl/nginx.xs	(revision 2440)
+++ src/http/modules/perl/nginx.xs	(working copy)
@@ -648,6 +648,7 @@
 
     ngx_memzero(&of, sizeof(ngx_open_file_info_t));
 
+    of.read_ahead = clcf->read_ahead;
     of.directio = clcf->directio;
     of.valid = clcf->open_file_cache_valid;
     of.min_uses = clcf->open_file_cache_min_uses;
Index: src/os/win32/ngx_files.c
===================================================================
--- src/os/win32/ngx_files.c	(revision 2440)
+++ src/os/win32/ngx_files.c	(working copy)
@@ -531,6 +531,13 @@
 
 
 ngx_int_t
+ngx_read_ahead(ngx_fd_t fd, size_t n)
+{
+    return ~NGX_FILE_ERROR;
+}
+
+
+ngx_int_t
 ngx_directio_on(ngx_fd_t fd)
 {
     return 0;
Index: src/os/win32/ngx_files.h
===================================================================
--- src/os/win32/ngx_files.h	(revision 2440)
+++ src/os/win32/ngx_files.h	(working copy)
@@ -232,6 +232,8 @@
 ssize_t ngx_write_chain_to_file(ngx_file_t *file, ngx_chain_t *ce,
     off_t offset, ngx_pool_t *pool);
 
+ngx_int_t ngx_read_ahead(ngx_fd_t fd, size_t n);
+#define ngx_read_ahead_n            "ngx_read_ahead_n"
 
 ngx_int_t ngx_directio_on(ngx_fd_t fd);
 #define ngx_directio_on_n           "ngx_directio_on_n"
Index: src/os/unix/ngx_files.h
===================================================================
--- src/os/unix/ngx_files.h	(revision 2440)
+++ src/os/unix/ngx_files.h	(working copy)
@@ -254,6 +254,28 @@
 #define ngx_unlock_fd_n          "fcntl(F_SETLK, F_UNLCK)"
 
 
+#if (NGX_HAVE_F_RDAHEAD)
+
+#define NGX_HAVE_READ_AHEAD      1
+
+#define ngx_read_ahead(fd, n)    fcntl(fd, F_RDAHEAD, (int) n)
+#define ngx_read_ahead_n         "fcntl(fd, F_RDAHEAD)"
+
+#elif (NGX_HAVE_POSIX_FADVISE)
+
+#define NGX_HAVE_READ_AHEAD      1
+
+#define ngx_read_ahead(fd, n)    posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)
+#define ngx_read_ahead_n         "posix_fadvise(POSIX_FADV_SEQUENTIAL)"
+
+#else
+
+#define ngx_read_ahead(fd, n)    0
+#define ngx_read_ahead_n         "ngx_read_ahead_n"
+
+#endif
+
+
 #if (NGX_HAVE_O_DIRECT)
 
 ngx_int_t ngx_directio_on(ngx_fd_t fd);
Index: src/core/ngx_open_file_cache.c
===================================================================
--- src/core/ngx_open_file_cache.c	(revision 2440)
+++ src/core/ngx_open_file_cache.c	(working copy)
@@ -17,6 +17,9 @@
  */
 
 
+#define NGX_MIN_READ_AHEAD  (128 * 1024)
+
+
 static void ngx_open_file_cache_cleanup(void *data);
 static ngx_int_t ngx_open_and_stat_file(u_char *name, ngx_open_file_info_t *of,
     ngx_log_t *log);
@@ -524,6 +527,13 @@
     } else {
         of->fd = fd;
 
+        if (of->read_ahead && ngx_file_size(&fi) > NGX_MIN_READ_AHEAD) {
+            if (ngx_read_ahead(fd, of->read_ahead) == NGX_ERROR) {
+                ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
+                              ngx_read_ahead_n " \"%s\" failed", name);
+            }
+        }
+
         if (of->directio <= ngx_file_size(&fi)) {
             if (ngx_directio_on(fd) == -1) {
                 ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
Index: src/core/ngx_open_file_cache.h
===================================================================
--- src/core/ngx_open_file_cache.h	(revision 2440)
+++ src/core/ngx_open_file_cache.h	(working copy)
@@ -21,6 +21,7 @@
     time_t                   mtime;
     off_t                    size;
     off_t                    directio;
+    size_t                   read_ahead;
 
     ngx_err_t                err;
     char                    *failed;
Index: auto/os/features
===================================================================
--- auto/os/features	(revision 2440)
+++ auto/os/features	(working copy)
@@ -172,6 +172,26 @@
 fi
 
 
+ngx_feature="F_RDAHEAD"
+ngx_feature_name="NGX_HAVE_F_RDAHEAD"
+ngx_feature_run=no
+ngx_feature_incs="#include <fcntl.h>"
+ngx_feature_path=
+ngx_feature_libs=
+ngx_feature_test="fcntl(0, F_RDAHEAD, 1);"
+. auto/feature
+
+
+ngx_feature="posix_fadvise()"
+ngx_feature_name="NGX_HAVE_POSIX_FADVISE"
+ngx_feature_run=no
+ngx_feature_incs="#include <fcntl.h>"
+ngx_feature_path=
+ngx_feature_libs=
+ngx_feature_test="posix_fadvise(0, 0, 0, POSIX_FADV_SEQUENTIAL);"
+. auto/feature
+
+
 ngx_feature="O_DIRECT"
 ngx_feature_name="NGX_HAVE_O_DIRECT"
 ngx_feature_run=no


More information about the nginx-ru mailing list