Hi Avati,
It seem no use with your suggestion to remove entries for page size
less than 4KB(even 4KB), the RES also increased with smbd.
Thanks
-Kane
在 2013-9-25，上午10:12，Anand Avati <av...@gluster.org> 写道：
> On Tue, Sep 24, 2013 at 6:37 PM, haiwei.xie-soulinfo
> <haiwei....@soulinfo.com> wrote:
> hi,
>
> Our patch for this bug, running looks good. smbd will not exit with
> oom-kill. But it's not correct method.
>
> git version: release-3.4/886021a31bdac83c2124d08d64b81f22d82039d6
>
> diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
> index 66e7d69..535ee53 100644
> --- a/api/src/glfs-fops.c
> +++ b/api/src/glfs-fops.c
> @@ -713,7 +713,9 @@ glfs_pwritev (struct glfs_fd *glfd, const struct iovec
> *iovec, int iovcnt,
> }
>
> size = iov_length (iovec, iovcnt);
> -
> +#define MIN_LEN 8 * 1024
> + if (size < MIN_LEN)
> + size = MIN_LEN;
> iobuf = iobuf_get2 (subvol->ctx->iobuf_pool, size);
> if (!iobuf) {
> ret = -1;
>
>
>
> Ah, looks like we need to tune the page_size/num_pages table in
> libglusterfs/src/iobuf.c. The table is allowing for too small pages. We
> should probably remove entries for page size less than 4KB. Just doing that
> might fix your issue:
>
> diff --git a/libglusterfs/src/iobuf.c b/libglusterfs/src/iobuf.c
> index a89e962..0269004 100644
> --- a/libglusterfs/src/iobuf.c
> +++ b/libglusterfs/src/iobuf.c
> @@ -24,9 +24,7 @@
> /* Make sure this array is sorted based on pagesize */
> struct iobuf_init_config gf_iobuf_init_config[] = {
> /* { pagesize, num_pages }, */
> - {128, 1024},
> - {512, 512},
> - {2 * 1024, 512},
> + {4 * 1024, 256},
> {8 * 1024, 128},
> {32 * 1024, 64},
> {128 * 1024, 32},
>
> Avati
>
>
> > On 09/13/2013 06:03 PM, kane wrote:
> > > Hi
> > >
> > > We use samba gluster vfs in IO test, but meet with gluster server smbd
> > > oom killer,
> > > The smbd process spend over 15g RES with top command show, in the end
> > > is our simple test code:
> > >
> > > gluster server vfs --> smbd --> client mount dir "/mnt/vfs"--> execute
> > > vfs test program "$ ./vfs 1000"
> > >
> > > then we can watch gluster server smbd RES with top command.
> > >
> > > PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
> > > 4000 soul 20 0 5486m 4.9g 10m R 108.4 31.5 111:07.07 smbd
> > > 3447 root 20 0 1408m 44m 2428 S 44.4 0.3 59:11.55 glusterfsd
> > >
> > > io test code:
> > > =======================================
> > > #define _LARGEFILE64_SOURCE
> > >
> > > #include <stdio.h>
> > > #include <unistd.h>
> > > #include <string.h>
> > > #include <pthread.h>
> > > #include <stdlib.h>
> > > #include <fcntl.h>
> > > #include <sys/types.h>
> > >
> > > int WT = 1;
> > >
> > > #define RND(x) ((x>0)?(genrand() % (x)):0)
> > > extern unsigned long genrand();
> > > extern void sgenrand();
> > >
> > > /* Period parameters */
> > > #define N 624
> > > #define M 397
> > > #define MATRIX_A 0x9908b0df /* constant vector a */
> > > #define UPPER_MASK 0x80000000 /* most significant w-r bits */
> > > #define LOWER_MASK 0x7fffffff /* least significant r bits */
> > >
> > > /* Tempering parameters */
> > > #define TEMPERING_MASK_B 0x9d2c5680
> > > #define TEMPERING_MASK_C 0xefc60000
> > > #define TEMPERING_SHIFT_U(y) (y >> 11)
> > > #define TEMPERING_SHIFT_S(y) (y << 7)
> > > #define TEMPERING_SHIFT_T(y) (y << 15)
> > > #define TEMPERING_SHIFT_L(y) (y >> 18)
> > >
> > > static unsigned long mt[N]; /* the array for the state vector */
> > > static int mti=N+1; /* mti==N+1 means mt[N] is not initialized */
> > >
> > > /* Initializing the array with a seed */
> > > void
> > > sgenrand(seed)
> > > unsigned long seed;
> > > {
> > > int i;
> > >
> > > for (i=0;i<N;i++) {
> > > mt[i] = seed & 0xffff0000;
> > > seed = 69069 * seed + 1;
> > > mt[i] |= (seed & 0xffff0000) >> 16;
> > > seed = 69069 * seed + 1;
> > > }
> > > mti = N;
> > > }
> > >
> > > unsigned long
> > > genrand()
> > > {
> > > unsigned long y;
> > > static unsigned long mag01[2]={0x0, MATRIX_A};
> > > /* mag01[x] = x * MATRIX_A for x=0,1 */
> > >
> > > if (mti >= N) { /* generate N words at one time */
> > > int kk;
> > >
> > > if (mti == N+1) /* if sgenrand() has not been called, */
> > > sgenrand(4357); /* a default initial seed is used */
> > >
> > > for (kk=0;kk<N-M;kk++) {
> > > y = (mt[kk]&UPPER_MASK)|(mt[kk+1]&LOWER_MASK);
> > > mt[kk] = mt[kk+M] ^ (y >> 1) ^ mag01[y & 0x1];
> > > }
> > > for (;kk<N-1;kk++) {
> > > y = (mt[kk]&UPPER_MASK)|(mt[kk+1]&LOWER_MASK);
> > > mt[kk] = mt[kk+(M-N)] ^ (y >> 1) ^ mag01[y & 0x1];
> > > }
> > > y = (mt[N-1]&UPPER_MASK)|(mt[0]&LOWER_MASK);
> > > mt[N-1] = mt[M-1] ^ (y >> 1) ^ mag01[y & 0x1];
> > >
> > > mti = 0;
> > > }
> > > y = mt[mti++];
> > > y ^= TEMPERING_SHIFT_U(y);
> > > y ^= TEMPERING_SHIFT_S(y) & TEMPERING_MASK_B;
> > > y ^= TEMPERING_SHIFT_T(y) & TEMPERING_MASK_C;
> > > y ^= TEMPERING_SHIFT_L(y);
> > >
> > > return y;
> > > }
> > >
> > > char *initialize_file_source(int size)
> > > {
> > > char *new_source;
> > > int i;
> > >
> > > if ((new_source=(char *)malloc(size))==NULL) /* allocate buffer */
> > > fprintf(stderr,"Error: failed to allocate source file of size %d\n",size);
> > > else
> > > for (i=0; i<size; i++) /* file buffer with junk */
> > > new_source[i]=32+RND(95);
> > >
> > > return(new_source);
> > > }
> > >
> > > void *tran_file(void *map)
> > > {
> > > int block_size = 512;
> > > char *read_buffer; /* temporary space for reading file data into */
> > > int fd = open((char *)map, O_RDWR | O_CREAT | O_TRUNC, 0644);
> > > if(fd == -1) {
> > > perror("open");
> > > return ;
> > > }
> > >
> > > //read_buffer=(char *)malloc(block_size);
> > > //memset(read_buffer, 0x56, block_size);
> > > read_buffer = initialize_file_source(block_size);
> > > while(WT){
> > > sleep(1);
> > > pwrite(fd, read_buffer, block_size, 0);
> > > pread(fd, read_buffer, block_size, 0);
> > > }
> > > free(read_buffer);
> > > close(fd);
> > > }
> > >
> > > int main(int argc, char *argv[])
> > > {
> > > if(argc == 1) {
> > > printf("Set file name and file size in GB!\n");
> > > return -1;
> > > }
> > >
> > > unsigned long long length = atoll(argv[1]) * 1024 * 1024 * 1024;
> > > unsigned long long index = 0;
> > > unsigned long long count = 0;
> > > int fd = 0;
> > > char path[32];
> > > char *swap = NULL;
> > > int psize = 65536;
> > > int ret = 0;
> > > int ret_i = 0;
> > > void *tret;
> > > pthread_t rtid_1;
> > > pthread_t rtid_2;
> > > pthread_t rtid_3;
> > > pthread_t rtid_4;
> > > int fd_i = 0;
> > > int a_size = 256;
> > > char *append_buffer; /* temporary space for reading file data into */
> > >
> > > swap = (char *)malloc(sizeof(char) * psize);
> > > if(swap == NULL) {
> > > printf("Malloc Swap Failed!\n");
> > > close(fd);
> > > return -1;
> > > }
> > > memset(swap, 0x55, psize);
> > >
> > > append_buffer = initialize_file_source(a_size);
> > >
> > > pthread_create(&rtid_1, NULL, tran_file, "/mnt/vfs/a.map");
> > > pthread_create(&rtid_2, NULL, tran_file, "/mnt/vfs/b.map");
> > > pthread_create(&rtid_3, NULL, tran_file, "/mnt/vfs/c.map");
> > > pthread_create(&rtid_4, NULL, tran_file, "/mnt/vfs/d.map");
> > >
> > > while(index < length)
> > > {
> > > if((index % (1024*1024*1024)) == 0) {
> > > count++;
> > > sprintf(path, "/mnt/vfs/data/%08lld.dat", count);
> > > if((fd = open(path, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR)) < 0)
> > > {
> > > printf("Can not open io file: %s\n", path);
> > > perror("open");
> > > return -1;
> > > }
> > > sprintf(path, "/mnt/vfs/data/%08lld.index", count);
> > > if((fd_i = open(path, O_RDWR | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR))
> > > < 0)
> > > {
> > > printf("Can not open io file: %s\n", path);
> > > perror("open");
> > > return -1;
> > > }
> > > printf("...1GiB\n");
> > > }
> > >
> > > ret = pwrite(fd, swap, psize, (index % (1024*1024*1024)));
> > > if(ret != psize) {
> > > perror("write");
> > > printf("Write data file failed!\n");
> > > continue;
> > > }
> > > lseek64(fd_i, ((index*a_size)/65536), SEEK_SET);
> > > ret_i = write(fd_i, append_buffer, a_size);
> > > if(ret_i != a_size) {
> > > perror("write");
> > > printf("Write index file failed!\n");
> > > continue;
> > > }
> > >
> > > index += ret;
> > > if((index % (1024*1024*1024)) == 0) {
> > > close(fd);
> > > close(fd_i);
> > > }
> > > }
> > >
> > > WT = 0;
> > > pthread_join(rtid_1, &tret);
> > > pthread_join(rtid_2, &tret);
> > > pthread_join(rtid_3, &tret);
> > > pthread_join(rtid_4, &tret);
> > > printf("%s GB write\n", argv[1]);
> > > sync();
> > > free(swap);
> > > free(append_buffer);
> > >
> > > return 0;
> > > }
> > >
> > > =======================================
> > >
> > >
> > >
> > >
> > > kane
> > > ----------------------------------------------------------------
> > > Email: kai.z...@soulinfo.com <mailto:kai.z...@soulinfo.com>
> > > 电话: 0510-85385788-616
> > >
> > >
> > Thanks for the code. I will try to reproduce the issue using your code.
> >
> > -Lala
>
>
> --
> 谢海威
> 软件项目经理
> 电话: +86 10-68920588
> 手机: +86 13911703586
> Email: haiwei....@soulinfo.com
>
> _______________________________________________
> Gluster-devel mailing list
> Gluster-devel@nongnu.org
> https://lists.nongnu.org/mailman/listinfo/gluster-devel
>
_______________________________________________
Gluster-devel mailing list
Gluster-devel@nongnu.org
https://lists.nongnu.org/mailman/listinfo/gluster-devel
