hugepage-shm.c をコンパイルして実行してみる

doc/Documentation/vm/hugetlbpage.txt に

Examples
========

1) map_hugetlb: see tools/testing/selftests/vm/map_hugetlb.c

2) hugepage-shm: see tools/testing/selftests/vm/hugepage-shm.c

3) hugepage-mmap: see tools/testing/selftests/vm/hugepage-mmap.c

4) The libhugetlbfs (http://libhugetlbfs.sourceforge.net) library provides a
wide range of userspace tools to help with huge page usability, environment
setup, and control. Furthermore it provides useful test cases that should be
used when modifying code to ensure no regressions are introduced.

https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt

と書かれていたので、hugepage-shm.c をダウンロードしてコンパイルして実行してみた。

% wget https://raw.github.com/mirrors/linux-2.6/master/tools/testing/selftests/vm/hugepage-shm.c
% gcc -g -o hugepage-shm hugepage-shm.c
% su
# sysctl -w vm.nr_hugepages=200
vm.nr_hugepages = 200
# ./hugepage-shm 
shmid: 0xb78025
shmaddr: 0x7f9319800000
Starting the writes:
................................................................................................................................................................................................................................................................
Starting the Check...Done.

hugepage-shm を gdb でステップ実行しながら、

# cat /proc/meminfo |grep -i huge
AnonHugePages:         0 kB
HugePages_Total:     200
HugePages_Free:      200
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:       2048 kB
# gdb ./hugepage-shm
NU gdb (GDB) Red Hat Enterprise Linux (7.2-60.el6)
Copyright (C) 2010 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /home/yazekats/work/hugepage_test/hugepage-shm...done.
(gdb) 
(gdb) b main
Breakpoint 1 at 0x40070c: file hugepage-shm.c, line 60.
(gdb) run
Starting program: /home/yazekats/work/hugepage_test/hugepage-shm 

Breakpoint 1, main () at hugepage-shm.c:60
60		shmid = shmget(2, LENGTH, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W);
Missing separate debuginfos, use: debuginfo-install glibc-2.12-1.132.el6.x86_64
(gdb) n
61		if (shmid < 0) {
(gdb) n
65		printf("shmid: 0x%x\n", shmid);
(gdb) n
shmid: 0xd1802d
67		shmaddr = shmat(shmid, ADDR, SHMAT_FLAGS);
(gdb) n
68		if (shmaddr == (char *)-1) {
(gdb) n
73		printf("shmaddr: %p\n", shmaddr);
(gdb) n
shmaddr: 0x2aaaaac00000
75		dprintf("Starting the writes:\n");
(gdb) n
Starting the writes:
76		for (i = 0; i < LENGTH; i++) {
(gdb) n
77			shmaddr[i] = (char)(i);
(gdb) n
78			if (!(i % (1024 * 1024)))

/proc/meminfo を見てみると、

% cat /proc/meminfo|grep -i huge
AnonHugePages:         0 kB
HugePages_Total:     200
HugePages_Free:      199
HugePages_Rsvd:      127
HugePages_Surp:        0
Hugepagesize:       2048 kB

HugePages_Free が減って、HugePages_Rsvd が使われ、HugePage が使われている様子が見れる。

補足

vm.nr_hugepages が足りないと "shmget: Cannot allocate memory" と怒られる。

[root@yazekats-linux hugepage_test]# sysctl -w vm.nr_hugepages=100
vm.nr_hugepages = 100
[root@yazekats-linux hugepage_test]# ./hugepage-shm 
shmget: Cannot allocate memory

strace でシステムコールのトレースを調べると shmget の戻り値が "-1 ENOMEM (Cannot allocate memory)" となっている。

[root@yazekats-linux hugepage_test]# strace ./hugepage-shm 

...

shmget(0x2, 268435456, IPC_CREAT|SHM_HUGETLB|0600) = -1 ENOMEM (Cannot allocate memory)
dup(2)                                  = 3
fcntl(3, F_GETFL)                       = 0x8002 (flags O_RDWR|O_LARGEFILE)
brk(0)                                  = 0x1fdf000
brk(0x2000000)                          = 0x2000000
fstat(3, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 0), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f9bfc7f4000
lseek(3, 0, SEEK_CUR)                   = -1 ESPIPE (Illegal seek)
write(3, "shmget: Cannot allocate memory\n", 31shmget: Cannot allocate memory
) = 31
close(3)                                = 0
munmap(0x7f9bfc7f4000, 4096)            = 0
exit_group(1)                           = ?

追記:
ipcs コマンドで共有メモリが確保される様子を追ってみた。
まず実行前の状況を確認して、

# cat /proc/meminfo|grep -i huge
AnonHugePages:         0 kB
HugePages_Total:     200
HugePages_Free:      200
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:       2048 kB
# ipcs -um

------ Shared Memory Status --------
segments allocated 45
pages allocated 3930
pages resident  3188
pages swapped   0
Swap performance: 0 attempts	 0 successes

hugepage-shm を実行して、

# gdb ./hugepage-shm 
GNU gdb (GDB) Red Hat Enterprise Linux (7.2-60.el6)
Copyright (C) 2010 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /home/yazekats/work/hugepage_test/hugepage-shm...done.
(gdb) b main
Breakpoint 1 at 0x40070c: file hugepage-shm.c, line 60.
(gdb) run
Starting program: /home/yazekats/work/hugepage_test/hugepage-shm 

Breakpoint 1, main () at hugepage-shm.c:60
60		shmid = shmget(2, LENGTH, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W);
Missing separate debuginfos, use: debuginfo-install glibc-2.12-1.132.el6.x86_64
(gdb) n
61		if (shmid < 0) {
(gdb) n
65		printf("shmid: 0x%x\n", shmid);
(gdb) n
shmid: 0x1078026
67		shmaddr = shmat(shmid, ADDR, SHMAT_FLAGS);
(gdb) n
68		if (shmaddr == (char *)-1) {
(gdb) n
73		printf("shmaddr: %p\n", shmaddr);
(gdb) n
shmaddr: 0x2aaaaac00000
75		dprintf("Starting the writes:\n");
(gdb) 

実行中の状況を確認すると、

# cat /proc/meminfo|grep -i huge
AnonHugePages:         0 kB
HugePages_Total:     200
HugePages_Free:      200
HugePages_Rsvd:      128 <-- 増えている
HugePages_Surp:        0
Hugepagesize:       2048 kB
# ipcs -um

------ Shared Memory Status --------
segments allocated 46 <-- 増えている
pages allocated 69466 <-- 増えている
pages resident  3188
pages swapped   0
Swap performance: 0 attempts	 0 successes

# ipcs -a|grep root
0x00000002 17268774   root       600        268435456  1  <-- たぶんこれ
...
# ipcs -m -i 17268774

Shared memory Segment shmid=17268774
uid=0	gid=0	cuid=0	cgid=0
mode=0600	access_perms=0600
bytes=268435456	lpid=5296	cpid=5296	nattch=1
att_time=Sun Feb 23 22:58:37 2014  
det_time=Not set                   
change_time=Sun Feb 23 22:58:34 2014  

さらに追記:
pmap も見てみた。

# pmap -x 5374
5374:   /home/yazekats/work/hugepage_test/hugepage-shm
Address           Kbytes     RSS   Dirty Mode   Mapping
0000000000400000       4       4       4 r-x--  hugepage-shm
0000000000600000       4       4       4 rw---  hugepage-shm
0000003ba6200000     128     108      12 r-x--  ld-2.12.so
0000003ba641f000       4       4       4 r----  ld-2.12.so
0000003ba6420000       4       4       4 rw---  ld-2.12.so
0000003ba6421000       4       4       4 rw---    [ anon ]
0000003ba6600000    1580     224       4 r-x--  libc-2.12.so
0000003ba678b000    2044       0       0 -----  libc-2.12.so
0000003ba698a000      16       8       8 r----  libc-2.12.so
0000003ba698e000       4       4       4 rw---  libc-2.12.so
0000003ba698f000      20      12      12 rw---    [ anon ]
00002aaaaac00000  262144       0       0 rw-s-  SYSV00000002 (deleted) <-- これ
00007ffff7fdf000      12      12      12 rw---    [ anon ]
00007ffff7ffc000       8       8       8 rw---    [ anon ]
00007ffff7ffe000       4       4       0 r-x--    [ anon ]
00007ffffffde000     132       8       8 rw---    [ stack ]
ffffffffff600000       4       0       0 r-x--    [ anon ]
----------------  ------  ------  ------
total kB          266116     408      88

HugePage は最初はカーネル空間に確保されて、使われるとユーザー空間に移動するのかな?