Command to trace rsh server to check behaviour of particular system call

Question

I am working with rsh. I want to check the whole process from beginning to end. For that I used strace.

Os name is CentOS. I am working on single machine, server and client are on same machine.

My command is , rsh localhost ulimit -n

To take a trace, I used strace rsh localhost ulimit -n.

I read all the files that is open during executing above command. But I want to trace how the rsh server sets the limit of ulimit -n, because all commands in rsh run by rsh daemon.

The system call I am looking for is, setrlimit, but it didn't show this system call by using strace rsh localhost ulimit -n.

For that I have to trace rsh server , i.e, rsh daemon. But I don't know, how I perform this task.

Please tell me the command and their explanations also.

I know rsh is not used in current scenario, but my project is using that, so please don't tell , rsh is not good. I know all these stuffs.

Edit No. 1

$ sudo lsof -i :514


COMMAND  PID USER    FD   TYPE DEVICE SIZE  NODE  NAME
syslogd 2210 root    9u  IPv4   6259       UDP *:syslog
xinetd  2658 root    8u  IPv4   8745       TCP *:shell (LISTEN)

and, /etc/xinetd.d, does not contain rshd, it contains rsh, rexec, rlogin, rsync, etc.

Edit no 2 [related to comment by Chris Down]

rsh localhost strace -o log_new bash -c 'ulimit -n'

It gives different answer that it gives when I run strace rsh localhost ulimit -n

execve("/bin/bash", ["bash", "-c", "ulimit", "-n"], [/* 15 vars */]) = 0
brk(0)                                  = 0x13e86000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2af7bbab2000
uname({sys="Linux", node="jhamb.XXX.XXX", ...}) = 0
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=57641, ...}) = 0
mmap(NULL, 57641, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2af7bbab3000
close(3)                                = 0
open("/lib64/libtermcap.so.2", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\17\300T4\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=15584, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2af7bbac2000
mmap(0x3454c00000, 2108688, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3454c00000
mprotect(0x3454c03000, 2093056, PROT_NONE) = 0
mmap(0x3454e02000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x3454e02000
close(3)                                = 0
open("/lib64/libdl.so.2", O_RDONLY)     = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\16@T4\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=23360, ...}) = 0
mmap(0x3454400000, 2109696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3454400000
mprotect(0x3454402000, 2097152, PROT_NONE) = 0
mmap(0x3454602000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x3454602000
close(3)                                = 0
open("/lib64/libc.so.6", O_RDONLY)      = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\332\1T4\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1726320, ...}) = 0
mmap(0x3454000000, 3506520, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3454000000
mprotect(0x345414f000, 2097152, PROT_NONE) = 0
mmap(0x345434f000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14f000) = 0x345434f000
mmap(0x3454354000, 16728, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x3454354000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2af7bbac3000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2af7bbac4000
arch_prctl(ARCH_SET_FS, 0x2af7bbac3dd0) = 0
mprotect(0x3454602000, 4096, PROT_READ) = 0
mprotect(0x345434f000, 16384, PROT_READ) = 0
mprotect(0x3453e1c000, 4096, PROT_READ) = 0
munmap(0x2af7bbab3000, 57641)           = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
open("/dev/tty", O_RDWR|O_NONBLOCK)     = -1 ENXIO (No such device or address)
ioctl(0, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fffb504cb00) = -1 EINVAL (Invalid argument)
brk(0)                                  = 0x13e86000
brk(0x13ea7000)                         = 0x13ea7000
getuid()                                = 500
getgid()                                = 500
geteuid()                               = 500
getegid()                               = 500
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
open("/proc/meminfo", O_RDONLY)         = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2af7bbab3000
read(3, "MemTotal:      3920228 kB\nMemFre"..., 4096) = 777
close(3)                                = 0
munmap(0x2af7bbab3000, 4096)            = 0
rt_sigaction(SIGCHLD, {SIG_DFL, [], SA_RESTORER, 0x3454030330}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGCHLD, {SIG_DFL, [], SA_RESTORER, 0x3454030330}, {SIG_DFL, [], SA_RESTORER, 0x3454030330}, 8) = 0
rt_sigaction(SIGINT, {SIG_DFL, [], SA_RESTORER, 0x3454030330}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGINT, {SIG_DFL, [], SA_RESTORER, 0x3454030330}, {SIG_DFL, [], SA_RESTORER, 0x3454030330}, 8) = 0
rt_sigaction(SIGQUIT, {SIG_DFL, [], SA_RESTORER, 0x3454030330}, {SIG_DFL, [], 0}, 8) = 0
rt_sigaction(SIGQUIT, {SIG_DFL, [], SA_RESTORER, 0x3454030330}, {SIG_DFL, [], SA_RESTORER, 0x3454030330}, 8) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
rt_sigaction(SIGQUIT, {0x1, [], SA_RESTORER, 0x3454030330}, {SIG_DFL, [], SA_RESTORER, 0x3454030330}, 8) = 0
uname({sys="Linux", node="jhamb.XXX.XXX", ...}) = 0
stat("/home/service", {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
stat(".", {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
getpid()                                = 30873
getppid()                               = 30829
stat(".", {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
stat("/home/service/bin/bash", 0x7fffb504cab0) = -1 ENOENT (No such file or directory)
stat("/usr/local/sbin/bash", 0x7fffb504cab0) = -1 ENOENT (No such file or directory)
stat("/usr/local/bin/bash", 0x7fffb504cab0) = -1 ENOENT (No such file or directory)
stat("/usr/sbin/bash", 0x7fffb504cab0)  = -1 ENOENT (No such file or directory)
stat("/sbin/bash", 0x7fffb504cab0)      = -1 ENOENT (No such file or directory)
stat("/usr/kerberos/bin/bash", 0x7fffb504cab0) = -1 ENOENT (No such file or directory)
stat("/usr/bin/bash", 0x7fffb504cab0)   = -1 ENOENT (No such file or directory)
stat("/bin/bash", {st_mode=S_IFREG|0755, st_size=801512, ...}) = 0
access("/bin/bash", X_OK)               = 0
access("/bin/bash", R_OK)               = 0
stat("/bin/bash", {st_mode=S_IFREG|0755, st_size=801512, ...}) = 0
access("/bin/bash", X_OK)               = 0
access("/bin/bash", R_OK)               = 0
getpgrp()                               = 30829
rt_sigaction(SIGCHLD, {0x436080, [], SA_RESTORER, 0x3454030330}, {SIG_DFL, [], SA_RESTORER, 0x3454030330}, 8) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
getpeername(0, {sa_family=AF_INET, sin_port=htons(61000), sin_addr=inet_addr("127.0.0.1")}, [5255137823777882128]) = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
getrlimit(RLIMIT_FSIZE, {rlim_cur=RLIM_INFINITY, rlim_max=RLIM_INFINITY}) = 0
fstat(1, {st_mode=S_IFSOCK|0777, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2af7bbab3000
write(1, "unlimited\n", 10)             = 10
exit_group(0)                           = ?

Edit No.3

# grep -e ulimit -e setrlimit rsh.strace.



rsh.strace.31472:14:22:42.966361 setrlimit(RLIMIT_NOFILE, {rlim_cur=1024, rlim_max=1024}) = 0
rsh.strace.31474:14:22:43.085822 execve("/bin/bash", ["bash", "-c", "ulimit -n"], [/* 4 vars */]) = 0
rsh.strace.31474:14:22:43.546754 setrlimit(RLIMIT_CORE, {rlim_cur=RLIM_INFINITY, rlim_max=RLIM_INFINITY}) = 0

Edit No. 4: /etc/security/limits.conf with comments removed

*                soft    core            unlimited
*                hard    core            unlimited
@service          hard    nofile          13000
@service          soft    nofile          13000
*                soft    nofile          12000
*                hard    nofile          12000

Answer 1

You'll need to identify what server process on the server runs the rsh aka shell service. Traditionally, it's started by the inetd or xinetd meta-daemon which listens on the shell TCP port (514) and runs the rshd command upon an incoming connection.

lsof -i tcp:shell

(as root) Will tell you what process is listening on that port.

You can strace that with:

strace -tt -ff -o rsh.strace -p "the-PID"

The -ff option follows forks and creates one log file per process which makes it easier to read.

The log files will be named rsh.strace.<pid> where <pid> is the process ID of the corresponding process. xinetd will spawn a new process to run the rshd server, which itself may spawn another process to run the login shell of the user which itself will possibly spawn several processes when interpreting ~/.bashrc (yes bash (if it's the user's login shell), does interpret ~/.bashrc when run over rsh even if it's not a login shell).

Then, you can look in there who does the setrlimit with:

grep setrlimit rsh.strace.*

Once you've identified the process. You can do a

grep execve rsh.strace.<that-pid>

To see if that process executed a command before doing that setrlimit which will tell you what command did the ulimit. If that process didn't do the execve then its parent or grandparent did. You can find out the parent, by checking which process did the fork/clone that resulted in that <pid> like:

grep -E '(clone|fork).*= <that-pid>' rsh.strace.*

If the process is inetd/xinetd and inetd is serving a lot of other services beside shell, alternatively you could change its configuration to run strace -tt -ff -o /var/log/rsh.strace in.rshd instead of in.rshd for the shell service, or make a wrapper script around in.rshd to call the real in.rshd under strace.

Now the two likely things that set the ulimit are either PAM (via the pam_limits module and /etc/security/limits.conf), and the remote user's login shell startup scripts.

In the latter case, instead of stracing rshd, you could enable shell tracing in the login shell. For instance, if the remote user's login shell is bash or sh, sh being a symlink to bash, you can change /usr/sbin/in.rshd (or whatever the location of the rsh daemon command is) to a wrapper script that does:

#! /bin/sh -
exec /usr/bin/env SHELLOPTS=xtrace "$0.bin" "$@"

After having renamed it to in.rshd.bin.

Answer 2

I don't have that much experience with rsh, but this is how I would solve it using strace.

You can strace a running process using the -p flag. So something like this

linux$ strace -p $(pidof rshd) -o logfile.txt

Either that or you can modify the script that starts the rsh deamon to use strace. It might be good to use strace -o logfile for this, since otherwise the output might be hidden by the scripts starting the rsh daemon.

Answer 3

Very easy: What rsh machine command does is to launch command on machine. I.e., if you do rsh localhost ulimit -u, what happens is that the command ulimit -u runs on localhost (i.e., on this same machine). That rsh is involved in calling it is completely irrelevant. If you want to know what happens there, just look at what ulimit -u does.

You can analyze rshd until you are blue in the face, it won't ever do anything but (1) get the connection and check it is allowed, (2) collect the command to run, (3) fork/exec to run the command, connected to send the output back. In particular, you won't see any ulimit frobbing that way.