diff --git a/_toc.yml b/_toc.yml index 86a1c106..88d65e37 100644 --- a/_toc.yml +++ b/_toc.yml @@ -104,6 +104,7 @@ chapters: - file: admin-guide/troubleshooting/boot - file: admin-guide/troubleshooting/kerberos - file: admin-guide/troubleshooting/sssd + - file: admin-guide/troubleshooting/pam - file: admin-guide/troubleshooting/pcie_bus_error - file: admin-guide/order-vm diff --git a/admin-guide/troubleshooting/pam.md b/admin-guide/troubleshooting/pam.md new file mode 100644 index 00000000..60889798 --- /dev/null +++ b/admin-guide/troubleshooting/pam.md @@ -0,0 +1,57 @@ +# PAM + +Checking PAM issues can be quite tricky. + + +# Debugging Messages with `pam_echo.so` + +This can be used to print a message. When placed on every PAM step it shows how far the processing goes and maybe where the error might be: + + +``` +auth required pam_env.so +auth optional pam_echo.so "Have passed pam_env" +auth [default=1 success=ok] pam_localuser.so +auth optional pam_echo.so "Have passed pam_localuser" +auth [success=done ignore=ignore default=die] pam_unix.so nullok try_first_pass +auth optional pam_echo.so "Have passed pam_unix" +auth requisite pam_succeed_if.so uid >= 1000 quiet_success +auth optional pam_echo.so "Have passed pam_succeed_if" +auth required pam_sss.so forward_pass +auth optional pam_echo.so "Have passed pam_sss" +auth optional pam_afs_session.so debug always_aklog program=/usr/bin/aklog minimum_uid=1000 +auth optional pam_echo.so "Have passed pam_afs_session" +``` + Please note that above example introduced a bug: the `default=1` on line 3 ignores the next entry (`1`) if the user is local. Now with adding the debugging statements it needs to be `default=2`, but then the message does not fit, so it should be more specific: +``` +auth [default=2 success=ok] pam_localuser.so +auth optional pam_echo.so "Have local user (pam_localuser) and now try to check local password database (pam_unix)" +auth [success=done ignore=ignore default=die] pam_unix.so nullok try_first_pass +auth optional pam_echo.so "Have passed pam_localuser and pam_unix" +``` + +Or just one message after this block without changing the skip number: +``` +auth [default=1 success=ok] pam_localuser.so +auth [success=done ignore=ignore default=die] pam_unix.so nullok try_first_pass +auth optional pam_echo.so "Have passed pam_localuser and pam_unix" +``` + + +# PAM Call Stack with SystemTap + +When debugging an PAM issue with the RedHat support, they provided us a neat Systemtab script to create a PAM call stack of the `do_pam_setcred()` function in sshd. You find it [here](pam/para-callgraph-pam.stp). + +For other PAM functions or other processes this script would need changes. + +In order to run it, you need to have the debug rpms enabled. In our RHEL8 you can do that manually by setting in +/etc/yum.repos.d/rhel8_baseos_debug.repo and /etc/yum.repos.d/rhel8_appstream_debug.repo `enabled = 1`. It will then be reset with the next Puppet run. + +Then install `systemtap` and `yum-utils`. + +Finally a +``` +stap -x $PID -v ./para-callgraph-pam.stp +``` +will make the systemtab script observe the given process `$PID`. On the first run it will fail and tell what debug packages it needs to have installed. + diff --git a/admin-guide/troubleshooting/pam/para-callgraph-pam.stp b/admin-guide/troubleshooting/pam/para-callgraph-pam.stp new file mode 100644 index 00000000..ee0c3f49 --- /dev/null +++ b/admin-guide/troubleshooting/pam/para-callgraph-pam.stp @@ -0,0 +1,40 @@ +#! /usr/bin/env stap + +global trace + +function trace(entry_p, extra) { + if (tid() in trace) + printf("%s%s%s %s\n", + thread_indent (entry_p), + (entry_p>0?"->":"<-"), + ppfunc (), + extra) +} + +probe process("/usr/sbin/sshd").function("do_pam_setcred").call { + if (pid() != target()) next + if ($init != 1) next + printf("Tracing %ld (%s)\n", pid(), execname()) + trace[tid()] = 1 +} +probe process("/usr/sbin/sshd").function("do_pam_setcred").return { + delete trace[tid()] +} + +probe process("/usr/lib64/libpam.so.0").function("*").call { trace(1, $$parms) } +probe process("/usr/lib64/security/pam_env.so").function("*").call { trace(1, $$parms) } +probe process("/usr/lib64/security/pam_localuser.so").function("*").call { trace(1, $$parms) } +probe process("/usr/lib64/security/pam_access.so").function("*").call { trace(1, $$parms) } +probe process("/usr/lib64/security/pam_succeed_if.so").function("*").call { trace(1, $$parms) } +probe process("/usr/lib64/security/pam_unix.so").function("*").call { trace(1, $$parms) } +probe process("/usr/lib64/security/pam_radius_auth.so").function("*").call { trace(1, $$parms) } +probe process("/usr/lib64/security/pam_sss.so").function("*").call { trace(1, $$parms) } + +probe process("/usr/lib64/libpam.so.0").function("*").return { trace(-1, $$return) } +probe process("/usr/lib64/security/pam_env.so").function("*").return { trace(-1, $$return) } +probe process("/usr/lib64/security/pam_localuser.so").function("*").return { trace(-1, $$return) } +probe process("/usr/lib64/security/pam_access.so").function("*").return { trace(-1, $$return) } +probe process("/usr/lib64/security/pam_succeed_if.so").function("*").return { trace(-1, $$return) } +probe process("/usr/lib64/security/pam_unix.so").function("*").return { trace(-1, $$return) } +probe process("/usr/lib64/security/pam_radius_auth.so").function("*").return { trace(-1, $$return) } +probe process("/usr/lib64/security/pam_sss.so").function("*").return { trace(-1, $$return) }