add PAM troubleshouting guide

2023-10-19 15:43:57 +02:00
parent 8759efb46e
commit 489d6efbc7
3 changed files with 98 additions and 0 deletions
@@ -104,6 +104,7 @@ chapters:
          - file: admin-guide/troubleshooting/boot
          - file: admin-guide/troubleshooting/kerberos
          - file: admin-guide/troubleshooting/sssd
+          - file: admin-guide/troubleshooting/pam
          - file: admin-guide/troubleshooting/pcie_bus_error
      - file: admin-guide/order-vm

@@ -0,0 +1,57 @@
+# PAM
+
+Checking PAM issues can be quite tricky.
+
+
+# Debugging Messages with `pam_echo.so`
+
+This can be used to print a message. When placed on every PAM step it shows how far the processing goes and maybe where the error might be:
+
+
+```
+auth        required      pam_env.so
+auth        optional      pam_echo.so "Have passed pam_env"
+auth        [default=1 success=ok] pam_localuser.so
+auth        optional      pam_echo.so "Have passed pam_localuser"
+auth        [success=done ignore=ignore default=die] pam_unix.so nullok try_first_pass
+auth        optional      pam_echo.so "Have passed pam_unix"
+auth        requisite     pam_succeed_if.so uid >= 1000 quiet_success
+auth        optional      pam_echo.so "Have passed pam_succeed_if"
+auth        required      pam_sss.so forward_pass
+auth        optional      pam_echo.so "Have passed pam_sss"
+auth        optional      pam_afs_session.so debug always_aklog program=/usr/bin/aklog minimum_uid=1000
+auth        optional      pam_echo.so "Have passed pam_afs_session"
+```
+ Please note that above example introduced a bug: the `default=1` on line 3 ignores the next entry (`1`) if the user is local. Now with adding the debugging statements it needs to be `default=2`, but then the message does not fit, so it should be more specific:
+```
+auth        [default=2 success=ok] pam_localuser.so
+auth        optional      pam_echo.so "Have local user (pam_localuser) and now try to check local password database (pam_unix)"
+auth        [success=done ignore=ignore default=die] pam_unix.so nullok try_first_pass
+auth        optional      pam_echo.so "Have passed pam_localuser and pam_unix"
+```
+
+Or just one message after this block without changing the skip number:
+```
+auth        [default=1 success=ok] pam_localuser.so
+auth        [success=done ignore=ignore default=die] pam_unix.so nullok try_first_pass
+auth        optional      pam_echo.so "Have passed pam_localuser and pam_unix"
+```
+
+
+# PAM Call Stack with SystemTap
+
+When debugging an PAM issue with the RedHat support, they provided us a neat Systemtab script to create a PAM call stack of the `do_pam_setcred()` function in sshd. You find it [here](pam/para-callgraph-pam.stp).
+
+For other PAM functions or other processes this script would need changes.
+
+In order to run it, you need to have the debug rpms enabled. In our RHEL8 you can do that manually by setting in 
+/etc/yum.repos.d/rhel8_baseos_debug.repo and /etc/yum.repos.d/rhel8_appstream_debug.repo `enabled = 1`. It will then be reset with the next Puppet run.
+
+Then install `systemtap` and `yum-utils`.
+
+Finally a
+```
+stap -x $PID -v ./para-callgraph-pam.stp
+```
+will make the systemtab script observe the given process `$PID`. On the first run it will fail and tell what debug packages it needs to have installed.
+
@@ -0,0 +1,40 @@
+#! /usr/bin/env stap
+
+global trace
+
+function trace(entry_p, extra) {
+  if (tid() in trace)
+    printf("%s%s%s %s\n",
+           thread_indent (entry_p),
+           (entry_p>0?"->":"<-"),
+           ppfunc (),
+           extra)
+}
+
+probe process("/usr/sbin/sshd").function("do_pam_setcred").call {
+  if (pid() != target()) next
+  if ($init != 1) next
+  printf("Tracing %ld (%s)\n", pid(), execname())
+  trace[tid()] = 1
+}
+probe process("/usr/sbin/sshd").function("do_pam_setcred").return {
+  delete trace[tid()]
+}
+
+probe process("/usr/lib64/libpam.so.0").function("*").call   { trace(1, $$parms) }
+probe process("/usr/lib64/security/pam_env.so").function("*").call   { trace(1, $$parms) }
+probe process("/usr/lib64/security/pam_localuser.so").function("*").call   { trace(1, $$parms) }
+probe process("/usr/lib64/security/pam_access.so").function("*").call   { trace(1, $$parms) }
+probe process("/usr/lib64/security/pam_succeed_if.so").function("*").call   { trace(1, $$parms) }
+probe process("/usr/lib64/security/pam_unix.so").function("*").call   { trace(1, $$parms) }
+probe process("/usr/lib64/security/pam_radius_auth.so").function("*").call   { trace(1, $$parms) }
+probe process("/usr/lib64/security/pam_sss.so").function("*").call   { trace(1, $$parms) }
+
+probe process("/usr/lib64/libpam.so.0").function("*").return { trace(-1, $$return) }
+probe process("/usr/lib64/security/pam_env.so").function("*").return { trace(-1, $$return) }
+probe process("/usr/lib64/security/pam_localuser.so").function("*").return { trace(-1, $$return) }
+probe process("/usr/lib64/security/pam_access.so").function("*").return { trace(-1, $$return) }
+probe process("/usr/lib64/security/pam_succeed_if.so").function("*").return { trace(-1, $$return) }
+probe process("/usr/lib64/security/pam_unix.so").function("*").return { trace(-1, $$return) }
+probe process("/usr/lib64/security/pam_radius_auth.so").function("*").return { trace(-1, $$return) }
+probe process("/usr/lib64/security/pam_sss.so").function("*").return { trace(-1, $$return) }