$ ls /etc/apt/sources*

$ cat /proc/cmdline

BOOT_IMAGE=/boot/vmlinuz-4.9.0-9-amd64 root=UUID=5216e1e4-ae0e-441f-b8f5-8061c0034c74 ro quiet

$ cat /proc/modules

nls_utf8 16384 1 - Live 0xffffffffc0644000
isofs 40960 1 - Live 0xffffffffc0635000
udf 90112 0 - Live 0xffffffffc061e000
crc_itu_t 16384 1 udf, Live 0xffffffffc04be000
fuse 98304 3 - Live 0xffffffffc0605000
vboxsf 45056 0 - Live 0xffffffffc05f9000 (O)
joydev 20480 0 - Live 0xffffffffc056e000
vboxguest 327680 5 vboxsf, Live 0xffffffffc05a8000 (O)
hid_generic 16384 0 - Live 0xffffffffc0569000
        

$ ls /proc/sys

abi  debug  dev  fs  kernel  net  user  vm

$ cat /proc/sys/net/ipv4/ip_forward

$ cat /proc/sys/kernel/pid_max

# ls -l /dev/sd*

brw-rw---- 1 root disk 8, 0 may 25 17:02 /dev/sda
brw-rw---- 1 root disk 8, 1 may 25 17:02 /dev/sda1
brw-rw---- 1 root disk 8, 2 may 25 17:02 /dev/sda2
        

# ls -l /dev/tty*

crw-rw-rw- 1 root tty 5, 0 may 25 17:26 /dev/tty
crw--w---- 1 root tty 4, 0 may 25 17:26 /dev/tty0
crw--w---- 1 root tty 4, 1 may 25 17:26 /dev/tty1
        

$ cat /sys/class/net/enp0s3/address

08:00:27:02:b2:74

$ free
  total        used       free       shared  buff/cache   available
Mem: 4050960  1474960    1482260      96900    1093740      2246372
Swap: 4192252        0    4192252
    

$ free -h
  total        used       free     shared  buff/cache   available
Mem:  3.9G      1.4G       1.5G       75M       1.0G        2.2G
Swap: 4.0G        0B       4.0G
    

$ top
top - 11:10:29 up 2:21, 1 user, load average: 0,11, 0,20, 0,14
Tasks: 73 total, 1 running, 72 sleeping, 0 stopped, 0 zombie
%Cpu(s): 0,0 us, 0,3 sy, 0,0 ni, 99,7 id, 0,0 wa, 0,0 hi, 0,0 si, 0,0 st
KiB Mem : 1020332 total, 909492 free, 38796 used, 72044 buff/cache
KiB Swap: 1046524 total, 1046524 free, 0 used. 873264 avail Mem
  PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
  436 carol 20 0 42696 3624 3060 R 0,7 0,4 0:00.30 top
  4 root 20 0 0 0 0 S 0,3 0,0 0:00.12 kworker/0:0
  399 root 20 0 95204 6748 5780 S 0,3 0,7 0:00.22 sshd
  1 root 20 0 56872 6596 5208 S 0,0 0,6 0:01.29 systemd
  2 root 20 0 0 0 0 S 0,0 0,0 0:00.00 kthreadd
  3 root 20 0 0 0 0 S 0,0 0,0 0:00.02 ksoftirqd/0
  5 root 0 -20 0 0 0 S 0,0 0,0 0:00.00 kworker/0:0H
  6 root 20 0 0 0 0 S 0,0 0,0 0:00.00 kworker/u2:0
  7 root 20 0 0 0 0 S 0,0 0,0 0:00.08 rcu_sched
  8 root 20 0 0 0 0 S 0,0 0,0 0:00.00 rcu_bh
  9 root rt 0 0 0 0 S 0,0 0,0 0:00.00 migration/0
  10 root 0 -20 0 0 0 S 0,0 0,0 0:00.00 lru-add-drain

$ ps
  PID TTY TIME CMD
 2318 pts/0 00:00:00 bash
 2443 pts/0 00:00:00 ps

$ ps -f
UID PID PPID C STIME TTY TIME CMD
carol 2318 1682 0 08:38 pts/1 00:00:00 bash
carol 2443 2318 0 08:46 pts/1 00:00:00 ps -f

carol@debian:~# ls /proc
1 108 13 17 21 27 354 41 665 8 9
10 109 14 173 22 28 355 42 7 804 915
103 11 140 18 23 29 356 428 749 810 918
104 111 148 181 24 3 367 432 75 811
105 112 149 19 244 349 370 433 768 83
106 115 15 195 25 350 371 5 797 838
107 12 16 2 26 353 404 507 798 899
(...)

# ls /proc/1/
attr cmdline environ io mem ns
autogroup comm exe limits mountinfo numa_maps
auxv coredump_filter fd loginuid mounts oom_adj
...

# cat /proc/1/cmdline; echo
/sbin/init

$ uptime
22:12:54 up 13 days, 20:26, 1 user, load average: 2.91, 1.59, 0.39

carol@debian:~# ls /proc
1 108 13 17 21 27 354 41 665 8 9
10 109 14 173 22 28 355 42 7 804 915
103 11 140 18 23 29 356 428 749 810 918
104 111 148 181 24 3 367 432 75 811
105 112 149 19 244 349 370 433 768 83
106 115 15 195 25 350 371 5 797 838
107 12 16 2 26 353 404 507 798 899
(...)

# ls /proc/1/
attr cmdline environ io mem ns
autogroup comm exe limits mountinfo numa_maps
auxv coredump_filter fd loginuid mounts oom_adj
...

# cat /proc/1/cmdline; echo
/sbin/init

$ uptime
22:12:54 up 13 days, 20:26, 1 user, load average: 2.91, 1.59, 0.39

# ls /var/log/apache2/
access.log error.log error.log.1 error.log.2.gz other_vhosts_access.log

$ dmesg | grep boot
[ 0.000000] Command line: BOOT_IMAGE=/boot/vmlinuz-4.9.0-9-amd64 root=UUID=5216e1e4-ae0e-441f-b8f5-8061c0034c74 ro quiet
[ 0.000000] smpboot: Allowing 1 CPUs, 0 hotplug CPUs
[ 0.000000] Kernel command line: BOOT_IMAGE=/boot/vmlinuz-4.9.0-9-amd64 root=UUID=5216e1e4-ae0e-441f-b8f5-8061c0034c74 ro quiet
[ 0.144986] AppArmor: AppArmor disabled by boot time parameter

# journalctl
-- Logs begin at Tue 2019-06-04 17:49:40 CEST, end at Tue 2019-06-04 18:13:10 CEST.
--
jun 04 17:49:40 debian systemd-journald[339]: Runtime journal (/run/log/journal/) is 8.0M, max 159.6M, 151.6M free.
jun 04 17:49:40 debian kernel: microcode: microcode updated early to revision 0xcc, date = 2019-04-01
Jun 04 17:49:40 debian kernel: Linux version 4.9.0-8-amd64 ([email protected]) (gcc version 6.3.0 20170516 (Debian 6.3.0-18+deb9u1) )
Jun 04 17:49:40 debian kernel: Command line: BOOT_IMAGE=/boot/vmlinuz-4.9.0-8-amd64 root=/dev/sda1 ro quiet

# journalctl -k
[ 0.000000] Linux version 4.9.0-9-amd64 ([email protected]) (gcc version 6.3.0 20170516 (Debian 6.3.0-18+deb9u1) ) #1 SMP Debian 4.9.168-1+deb9u2 (2019-05-13)
[ 0.000000] Command line: BOOT_IMAGE=/boot/vmlinuz-4.9.0-9-amd64 root=UUID=5216e1e4-ae0e-441f-b8f5-8061c0034c74 ro quiet

carol@debian:~$ top
top - 13:39:16 up 31 min, 1 user, load average: 0.12, 0.15, 0.10
Tasks: 73 total, 2 running, 71 sleeping, 0 stopped, 0 zombie
%Cpu(s): 1.1 us, 0.4 sy, 0.0 ni, 98.6 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 1020332 total, 698700 free, 170664 used, 150968 buff/cache
KiB Swap: 1046524 total, 1046524 free, 0 used. 710956 avail Mem
  PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
  605 nobody 20 0 1137620 132424 34256 S 6.3 13.0 1:47.24 ntopng
  444 www-data 20 0 364780 4132 2572 S 0.3 0.4 0:00.44 apache2
  734 root 20 0 95212 7004 6036 S 0.3 0.7 0:00.36 sshd
  887 carol 20 0 46608 3680 3104 R 0.3 0.4 0:00.03 top
  1 root 20 0 56988 6688 5240 S 0.0 0.7 0:00.42 systemd
  2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kthreadd
  3 root 20 0 0 0 0 S 0.0 0.0 0:00.09 ksoftirqd/0
  4 root 20 0 0 0 0 S 0.0 0.0 0:00.87 kworker/0:0
(...)

carol@debian:~$ top
top - 13:39:16 up 31 min, 1 user, load average: 0.12, 0.15, 0.10
Tasks: 73 total, 2 running, 71 sleeping, 0 stopped, 0 zombie
%Cpu(s): 1.1 us, 0.4 sy, 0.0 ni, 98.6 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 1020332 total, 698700 free, 170664 used, 150968 buff/cache
KiB Swap: 1046524 total, 1046524 free, 0 used. 710956 avail Mem
  PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
  605 nobody 20 0 1137620 132424 34256 S 6.3 13.0 1:47.24 ntopng
  444 www-data 20 0 364780 4132 2572 S 0.3 0.4 0:00.44 apache2
  734 root 20 0 95212 7004 6036 S 0.3 0.7 0:00.36 sshd
  887 carol 20 0 46608 3680 3104 R 0.3 0.4 0:00.03 top
  1 root 20 0 56988 6688 5240 S 0.0 0.7 0:00.42 systemd
  2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kthreadd
  3 root 20 0 0 0 0 S 0.0 0.0 0:00.09 ksoftirqd/0
  4 root 20 0 0 0 0 S 0.0 0.0 0:00.87 kworker/0:0
(...)

Programs and their Configuration
Important data on a Linux system are—no doubt—its programs and their configuration files. The
former are executable files containing sets of instructions to be run by the computer’s processor,
Linux Essentials (Version 1.6) | 4.3 Where Data is Stored
Version: 2022-04-29 | Licensed for Cyber School. | learning.lpi.org | 283
whereas the latter are usually text documents that control the operation of a program. Executable
files can be either binary files or text files. Executable text files are called scripts. Configuration data
on Linux is traditionally stored in text files too, although there are various styles of representing
configuration data.
Where Binary Files are Stored
Like any other file, executable files live in directories hanging ultimately from /. More specifically,
programs are distributed across a three-tier structure: the first tier (/) includes programs that can be
necessary in single-user mode, the second tier (/usr) contains most multi-user programs and the
third tier (/usr/local) is used to store software that is not provided by the distribution and has been
compiled locally.
Typical locations for programs include:
/sbin
It contains essential binaries for system administration such as parted or ip.
/bin
It contains essential binaries for all users such as ls, mv, or mkdir.
/usr/sbin
It stores binaries for system administration such as deluser, or groupadd.
/usr/bin
It includes most executable files—such as free, pstree, sudo or man—that can be used by all
users.
/usr/local/sbin
It is used to store locally installed programs for system administration that are not managed by
the system’s package manager.
/usr/local/bin
It serves the same purpose as /usr/local/sbin but for regular user programs.
Recently some distributions started to replace /bin and /sbin with symbolic links to /usr/bin and
/usr/sbin.
NOTE The /opt directory is sometimes used to store optional third-party applications.
Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System
284 | learning.lpi.org | Licensed for Cyber School. | Version: 2022-04-29
Apart from these directories, regular users can have their own programs in either:
• /home/$USER/bin
• /home/$USER/.local/bin
TIP
You can find out what directories are available for you to run binaries from by
referencing the PATH variable with echo $PATH. For more information on PATH, review
the lessons on variables and shell customization.
We can find the location of programs with the which command:
$ which git
/usr/bin/git
Where Configuration Files are Stored
The /etc Directory
In the early days of Unix there was a folder for each type of data, such as /bin for binaries and /boot
for the kernel(s). However, /etc (meaning et cetera) was created as a catch-all directory to store any
files that did not belong in the other categories. Most of these files were configuration files. With the
passing of time more and more configuration files were added so /etc became the main folder for
configuration files of programs. As said above, a configuration file usually is a local, plain text (as
opposed to binary) file which controls the operation of a program.
In /etc we can find different patterns for config files names:
• Files with an ad hoc extension or no extension at all, for example
group
System group database.
hostname
Name of the host computer.
hosts
List of IP addresses and their hostname translations.
Linux Essentials (Version 1.6) | 4.3 Where Data is Stored
Version: 2022-04-29 | Licensed for Cyber School. | learning.lpi.org | 285
passwd
System user database—made up of seven fields separated by colons providing information
about the user.
profile
System-wide configuration file for Bash.
shadow
Encrypted file for user passwords.
• Initialization files ending in rc:
bash.bashrc
System-wide .bashrc file for interactive bash shells.
nanorc
Sample initialization file for GNU nano (a simple text editor that normally ships with any
distribution).
• Files ending in .conf:
resolv.conf
Config file for the resolver—which provide access to the Internet Domain Name System
(DNS).
sysctl.conf
Config file to set system variables for the kernel.
• Directories with the .d suffix:
Some programs with a unique config file (*.conf or otherwise) have evolved to have a dedicated
*.d directory which help build modular, more robust configurations. For example, to configure
logrotate, you will find logrotate.conf, but also the logrotate.d directories.
This approach comes in handy in those cases where different applications need configurations
for the same specific service. If, for example, a web server package contains a logrotate
configuration, this configuration can now be placed in a dedicated file in the logrotate.d
directory. This file can be updated by the webserver package without interfering with the
remaining logrotate configuration. Likewise, packages can add specific tasks by placing files in
the /etc/cron.d directory instead of modifying /etc/crontab.
Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System
286 | learning.lpi.org | Licensed for Cyber School. | Version: 2022-04-29
In Debian—and Debian derivatives—such an approach has been applied to the list of reliable
sources read by the package management tool apt: apart from the classic
/etc/apt/sources.list, now we find the /etc/apt/sources.list.d directory:
$ ls /etc/apt/sources*
/etc/apt/sources.list
/etc/apt/sources.list.d:

Configuration Files in HOME (Dotfiles)
At user level, programs store their configurations and settings in hidden files in the user’s home
directory (also represented ~). Remember, hidden files start with a dot (.)—hence their name:
dotfiles.
Some of these dotfiles are Bash scripts that customize the user’s shell session and are sourced as
soon as the user logs into the system:
.bash_history
It stores the command line history.
.bash_logout
It includes commands to execute when leaving the login shell.
.bashrc
Bash’s initialization script for non-login shells.
.profile
Bash’s initialization script for login shells.
NOTE Refer to the lesson on “Command Line Basics” to learn more about Bash and its init
files.
Other user-specific programs’ config files get sourced when their respective programs are started:
.gitconfig, .emacs.d, .ssh, etc.
The Linux Kernel
Before any process can run, the kernel must be loaded into a protected area of memory. After that,
the process with PID 1 (more often than not systemd nowadays) sets off the chain of processes, that
Linux Essentials (Version 1.6) | 4.3 Where Data is Stored
Version: 2022-04-29 | Licensed for Cyber School. | learning.lpi.org | 287
is to say, one process starts other(s) and so on. Once the processes are active, the Linux kernel is in
charge of allocating resources to them (keyboard, mouse, disks, memory, network interfaces, etc).
NOTE
Prior to systemd, /sbin/init was always the first process in a Linux system as part
of the System V Init system manager. In fact, you still find /sbin/init currently but
linked to /lib/systemd/systemd.
Where Kernels are Stored: /boot
The kernel resides in /boot—together with other boot-related files. Most of these files include the
kernel version number components in their names (kernel version, major revision, minor revision
and patch number).
The /boot directory includes the following types of files, with names corresponding with the
respective kernel version:
config-4.9.0-9-amd64
Configuration settings for the kernel such as options and modules that were compiled along with
the kernel.
initrd.img-4.9.0-9-amd64
Initial RAM disk image that helps in the startup process by loading a temporary root filesystem
into memory.
System-map-4.9.0-9-amd64
The System-map (on some systems it will be named System.map) file contains memory address
locations for kernel symbol names. Each time a kernel is rebuilt the file’s contents will change as
the memory locations could be different. The kernel uses this file to lookup memory address
locations for a particular kernel symbol, or vice-versa.
vmlinuz-4.9.0-9-amd64
The kernel proper in a self-extracting, space-saving, compressed format (hence the z in vmlinuz;
vm stands for virtual memory and started to be used when the kernel first got support for virtual
memory).
grub
Configuration directory for the grub2 bootloader.
Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System
288 | learning.lpi.org | Licensed for Cyber School. | Version: 2022-04-29
TIP
Because it is a critical feature of the operating system, more than one kernel and its
associated files are kept in /boot in case the default one becomes faulty and we have
to fall back on a previous version to—at least—be able to boot the system up and fix
it.
The /proc Directory
The /proc directory is one of the so-called virtual or pseudo filesystems since its contents are not
written to disk, but loaded in memory. It is dynamically populated every time the computer boots up
and constantly reflects the current state of the system. /proc includes information about:
• Running processes
• Kernel configuration
• System hardware
Besides all the data concerning processes that we will see in the next lesson, this directory also
stores files with information about the system’s hardware and the kernel’s configuration settings.
Some of these files include:
/proc/cpuinfo
It stores information about the system’s CPU:
$ cat /proc/cpuinfo
processor : 0
vendor_id : GenuineIntel
cpu family : 6
model : 158
model name : Intel(R) Core(TM) i7-8700K CPU @ 3.70GHz
stepping : 10
cpu MHz : 3696.000
cache size : 12288 KB
(…)
/proc/cmdline
It stores the strings passed to the kernel on boot:

$ cat /proc/cmdline
BOOT_IMAGE=/boot/vmlinuz-4.9.0-9-amd64 root=UUID=5216e1e4-ae0e-441f-b8f5-
8061c0034c74 ro quiet
/proc/modules
It shows the list of modules loaded into the kernel:
$ cat /proc/modules
nls_utf8 16384 1 – Live 0xffffffffc0644000
isofs 40960 1 – Live 0xffffffffc0635000
udf 90112 0 – Live 0xffffffffc061e000
crc_itu_t 16384 1 udf, Live 0xffffffffc04be000
fuse 98304 3 – Live 0xffffffffc0605000
vboxsf 45056 0 – Live 0xffffffffc05f9000 (O)
joydev 20480 0 – Live 0xffffffffc056e000
vboxguest 327680 5 vboxsf, Live 0xffffffffc05a8000 (O)
hid_generic 16384 0 – Live 0xffffffffc0569000
(…)
The /proc/sys Directory
This directory includes kernel configuration settings in files classified into categories per
subdirectory:
$ ls /proc/sys
abi debug dev fs kernel net user vm
Most of these files act like a switch and—therefore—only contain either of two possible values: 0 or
1 (“on” or “off”). For instance:
/proc/sys/net/ipv4/ip_forward
The value that enables or disables our machine to act as a router (be able to forward packets):
$ cat /proc/sys/net/ipv4/ip_forward
0
There are some exceptions, though:
Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System
290 | learning.lpi.org | Licensed for Cyber School. | Version: 2022-04-29
/proc/sys/kernel/pid_max
The maximum PID allowed:
$ cat /proc/sys/kernel/pid_max
32768
WARNING Be extra careful when changing the kernel settings as the wrong value may
result in an unstable system.
Hardware Devices
Remember, in Linux “everything is a file”. This implies that hardware device information as well as
the kernel’s own configuration settings are all stored in special files that reside in virtual directories.
The /dev Directory
The device directory /dev contains device files (or nodes) for all connected hardware devices. These
device files are used as an interface between the devices and the processes using them. Each device
file falls into one of two categories:
Block devices
Are those in which data is read and written in blocks which can be individually addressed.
Examples include hard disks (and their partitions, like /dev/sda1), USB flash drives, CDs, DVDs,
etc.
Character devices
Are those in which data is read and written sequentially one character at a time. Examples
include keyboards, the text console (/dev/console), serial ports (such as /dev/ttyS0 and so on),
etc.
When listing device files, make sure you use ls with the -l switch to differentiate between the two.
We can—for instance—check for hard disks and partitions:
# ls -l /dev/sd*
brw-rw—- 1 root disk 8, 0 may 25 17:02 /dev/sda
brw-rw—- 1 root disk 8, 1 may 25 17:02 /dev/sda1
brw-rw—- 1 root disk 8, 2 may 25 17:02 /dev/sda2
(…)
Linux Essentials (Version 1.6) | 4.3 Where Data is Stored
Version: 2022-04-29 | Licensed for Cyber School. | learning.lpi.org | 291
Or for serial terminals (TeleTYpewriter):
# ls -l /dev/tty*
crw-rw-rw- 1 root tty 5, 0 may 25 17:26 /dev/tty
crw–w—- 1 root tty 4, 0 may 25 17:26 /dev/tty0
crw–w—- 1 root tty 4, 1 may 25 17:26 /dev/tty1
(…)
Notice how the first character is b for block devices and c for character devices.
TIP
The asterisk (*) is a globbing character than means 0 or more characters. Hence its
importance in the ls -l /dev/sd* and ls -l /dev/tty* commands above. To learn
more about these special characters, refer to the lesson on globbing.
Furthermore, /dev includes some special files which are quite useful for different programming
purposes:
/dev/zero
It provides as many null characters as requested.
/dev/null
Aka bit bucket. It discards all information sent to it.
/dev/urandom
It generates pseudo-random numbers.
The /sys Directory
The sys filesystem (sysfs) is mounted on /sys. It was introduced with the arrival of kernel 2.6 and
meant a great improvement on /proc/sys.
Processes need to interact with the devices in /dev and so the kernel needs a directory which
contains information about these hardware devices. This directory is /sys and its data is orderly
arranged into categories. For instance, to check on the MAC address of your network card (enp0s3),
you would cat the following file:
$ cat /sys/class/net/enp0s3/address
08:00:27:02:b2:74

Memory and Memory Types
Basically, for a program to start running, it has to be loaded into memory. By and large, when we
speak of memory we refer to Random Access Memory (RAM) and—when compared to mechanical
hard disks—it has the advantage of being a lot faster. On the down side, it is volatile (i.e., once the
computer shuts down, the data is gone).
Notwithstanding the aforementioned—when it comes to memory—we can differentiate two main
types in a Linux system:
Physical memory
Also known as RAM, it comes in the form of chips made up of integrated circuits containing
millions of transistors and capacitors. These, in turn, form memory cells (the basic building block
of computer memory). Each of these cells has an associated hexadecimal code—a memory
address—so that it can be referenced when needed.
Swap
Also known as swap space, it is the portion of virtual memory that lives on the hard disk and is
used when there is no more RAM available.
On the other hand, there is the concept of virtual memory which is an abstraction of the total
amount of usable, addressing memory (RAM, but also disk space) as seen by applications.
free parses /proc/meminfo and displays the amount of free and used memory in the system in a very
clear manner:
$ free
total used free shared buff/cache available
Mem: 4050960 1474960 1482260 96900 1093740 2246372
Swap: 4192252 0 4192252
Let us explain the different columns:
total
Total amount of physical and swap memory installed.
used
Amount of physical and swap memory currently in use.
Linux Essentials (Version 1.6) | 4.3 Where Data is Stored
Version: 2022-04-29 | Licensed for Cyber School. | learning.lpi.org | 293
free
Amount of physical and swap memory currently not in use.
shared
Amount of physical memory used—mostly—by tmpfs.
buff/cache
Amount of physical memory currently in use by kernel buffers and the page cache and slabs.
available
Estimate of how much physical memory is available for new processes.
By default free shows values in kibibytes, but allows for a variety of switches to display its results in
different units of measurement. Some of these options include:
-b
Bytes.
-m
Mebibytes.
-g
Gibibytes.
-h
Human-readable format.
-h is always comfortable to read:
$ free -h
total used free shared buff/cache available
Mem: 3,9G 1,4G 1,5G 75M 1,0G 2,2G
Swap: 4,0G 0B 4,0G
NOTE A kibibyte (KiB) equals 1,024 bytes while a kilobytes (KB) equals 1000 bytes. The
same is respectively true for mebibytes, gibibytes, etc.

Guided Exercises
1. Use the which command to find out the location of the following programs and complete the
table:
Program which command Path to Executable
(output)
User needs root
privileges?
swapon
kill
cut
usermod
cron
ps
2. Where are the following files to be found?
File /etc ~
.bashrc
bash.bashrc
passwd
.profile
resolv.conf
sysctl.conf
3. Explain the meaning of the number elements for kernel file vmlinuz-4.15.0-50-generic found
in /boot:
Number Element Meaning
4
15
0
50
4. What command would you use to list all hard drives and partitions in /dev?
Linux Essentials (Version 1.6) | 4.3 Where Data is Stored
Version: 2022-04-29 | Licensed for Cyber School. | learning.lpi.org | 295
Explorational Exercises
1. Device files for hard drives are represented based on the controllers they use—we saw /dev/sd*
for drives using SCSI (Small Computer System Interface) and SATA (Serial Advanced
Technology Attachment), but
◦ How were old IDE (Integrated Drive Electronics) drives represented?
◦ And modern NVMe (Non-Volatile Memory Express) drives?
2. Take a look at the file /proc/meminfo. Compare the contents of this file to the output of the
command free and identify which key from /proc/meminfo correspond to the following fields in
the output of free:
free output /proc/meminfo field
total
free
shared
buff/cache
available
Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System
296 | learning.lpi.org | Licensed for Cyber School. | Version: 2022-04-29
Summary
In this lesson you have learned about the location of programs and their configuration files in a
Linux system. Important facts to remember are:
• Basically, programs are to be found across a three-level directory structure: /, /usr and
/usr/local. Each of these levels may contain bin and sbin directories.
• Configuration files are stored in /etc and ~.
• Dotfiles are hidden files that start with a dot (.).
We have also discussed the Linux kernel. Important facts are:
• For Linux, everything is a file.
• The Linux kernel lives in /boot together with other boot-related files.
• For processes to start executing, the kernel has to first be loaded into a protected area of
memory.
• The kernel job is that of allocating system resources to processes.
• The /proc virtual (or pseudo) filesystem stores important kernel and system data in a volatile
way.
Likewise, we have explored hardware devices and learned the following:
• The /dev directory stores special files (aka nodes) for all connected hardware devices: block
devices or character devices. The former transfer data in blocks; the latter, one character at a time.
• The /dev directory also contains other special files such as /dev/zero, /dev/null or
/dev/urandom.
• The /sys directory stores information about hardware devices arranged into categories.
Finally, we touched upon memory. We learned:
• A program runs when it is loaded into memory.
• What RAM (Random Access Memory) is.
• What Swap is.
• How to display the use of memory.
Commands used in this lesson:
Linux Essentials (Version 1.6) | 4.3 Where Data is Stored
Version: 2022-04-29 | Licensed for Cyber School. | learning.lpi.org | 297
cat
Concatenate/print file content.
free
Display amount of free and used memory in the system.
ls
List directory contents.
which
Show location of program.

Answers to Guided Exercises
1. Use the which command to find out the location of the following programs and complete the
table:
Program which command Path to Binary
(output)
User needs root
privileges?
swapon which swapon /sbin/swapon Yes
kill which kill /bin/kill No
cut which cut /usr/bin/cut No
usermod which usermod /usr/sbin/usermod Yes
cron which cron /usr/sbin/cron Yes
ps which ps /bin/ps No
2. Where are the following files to be found?
File /etc ~
.bashrc No Yes
bash.bashrc Yes No
passwd Yes No
.profile No Yes
resolv.conf Yes No
sysctl.conf Yes No
3. Explain the meaning of the number elements for kernel file vmlinuz-4.15.0-50-generic found
in /boot:
Number Element Meaning
4 Kernel version
15 Major revision
0 Minor revision
50 Patch number
4. What command would you use to list all hard drives and partitions in /dev?
Linux Essentials (Version 1.6) | 4.3 Where Data is Stored
Version: 2022-04-29 | Licensed for Cyber School. | learning.lpi.org | 299
ls /dev/sd*
Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System
300 | learning.lpi.org | Licensed for Cyber School. | Version: 2022-04-29
Answers to Explorational Exercises
1. Device files for hard drives are represented based on the controllers they use—we saw /dev/sd*
for drives using SCSI (Small Computer System Interface) and SATA (Serial Advanced
Technology Attachment), but
◦ How were old IDE (Integrated Drive Electronics) drives represented?
/dev/hd*
◦ And modern NVMe (Non-Volatile Memory Express) drives?
/dev/nvme*
2. Take a look at the file /proc/meminfo. Compare the contents of this file to the output of the
command free and identify which key from /proc/meminfo correspond to the following fields in
the output of free:
free output /proc/meminfo field
total MemTotal / SwapTotal
free MemFree / SwapFree
shared Shmem
buff/cache Buffers, Cached and SReclaimable
available MemAvailable

Processes
Every time a user issues a command, a program is run and one or more processes are generated.
Processes exist in a hierarchy. After the kernel is loaded in memory on boot, the first process is
initiated which—in turn—starts other processes, which, again, can start other processes. Every
process has a unique identifier (PID) and parent process identifier (PPID). These are positive integers
that are assigned in sequential order.
Exploring Processes Dynamically: top
You can get a dynamic listing of all running processes with the top command:
Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System
302 | learning.lpi.org | Licensed for Cyber School. | Version: 2022-04-29
$ top
top – 11:10:29 up 2:21, 1 user, load average: 0,11, 0,20, 0,14
Tasks: 73 total, 1 running, 72 sleeping, 0 stopped, 0 zombie
%Cpu(s): 0,0 us, 0,3 sy, 0,0 ni, 99,7 id, 0,0 wa, 0,0 hi, 0,0 si, 0,0 st
KiB Mem : 1020332 total, 909492 free, 38796 used, 72044 buff/cache
KiB Swap: 1046524 total, 1046524 free, 0 used. 873264 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
436 carol 20 0 42696 3624 3060 R 0,7 0,4 0:00.30 top
4 root 20 0 0 0 0 S 0,3 0,0 0:00.12 kworker/0:0
399 root 20 0 95204 6748 5780 S 0,3 0,7 0:00.22 sshd
1 root 20 0 56872 6596 5208 S 0,0 0,6 0:01.29 systemd
2 root 20 0 0 0 0 S 0,0 0,0 0:00.00 kthreadd
3 root 20 0 0 0 0 S 0,0 0,0 0:00.02 ksoftirqd/0
5 root 0 -20 0 0 0 S 0,0 0,0 0:00.00 kworker/0:0H
6 root 20 0 0 0 0 S 0,0 0,0 0:00.00 kworker/u2:0
7 root 20 0 0 0 0 S 0,0 0,0 0:00.08 rcu_sched
8 root 20 0 0 0 0 S 0,0 0,0 0:00.00 rcu_bh
9 root rt 0 0 0 0 S 0,0 0,0 0:00.00 migration/0
10 root 0 -20 0 0 0 S 0,0 0,0 0:00.00 lru-add-drain
(…)
As we saw above, top can also give us information about memory and CPU consumption of the
overall system as well as for each process.
top allows the user some interaction.
By default, the output is sorted by the percentage of CPU time used by each process in descending
order. This behavior can be modified by pressing the following keys from within top:
M
Sort by memory usage.
N
Sort by process ID number.
T
Sort by running time.
Linux Essentials (Version 1.6) | 4.3 Where Data is Stored
Version: 2022-04-29 | Licensed for Cyber School. | learning.lpi.org | 303
P
Sort by percentage of CPU usage.
To switch between descending/ascending order just press R.
TIP
A fancier and more user-friendly version of top is htop. Another—perhaps more
exhaustive—alternative is atop. If not already installed in your system, I encourage
you to use your package manager to install them and give them a try.
A Snapshot of Processes: ps
Another very useful command to get information about processes is ps. Whereas top provides
dynamic information, that of ps is static.
If invoked without options, the output of ps is quite discrete and relates only to the processes
attached to the current shell:
$ ps
PID TTY TIME CMD
2318 pts/0 00:00:00 bash
2443 pts/0 00:00:00 ps
The displayed information has to do with the process identifier (PID), the terminal in which the
process is run (TTY), the CPU time taken by the process (TIME) and the command which started the
process (CMD).
A useful switch for ps is -f which shows the full-format listing:
$ ps -f
UID PID PPID C STIME TTY TIME CMD
carol 2318 1682 0 08:38 pts/1 00:00:00 bash
carol 2443 2318 0 08:46 pts/1 00:00:00 ps -f
In combination with other switches, -f shows the relationship between parent and child processes:
Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System
304 | learning.lpi.org | Licensed for Cyber School. | Version: 2022-04-29
$ ps -uf
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
carol 2318 0.0 0.1 21336 5140 pts/1 Ss 08:38 0:00 bash
carol 2492 0.0 0.0 38304 3332 pts/1 R+ 08:51 0:00 \_ ps -uf
carol 1780 0.0 0.1 21440 5412 pts/0 Ss 08:28 0:00 bash
carol 2291 0.0 0.7 305352 28736 pts/0 Sl+ 08:35 0:00 \_ emacs
index.en.adoc -nw
(…)
Likewise, ps can show the percentage of memory used when invoked with the -v switch:
$ ps -v
PID TTY STAT TIME MAJFL TRS DRS RSS %MEM COMMAND
1163 tty2 Ssl+ 0:00 1 67 201224 5576 0.1 /usr/lib/gdm3/gdm-xsession (…)
(…)
NOTE Another visually attractive command that shows the hierarchy of processes is
pstree. It ships with all major distributions.

Process Information in the /proc Directory
We have already seen the /proc filesystem. /proc includes a numbered subdirectory for every
running process in the system (the number is the PID of the process):
carol@debian:~# ls /proc
1 108 13 17 21 27 354 41 665 8 9
10 109 14 173 22 28 355 42 7 804 915
103 11 140 18 23 29 356 428 749 810 918
104 111 148 181 24 3 367 432 75 811
105 112 149 19 244 349 370 433 768 83
106 115 15 195 25 350 371 5 797 838
107 12 16 2 26 353 404 507 798 899
(…)
Thus, all the information about a particular process is included within its directory. Let us list the
contents of the first process—that whose PID is 1 (the output has been truncated for readability):
Linux Essentials (Version 1.6) | 4.3 Where Data is Stored
Version: 2022-04-29 | Licensed for Cyber School. | learning.lpi.org | 305
# ls /proc/1/
attr cmdline environ io mem ns
autogroup comm exe limits mountinfo numa_maps
auxv coredump_filter fd loginuid mounts oom_adj
…
You can check—for instance—the process executable:
# cat /proc/1/cmdline; echo
/sbin/init
As you can see, the binary that started the hierarchy of processes was /sbin/init.
NOTE
Commands can be concatenated with the semicolon (;). The point in using the
echo command above is to provide a newline. Try and run simply cat
/proc/1/cmdline to see the difference.
The System Load
Each process on a system can potentially consume system resources. The so-called system load tries
to aggregate the overall load of the system into a single numeric indicator. You can see the current
load with the command uptime:
$ uptime
22:12:54 up 13 days, 20:26, 1 user, load average: 2.91, 1.59, 0.39
The three last digits indicate the system’s load average for the last minute (2.91), the last five
minutes (1.59) and the last fifteen minutes (0.39), respectively.
Each of these numbers indicates how many processes were waiting either for CPU resources or for
input/output operations to complete. This means that these processes were ready to run if they had
received the respective resources.
System Logging and System Messaging
As soon as the kernel and the processes start executing and communicating with each other, a lot of
information is produced. Most of it is sent to files—the so-called log files or, simply, logs.
Without logging, searching for an event that happened on a server would give sysadmins many a
Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System
306 | learning.lpi.org | Licensed for Cyber School. | Version: 2022-04-29
headache, hence the importance of having a standardized and centralized way of keeping track of
any system events. Besides, logs are determinant and telling when it comes to troubleshooting and
security as well as reliable data sources for understanding system statistics and making trend
predictions.
Logging with the syslog Daemon
Traditionally, system messages have been managed by the standard logging facility—syslog—or any
of its derivatives—syslog-ng or rsyslog. The logging daemon collects messages from other services
and programs and stores them in log files, typically under /var/log. However, some services take
care of their own logs (take—for example—the Apache HTTPD web server). Likewise, the Linux
kernel uses an in-memory ring buffer for storing its log messages.

Log Files in /var/log
Because logs are data that varies over time, they are normally found in /var/log.
If you explore /var/log, you will realize that the names of logs are—to a certain degree—quite selfexplanatory. Some examples include:
/var/log/auth.log
It stores information about authentication.
/var/log/kern.log
It stores kernel information.
/var/log/syslog
It stores system information.
/var/log/messages
It stores system and application data.
NOTE The exact name and contents of log files may vary across Linux distributions.
Accessing Log Files
When exploring log files, remember to be root (if you do not have reading permissions) and use a
pager such as less;
Linux Essentials (Version 1.6) | 4.3 Where Data is Stored
Version: 2022-04-29 | Licensed for Cyber School. | learning.lpi.org | 307
# less /var/log/messages
Jun 4 18:22:48 debian liblogging-stdlog: [origin software=”rsyslogd”
swVersion=”8.24.0″ x-pid=”285″ x-info=”http://www.rsyslog.com”] rsyslogd was HUPed
Jun 29 16:57:10 debian kernel: [ 0.000000] Linux version 4.9.0-8-amd64 (debian[email protected]) (gcc version 6.3.0 20170516 (Debian 6.3.0-18+deb9u1) ) #1
SMP Debian 4.9.130-2 (2018-10-27)
Jun 29 16:57:10 debian kernel: [ 0.000000] Command line:
BOOT_IMAGE=/boot/vmlinuz-4.9.0-8-amd64 root=/dev/sda1 ro quiet
Alternatively, you can use tail with the -f switch to read the most recent messages of the file and
dynamically show new lines as they are appended:
# tail -f /var/log/messages
Jul 9 18:39:37 debian kernel: [ 2.350572] RAPL PMU: hw unit of domain psys 2^-0
Joules
Jul 9 18:39:37 debian kernel: [ 2.512802] input: VirtualBox USB Tablet as
/devices/pci0000:00/0000:00:06.0/usb1/1-1/1-1:1.0/0003:80EE:0021.0001/input/input7
Jul 9 18:39:37 debian kernel: [ 2.513861] Adding 1046524k swap on /dev/sda5.
Priority:-1 extents:1 across:1046524k FS
Jul 9 18:39:37 debian kernel: [ 2.519301] hid-generic 0003:80EE:0021.0001:
input,hidraw0: USB HID v1.10 Mouse [VirtualBox USB Tablet] on usb-0000:00:06.0-
1/input0
Jul 9 18:39:37 debian kernel: [ 2.623947] snd_intel8x0 0000:00:05.0: white list
rate for 1028:0177 is 48000
Jul 9 18:39:37 debian kernel: [ 2.914805] IPv6: ADDRCONF(NETDEV_UP): enp0s3:
link is not ready
Jul 9 18:39:39 debian kernel: [ 4.937283] e1000: enp0s3 NIC Link is Up 1000
Mbps Full Duplex, Flow Control: RX
Jul 9 18:39:39 debian kernel: [ 4.938493] IPv6: ADDRCONF(NETDEV_CHANGE):
enp0s3: link becomes ready
Jul 9 18:39:40 debian kernel: [ 5.315603] random: crng init done
Jul 9 18:39:40 debian kernel: [ 5.315608] random: 7 urandom warning(s) missed
due to ratelimiting
You will find the output into the following format:
• Timestamp
• Hostname from which the message came from
• Name of program/service that generated the message
• The PID of the program that generated the message
Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System
308 | learning.lpi.org | Licensed for Cyber School. | Version: 2022-04-29
• Description of the action that took place
Most log files are written in plain text; however, a few can contain binary data as is the case with
/var/log/wtmp—which stores data relevant to successful logins. You can use the file command to
determine which is the case:
$ file /var/log/wtmp
/var/log/wtmp: dBase III DBT, version number 0, next free block index 8
These files are normally read using special commands. last is used to interpret the data in
/var/log/wtmp:
$ last
carol tty2 :0 Thu May 30 10:53 still logged in
reboot system boot 4.9.0-9-amd64 Thu May 30 10:52 still running
carol tty2 :0 Thu May 30 10:47 – crash (00:05)
reboot system boot 4.9.0-9-amd64 Thu May 30 09:11 still running
carol tty2 :0 Tue May 28 08:28 – 14:11 (05:42)
reboot system boot 4.9.0-9-amd64 Tue May 28 08:27 – 14:11 (05:43)
carol tty2 :0 Mon May 27 19:40 – 19:52 (00:11)
reboot system boot 4.9.0-9-amd64 Mon May 27 19:38 – 19:52 (00:13)
carol tty2 :0 Mon May 27 19:35 – down (00:03)
reboot system boot 4.9.0-9-amd64 Mon May 27 19:34 – 19:38 (00:04)
NOTE Similar to /var/log/wtmp, /var/log/btmp stores information about failed login
attempts and the special command to read its contents is lastb.

Log Rotation
Log files can grow a lot over a few weeks or months and take up all free disk space. To tackle this,
the utility logrotate is used. It implements log rotation or cycling which implies actions such as
moving log files to a new name, archiving and/or compressing them, sometimes emailing them to
the sysadmin and eventually deleting them as they grow old. The conventions used for naming these
rotated log files are diverse (adding a suffix with the date, for example); however, simply adding a
suffix with an integer is commonplace:
# ls /var/log/apache2/
access.log error.log error.log.1 error.log.2.gz other_vhosts_access.log
Linux Essentials (Version 1.6) | 4.3 Where Data is Stored
Version: 2022-04-29 | Licensed for Cyber School. | learning.lpi.org | 309
Note how error.log.2.gz has already been compressed with gzip (hence the .gz suffix).
The Kernel Ring Buffer
The kernel ring buffer is a fixed-size data structure that records kernel boot messages as well as any
live kernel messages. The function of this buffer—a very important one—is that of logging all the
kernel messages produced on boot—when syslog is not yet available. The dmesg command prints
the kernel ring buffer (which used to be also stored in /var/log/dmesg). Because of the extension of
the ring buffer, this command is normally used in combination with the text filtering utility grep or
a pager such as less. For instance, to search for boot messages:
$ dmesg | grep boot
[ 0.000000] Command line: BOOT_IMAGE=/boot/vmlinuz-4.9.0-9-amd64
root=UUID=5216e1e4-ae0e-441f-b8f5-8061c0034c74 ro quiet
[ 0.000000] smpboot: Allowing 1 CPUs, 0 hotplug CPUs
[ 0.000000] Kernel command line: BOOT_IMAGE=/boot/vmlinuz-4.9.0-9-amd64
root=UUID=5216e1e4-ae0e-441f-b8f5-8061c0034c74 ro quiet
[ 0.144986] AppArmor: AppArmor disabled by boot time parameter
(…)
NOTE As the kernel ring buffer grows with new messages over time, the oldest ones will
fade away.
The System Journal: systemd-journald
As of 2015, systemd replaced SysV Init as a de facto system and service manager in most major Linux
distributions. As a consequence, the journal daemon—journald—has become the standard logging
component, superseding syslog in most aspects. The data is no longer stored in plain text but in
binary form. Thus, the journalctl utility is necessary to read the logs. On top of that, journald is
syslog compatible and can be integrated with syslog.
journalctl is the utility that is used to read and query systemd’s journal database. If invoked
without options, it prints the entire journal:
Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System
310 | learning.lpi.org | Licensed for Cyber School. | Version: 2022-04-29
# journalctl
— Logs begin at Tue 2019-06-04 17:49:40 CEST, end at Tue 2019-06-04 18:13:10 CEST.
—
jun 04 17:49:40 debian systemd-journald[339]: Runtime journal (/run/log/journal/)
is 8.0M, max 159.6M, 151.6M free.
jun 04 17:49:40 debian kernel: microcode: microcode updated early to revision 0xcc,
date = 2019-04-01
Jun 04 17:49:40 debian kernel: Linux version 4.9.0-8-amd64 (debian[email protected]) (gcc version 6.3.0 20170516 (Debian 6.3.0-18+deb9u1) )
Jun 04 17:49:40 debian kernel: Command line: BOOT_IMAGE=/boot/vmlinuz-4.9.0-8-amd64
root=/dev/sda1 ro quiet
(…)
However, if invoked with the -k or –dmesg switches, it will be equivalent to using the dmesg
command:
# journalctl -k
[ 0.000000] Linux version 4.9.0-9-amd64 ([email protected]) (gcc
version 6.3.0 20170516 (Debian 6.3.0-18+deb9u1) ) #1 SMP Debian 4.9.168-1+deb9u2
(2019-05-13)
[ 0.000000] Command line: BOOT_IMAGE=/boot/vmlinuz-4.9.0-9-amd64
root=UUID=5216e1e4-ae0e-441f-b8f5-8061c0034c74 ro quiet
(…)
Other interesting options for journalctl include:
-b, –boot
It shows boot information.
-u
It shows messages about a specified unit. Roughly, a unit can be defined as any resource handled
by systemd. For example journalctl -u apache2.service is used to read messages about the
apache2 web server.
-f
It shows most recent journal messages and keeps printing new entries as they are appended to
the journal—much like tail -f.

Guided Exercises 1. Have a look at the following listing of top and answer the following questions: carol@debian:~$ top top – 13:39:16 up 31 min, 1 user, load average: 0.12, 0.15, 0.10 Tasks: 73 total, 2 running, 71 sleeping, 0 stopped, 0 zombie %Cpu(s): 1.1 us, 0.4 sy, 0.0 ni, 98.6 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st KiB Mem : 1020332 total, 698700 free, 170664 used, 150968 buff/cache KiB Swap: 1046524 total, 1046524 free, 0 used. 710956 avail Mem   PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND   605 nobody 20 0 1137620 132424 34256 S 6.3 13.0 1:47.24 ntopng   444 www-data 20 0 364780 4132 2572 S 0.3 0.4 0:00.44 apache2   734 root 20 0 95212 7004 6036 S 0.3 0.7 0:00.36 sshd   887 carol 20 0 46608 3680 3104 R 0.3 0.4 0:00.03 top   1 root 20 0 56988 6688 5240 S 0.0 0.7 0:00.42 systemd   2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kthreadd   3 root 20 0 0 0 0 S 0.0 0.0 0:00.09 ksoftirqd/0   4 root 20 0 0 0 0 S 0.0 0.0 0:00.87 kworker/0:0 (…) ◦ Which processes have been started by the user carol? ◦ What virtual directory of /proc should you visit to search for data regarding the top command? ◦ What process was run first? How can you tell? ◦ Complete the table specifying in what area of top output the following information is found: Information about … Summary Area Task Area Memory Swap PID Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System 312     |         learning.lpi.org     |     Licensed for Cyber School.     |     Version: 2022-04-29 Information about … Summary Area Task Area CPU time Commands 2. What command is used to read the following binary logs? ◦ /var/log/wtmp ◦ /var/log/btmp ◦ /run/log/journal/2a7d9730cd3142f4b15e20d6be631836/system.journal 3. In combination with grep, what commands would you use to find out the following information about your Linux system? ◦ When the system was last rebooted (wtmp) ◦ Which hard disks are installed (kern.log) ◦ When the last login occurred (auth.log) 4. What two commands would you use to have the kernel ring buffer displayed? 5. Indicate where the following log messages belong: ◦ Jul 10 13:37:39 debian dbus[303]: [system] Successfully activated service ‘org.freedesktop.nm_dispatcher’ /var/log/auth.log /var/log/kern.log /var/log/syslog /var/log/messages Linux Essentials (Version 1.6) | 4.3 Where Data is Stored Version: 2022-04-29     |     Licensed for Cyber School.     |     learning.lpi.org         |     313 ◦ Jul 10 11:23:58 debian kernel: [ 1.923349] usbhid: USB HID core driver /var/log/auth.log /var/log/kern.log /var/log/syslog /var/log/messages ◦ Jul 10 14:02:53 debian sudo: pam_unix(sudo:session): session opened for user root by carol(uid=0) /var/log/auth.log /var/log/kern.log /var/log/syslog /var/log/messages ◦ Jul 10 11:23:58 debian NetworkManager[322]: [1562750638.8672] NetworkManager (version 1.6.2) is starting… /var/log/auth.log /var/log/kern.log /var/log/syslog /var/log/messages 6. Have journalctl query information about the following units? Unit Command ssh networking rsyslog cron Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System 314     |         learning.lpi.org     |     Licensed for Cyber School.     |     Version: 2022-04-29 Explorational Exercises 1. Reconsider the top output from the guided exercises and answer the following questions: ◦ What two steps would you follow to kill the apache web server? ◦ In the summary area, how could you display the information about physical memory and swap using progress bars? ◦ Now, sort the processes by memory usage: ◦ Now that you have memory information displayed in progress bars and processes sorted by memory usage, save these configurations so that you get them as default next time you use top: ◦ What file stores top’s configuration settings? Where does it live? How can you check for its existence? 2. Learn about the command exec in Bash. Try to demonstrate its functionality by starting a Bash session, finding the Bash process with ps, then run exec /bin/sh and search for the process with the same PID again. 3. Follow these steps to explore kernel events and udev’s dynamic management of devices: ◦ Hotplug a USB drive into your computer. Run dmesg and pay attention to the last lines. What is the most recent line? ◦ Bearing in mind the output from the previous command, run ls /dev/sd* and make sure your USB drive appears in the listing. What is the output? ◦ Now remove the USB drive and run dmesg again. How does the most recent line read? Linux Essentials (Version 1.6) | 4.3 Where Data is Stored Version: 2022-04-29     |     Licensed for Cyber School.     |     learning.lpi.org         |     315 ◦ Run ls /dev/sd* again and make sure your device disappeared from the listing. What is the output? Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System 316     |         learning.lpi.org     |     Licensed for Cyber School.     |     Version: 2022-04-29 Summary In the context of data storage, the following topics have been discussed in this lesson: process management and system logging and messaging. Regarding process management, we have learned the following: • Programs generate processes and processes exist in a hierarchy. • Every process has a unique identifier (PID) and a parent process identifier (PPID). • top is a very useful command to dynamically and interactively explore the running processes of the system. • ps can be used to obtain a snapshot of the current running processes in the system. • The /proc directory includes directories for every running process in the system named after their PIDs. • The concept of system load average—which is very useful to check on CPU utilization/overloading. Concerning system logging, we must remember that: • A log is a file where system events are recorded. Logs are invaluable when it comes to troubleshooting. • Logging has traditionally been handled by special services such as syslog, syslog-ng or rsyslog. Nevertheless, some programs use their own logging daemons. • Because logs are variable data, they are kept in /var and—sometimes—their names can give you a clue about their content (kern.log, auth.log, etc.) • Most logs are written in plain text and can be read with any text editor as long as you have the right permissions. However, a few of them are binary and must be read using special commands. • To avoid problems with disk space, log rotation is carried out by the logrotate utility. • As for the kernel, it uses a circular data structure—the ring buffer—where boot messages are kept (old messages fade away over time). • The system and service manager systemd replaced System V init in virtually all distros with journald becoming the standard logging service. • To read systemd’s journal, the journalctl utility is needed. Commands used in this lesson: Linux Essentials (Version 1.6) | 4.3 Where Data is Stored Version: 2022-04-29     |     Licensed for Cyber School.     |     learning.lpi.org         |     317 cat Concatenate/print file content. dmesg Print the kernel ring buffer. echo Display a line of text or a newline. file Determine file type. grep Print lines matching a pattern. last Show a listing of last logged in users. less Display contents of file one page at a time. ls List directory contents. journalctl Query the systemd journal. tail Display the last lines of a file.

Answers to Guided Exercises 1. Have a look the following listing of top and answer the following questions: carol@debian:~$ top top – 13:39:16 up 31 min, 1 user, load average: 0.12, 0.15, 0.10 Tasks: 73 total, 2 running, 71 sleeping, 0 stopped, 0 zombie %Cpu(s): 1.1 us, 0.4 sy, 0.0 ni, 98.6 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st KiB Mem : 1020332 total, 698700 free, 170664 used, 150968 buff/cache KiB Swap: 1046524 total, 1046524 free, 0 used. 710956 avail Mem   PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND   605 nobody 20 0 1137620 132424 34256 S 6.3 13.0 1:47.24 ntopng   444 www-data 20 0 364780 4132 2572 S 0.3 0.4 0:00.44 apache2   734 root 20 0 95212 7004 6036 S 0.3 0.7 0:00.36 sshd   887 carol 20 0 46608 3680 3104 R 0.3 0.4 0:00.03 top   1 root 20 0 56988 6688 5240 S 0.0 0.7 0:00.42 systemd   2 root 20 0 0 0 0 S 0.0 0.0 0:00.00 kthreadd   3 root 20 0 0 0 0 S 0.0 0.0 0:00.09 ksoftirqd/0   4 root 20 0 0 0 0 S 0.0 0.0 0:00.87 kworker/0:0 (…) ◦ Which processes have been started by the user carol? Answer: Only one: top. ◦ What virtual directory of /proc should you visit to search for data regarding the top command? Answer: /proc/887 ◦ What process was run first? How can you tell? Answer: systemd. Because it is the one with PID #1. ◦ Complete the table specifying in what area of top output the following information is found: Information about … Summary Area Task Area Memory Yes Yes Swap Yes No Linux Essentials (Version 1.6) | 4.3 Where Data is Stored Version: 2022-04-29     |     Licensed for Cyber School.     |     learning.lpi.org         |     319 Information about … Summary Area Task Area PID No Yes CPU time Yes Yes Commands No Yes 2. What command is used to read the following binary logs? ◦ /var/log/wtmp Answer: last ◦ /var/log/btmp Answer: lastb ◦ /run/log/journal/2a7d9730cd3142f4b15e20d6be631836/system.journal Answer: journalctl 3. In combination with grep, what commands would you use to find out the following information about your Linux system? ◦ When the system was last rebooted (wtmp) Answer: last ◦ Which hard disk are installed (kern.log) Answer: less /var/log/kern.log ◦ When the last login occurred (auth.log) Answer: less /var/log/auth.log 4. What two commands would you use to have the kernel ring buffer displayed? dmesg and journalctl -k (also journalctl –dmesg). 5. Indicate where the following log messages belong: ◦ Jul 10 13:37:39 debian dbus[303]: [system] Successfully activated service ‘org.freedesktop.nm_dispatcher’ Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System 320     |         learning.lpi.org     |     Licensed for Cyber School.     |     Version: 2022-04-29 /var/log/auth.log /var/log/kern.log /var/log/syslog X /var/log/messages ◦ Jul 10 11:23:58 debian kernel: [ 1.923349] usbhid: USB HID core driver /var/log/auth.log /var/log/kern.log X /var/log/syslog /var/log/messages X Jul 10 14:02:53 debian sudo: pam_unix(sudo:session): session opened for user root by carol(uid=0) /var/log/auth.log X /var/log/kern.log /var/log/syslog /var/log/messages ◦ Jul 10 11:23:58 debian NetworkManager[322]: [1562750638.8672] NetworkManager (version 1.6.2) is starting… /var/log/auth.log /var/log/kern.log /var/log/syslog /var/log/messages X 6. Have journalctl query information about the following units: Unit Command ssh journalctl -u ssh.service networking journalctl -u networking.service rsyslog journalctl -u rsyslog.service cron journalctl -u cron.service Linux Essentials (Version 1.6) | 4.3 Where Data is Stored Version: 2022-04-29     |     Licensed for Cyber School.     |     learning.lpi.org         |     321 Answers to Explorational Exercises 1. Reconsider the top output from the guided exercises and answer the following questions: ◦ What two steps would you follow to kill the apache web server? First, press k; then provide a kill value. ◦ In the summary area, how could you display the information about physical memory and swap using progress bars? By pressing m once or twice. ◦ Now, sort the processes by memory usage: M ◦ Now that you have memory information displayed in progress bars and processes sorted by memory usage, save these configurations so that you get them as default next time you use top: W ◦ What file stores top’s configuration settings? Where does it live? How can you check for its existence? The file is ~/.config/procps/toprc and lives in the user’s home directory (~). Since it is a hidden file (it resides in a directory whose name starts with a dot), we can check for its existence with ls -a (list all files). This file can be generated by pressing Shift + W while in top. 2. Learn about the command exec in Bash. Try to demonstrate its functionality by starting a Bash session, finding the Bash process with ps, then run exec /bin/sh and search for the process with the same PID again. exec replaces a process with another command. In the following example we can see that the Bash process is replaced by /bin/sh (instead of /bin/sh becoming a child process): Linux Essentials (Version 1.6) | Topic 4: The Linux Operating System 322     |         learning.lpi.org     |     Licensed for Cyber School.     |     Version: 2022-04-29 $ echo $$ 19877 $ ps auxf | grep 19877 | head -1 carol 19877 0.0 0.0 7448 3984 pts/25 Ss 21:17 0:00 \_ bash $ exec /bin/sh sh-5.0$ ps auxf | grep 19877 | head -1 carol 19877 0.0 0.0 7448 3896 pts/25 Ss 21:17 0:00 \_ /bin/sh 3. Follow these steps to explore kernel events and udev’s dynamic management of devices: ◦ Hotplug a USB drive into your computer. Run dmesg and pay attention to the last lines. What is the most recent line? You should get something along the lines of [ 1967.700468] sd 6:0:0:0: [sdb] Attached SCSI removable disk. ◦ Bearing in mind the output from the previous command, run ls /dev/sd* and make sure your USB drive appears in the listing. What is the output? Depending on the number of devices connected to your system, you should get something like /dev/sda /dev/sda1 /dev/sdb /dev/sdb1 /dev/sdb2. In our case, we find our USB drive (/dev/sdb) and its two partitions (/dev/sdb1 and /dev/sdb2). ◦ Now remove the USB drive and run dmesg again. How does the most recent line read? You should get something along the lines of [ 2458.881695] usb 1-9: USB disconnect, device number 6. ◦ Run ls /dev/sd* again and make sure your device disappeared from the listing. What is the output? In our case, /dev/sda /dev/sda1.