cd /tmp
# slow
dd if=/dev/zero of=./ddfile bs=1G count=3
# fastest
truncate -s 3G ./trunfile
# fast
fallocate -l 3G ./fallfile

# sync, depends on you needs
# if no sync, files may be still in memory
sync

# ls -ltrh
# in logical view, they are all 3GB
-rw-r--r--. 1 root root 3.0G Dec 30 07:01 ddfile
-rw-r--r--. 1 root root 3.0G Dec 30 07:02 trunfile
-rw-r--r--. 1 root root 3.0G Dec 30 07:02 fallfile

# check physical storage
# truncate space almost does not count 
# because of sparse
df -h .

# time test
# several seconds
time /bin/cp ./ddfile ~
time /bin/cp ./fallfile ~

# near instant
time /bin/cp ./trunfile ~

dd if=/dev/zero of=sparse_file bs=1 count=0 seek=3G
# Block is 0
stat sparse_file
Size: 3221225472	Blocks: 0          IO Block: 4096   regular file

dd if=/dev/urandom of=/dev/sdb1 bs=1M count=2048

dd if=/dev/zero of=./empty bs=1G count=3
# see the block number before and after change
stat ./empty
Size: 3221225472	Blocks: 6291544    IO Block: 4096   regular file

# deallocate empty block
fallocate -d ./empty

stat ./empty
Size: 3221225472	Blocks: 0          IO Block: 4096   regular file

# activate the venv first
# need to be the same yamllint pylint version as the team use
# different version has different output maybe 
pip install yamllint==1.17.0
pip install pylint==2.12.2

# filter and get updated yaml and python file name list
# diff between HEAD~1 HEAD, order matters!
# the between is for endpoint not for range!
YAML_FILES=$(git diff --name-only --diff-filter=ACMR HEAD~1 HEAD | grep -E "(.+\.yaml$|.+\.yml$)" || echo "")
PY_FILES=$(git diff --name-only --diff-filter=ACMR HEAD~1 HEAD | grep -E ".+\.py" || echo "")

# .pylintrc and .yamllint
# should be in the code repo
# -r: no run if input is empty
# xargs default input will be placed at end
echo $PY_FILES | xargs -r pylint -sn --rcfile=.pylintrc >> linter_results_tmp
echo $YAML_FILES | xargs -r yamllint -c .yamllint >> linter_results_tmp

cat linter_results_tmp && rm -f linter_results_tmp

# up
VAGRANT_EXPERIMENTAL="disks" vagrant up
# down
vagrant destroy -f

# check block device, their name, type, size and mount point
# sdb and sdc are the additional disks added in Vagrantfile
lsblk
NAME                    MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda                       8:0    0  40G  0 disk
├─sda1                    8:1    0   1G  0 part /boot
└─sda2                    8:2    0  31G  0 part
  ├─centos_centos7-root 253:0    0  29G  0 lvm  /
  └─centos_centos7-swap 253:1    0   2G  0 lvm  [SWAP]
sdb                       8:16   0   2G  0 disk
sdc                       8:32   0   2G  0 disk

# /dev/sda is not used up, let's add one more partition of 3GB
fdisk /dev/sda
Command (m for help): n
Partition type:
   # we have sda1 sda2 2 primart already
   p   primary (2 primary, 0 extended, 2 free)
   e   extended
Select (default p):
Using default response p
Partition number (3,4, default 3):
First sector (67108864-83886079, default 67108864):
Using default value 67108864
Last sector, +sectors or +size{K,M,G} (67108864-83886079, default 83886079): +3GB
Partition 3 of type Linux and of size 2.8 GiB is set

Command (m for help): w
The partition table has been altered!

Calling ioctl() to re-read partition table.

WARNING: Re-reading the partition table failed with error 16: Device or resource busy.
The kernel still uses the old table. The new table will be used at
the next reboot or after you run partprobe(8) or kpartx(8)
Syncing disks.

# re-read partition table for /dev/sda
partprobe -s /dev/sda

# now see lsblk, sda3
NAME                    MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda                       8:0    0   40G  0 disk
├─sda1                    8:1    0    1G  0 part /boot
├─sda2                    8:2    0   31G  0 part
│ ├─centos_centos7-root 253:0    0   29G  0 lvm  /
│ └─centos_centos7-swap 253:1    0    2G  0 lvm  [SWAP]
└─sda3                    8:3    0  2.8G  0 part
sdb                       8:16   0    2G  0 disk
sdc                       8:32   0    2G  0 disk

# format sda3
mkfs -t ext4 /dev/sda3

# see type
blkid | grep sda3
/dev/sda3: UUID="7d4a365c-1639-41de-a7c7-ebe79ea2830c" TYPE="ext4"

# mount to /data3 folder
mkdir /data3
mount /dev/sda3 /data3
mount | grep sda3
# check lsblk
# now see lsblk
NAME                    MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda                       8:0    0   40G  0 disk
├─sda1                    8:1    0    1G  0 part /boot
├─sda2                    8:2    0   31G  0 part
│ ├─centos_centos7-root 253:0    0   29G  0 lvm  /
│ └─centos_centos7-swap 253:1    0    2G  0 lvm  [SWAP]
└─sda3                    8:3    0  2.8G  0 part /data3
sdb                       8:16   0    2G  0 disk
sdc                       8:32   0    2G  0 disk

# check space and inode usage
cd /data3
df -k .
df -i .

# format whole disk sdb without partition
# use btrfs and mount with compress
mkfs -t btrfs /dev/sdb

mkdir /datab
mount /dev/sdb -o compress /datab

mount | grep sdb
/dev/sdb on /datab type btrfs (rw,relatime,seclabel,compress=zlib,space_cache,subvolid=5,subvol=/)

# check lsblk
NAME                    MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda                       8:0    0   40G  0 disk
├─sda1                    8:1    0    1G  0 part /boot
├─sda2                    8:2    0   31G  0 part
│ ├─centos_centos7-root 253:0    0   29G  0 lvm  /
│ └─centos_centos7-swap 253:1    0    2G  0 lvm  [SWAP]
└─sda3                    8:3    0  2.8G  0 part /data3
sdb                       8:16   0    2G  0 disk /datab
sdc                       8:32   0    2G  0 disk

# list partitions and see type of a disk
# `p` to see part 
# `n` to add new part
# `d` to delete part
# `w` to confirm
# `q` to quit
# /dev/sda is disk type
fdisk /dev/sda

# or
ls /dev/sda*

# format as ext4 type to /dev/sdb1 partition
# -t: type
# -f: force reformat
mkfs -t ext4 -f /dev/sdb1
mkfs -t xfs -f /dev/sdb1

mount /dev/sdb1 /data

# non-disk filesystem, system will do special mounts for you
# -t: type
mount -t proc proc /proc
mount -t sysfs sysfs /sys
mount -t debugfs debugfs /sys/kernel/debug
# NFS
mount server:dir /nfs-data

umount /data

man 5 filesystemd
man 5 ext4
man 5 xfs
man 5 btrfs

lvm
# show all physical volume
> pvs -a
# find lv on physical volume 
> pvck /dev/sda2
# show logical volume group
> vgs

# create physical volume
pvcreate /dev/sdc1
pvcreate /dev/sdc2
pvcreate /dev/sdc3

# list physical volume
# you can see volume group name
pvscan

# display detail
pvdisplay /dev/sdc1

# -c n: cluster no
# vg1: group name
vgcreate -c n vg1 /dev/sdc1 /dev/sdc2
pvscan
# add /dev/sdc3 to group
vgextend vg1 /dev/sdc3
pvscan
# remove unused pv
vgreduce vg1 /dev/sdc3
pvscan
# remove group
vgremove vg1
pvscan

# create a logical volume 600M from group vg1
# -L: size
# -n: lv name
lvcreate -L 600M -n apple vg1
# format and mount it
mkfs -t ext4 /dev/vg1/apple
mkdir /apple && mount /dev/vg1/apple /apple

umount /apple
# extend
# 注意不能超过所在group的总大小
# it will also extend filesystem automatically
lvextend -L +200M -r /dev/vg1/apple

# shrink, first resuze filesystem
# then lv
fsadm -e resize /dev/vg1/apple 300M
lvresize --size 300M /dev/vg1/apple 

# remove and reduce for logical volume
lvremove
lvreduce

# show physical volume info
pvdisplay
# show lv info
lvdisplay
# show volume group info
vgdisplay

# a partition
mkswap /dev/sdc2
# on and off, not persistent
swapon /dev/sdc2
swapoff /dev/sdc2

# check swap components
swapon -s

# generate 1G file
# or using truncate or fallocate
dd if=/dev/zero of=/tmp/myswap bs=1G count=1
chown root:root /tmp/myswap
chmod 0600 /tmp/myswap
# enable swap
mkswap /tmp/myswap
swapon /tmp/myswap
# off
swapoff /tmp/myswap

# shrink or extend file to specific size
# much faster than 'dd' command
truncate -s 1g /tmp/loop.img
# you can make partition on it or skip this step
# for example, make 2 partitions 200M and 400M each
fdisk /tmp/loop.img

# create loopback device from a file, will associate file with /dev/loop0
# -f: find unused loop device
# -P: force kernel to scan partition table on newly create loop device
losetup -fP /tmp/loop.img
# now from lsblk, you will see the partitions under /dev/loop0

# format
mkfs -t ext4 /dev/loop0p1
mkfs -t xfs /dev/loop0p2

# mount it
mkdir /loop{0,1}
mount /dev/loop0p1 /loop0
mount /dev/loop0p2 /loop1
mount | grep loop0
# unmount 
umount /loop0
umount /loop1

truncate -s 1g /tmp/loop.img2
mkdir /loop2
# must first format the file
mkfs -t xfs /tmp/loop.img2
# -o: option is loop
# asociate /dev/loop1 to the file
# no need losetup command
mount -o loop /tmp/loop.img2 /loop2

# unmount 
umount /loop2

# level1: mirror
# level5: redundancy
mdadm --create --verbose /dev/md/myraid --level=5 --raid-devices=3 /dev/sdd{1,2,3}
mkfs -t ext4 /dev/md/myraid
mkdir /mydir && mount /dev/md/myraid /mydir
# check
lsblk -o name,size,fstype,type
# cancel
umount /mydir
mdadm --stop  /dev/md/myraid

# on client install
sudo apt-get install sshfs
# or
yum install -y fuse-sshfs

mkdir /sshdir
# mount remote root home directory
# the connection may be flakely
sshfs [user]@<hostname or ip>:[dir] /sshdir [options]

# check on /sshdir side host
cd /sshdir && df -h .
mount | grep ssh

# unmount
fusermount -u /sshdir

# providing boot menu and excuting kernel
# -N: show line number
sudo less -N /boot/grub2/grub.cfg

# uname -r
sudo yum list installed kernel-*
# see new kernel available
sudo yum list available kernel

# update
sudo yum update -y kernel
# then reboot and check kernel version
sudo reboot

# check kernel index list, index starts from 0
sudo awk -F\' '$1=="menuentry " {print $2}' /etc/grub2.cfg

CentOS Linux (5.4.125-1.el7.elrepo.x86_64) 7 (Core)
CentOS Linux (3.10.0-1160.31.1.el7.x86_64) 7 (Core)
CentOS Linux (3.10.0-1160.25.1.el7.x86_64) 7 (Core)
CentOS Linux (0-rescue-adbe471f40421bfbf841690042db23fd) 7 (Core)

# set kernel index 0
sudo grub2-set-default 0
# reconfig boot loader code
sudo grub2-mkconfig -o /boot/grub2/grub.cfg
sudo reboot

uname -r

# # set kernel index 2, see above index list
sudo grub2-set-default 2
# reconfig boot loader code
sudo grub2-mkconfig -o /boot/grub2/grub.cfg
sudo reboot

cat /etc/centos-release
# or
rpm -qa centos-release

systemctl status rsyslog

#### RULES ####

# Log all kernel messages to the console.
# Logging much else clutters up the screen.
#kern.*                                                 /dev/console

# Log anything (except mail) of level info or higher.
# Don't log private authentication messages!
*.info;mail.none;authpriv.none;cron.none                /var/log/messages

# The authpriv file has restricted access.
authpriv.*                                              /var/log/secure

# Log all the mail messages in one place.
mail.*                                                  -/var/log/maillog


# Log cron stuff
cron.*                                                  /var/log/cron

# Everybody gets emergency messages
*.emerg                                                 :omusrmsg:*

# Save news errors of level crit and higher in a special file.
uucp,news.crit                                          /var/log/spooler

# Save boot messages also to boot.log
local7.*                                                /var/log/boot.log

# you will see it in /var/log/messages
logger "hello"
# -p: priority
logger -p local4.info " This is a info message from local 4" 

# persist journald by making a dir
sudo mkdir -p /var/log/journal
sudo systemctl restart systemd-journald
# you will see journal records here
ls -l /var/log/journal

journalctl --since "2020-12-11 15:44:32"
# time left off is 00:00:00 midight
journalctl --since "2020-10-01" --until "2020-10-03 03:00"
journalctl --since yesterday
journalctl --since 09:00 --until "1 hour ago"

# list boots
journalctl --list-boots
# check last boot journal
# -b -1: last boot
sudo reboot
sudo journalctl -b -1

# combine
journalctl -u nginx.service -u php-fpm.service --since today

# pid, uid, gid
journalctl _PID=8088
journalctl _UID=33 --since today

# -F: show available values
journalctl -F _GID
journalctl -F _UID

# check executable
journalctl /usr/bin/bash

# display only kernel message
journalctl -k

# by priority, can use number or name
#0: emerg
#1: alert
#2: crit
#3: err
#4: warning
#5: notice
#6: info
#7: debug
journalctl -p err -b

# the same as tail -n/-f
journalctl -n 10
journalctl -f

# disk usage
journalctl --disk-usage
# shrink
sudo journalctl --vacuum-size=1G
sudo journalctl --vacuum-time=1years

# print all on stdout, no pager with less
journalctl --no-pager

# -o output format
#cat: Displays only the message field itself.
#export: A binary format suitable for transferring or backing up.
#json: Standard JSON with one entry per line.
#json-pretty: JSON formatted for better human-readability
#json-sse: JSON formatted output wrapped to make add server-sent event compatible
#short: The default syslog style output
#short-iso: The default format augmented to show ISO 8601 wallclock timestamps.
#short-monotonic: The default format with monotonic timestamps.
#short-precise: The default format with microsecond precision
#verbose: Shows every journal field available for the entry, including those usually hidde 
#internally.
journalctl -b -u nginx -o json
journalctl -b -u nginx -o json-pretty

# list sessions
loginctl list-sessions

# session status 
# you can see the user action history
loginctl session-status [session id]
loginctl show-session [session id]
loginctl kill-session [session id]

systemctl --version

man 5 systemd.service

# start on boot
# This hooks it up to a certain boot “target”
# causing it to be triggered when that target is started.
sudo systemctl enable nginx.service
sudo systemctl disable nginx.service

# start and stop
sudo systemctl start nginx.service
sudo systemctl stop nginx.service

# when change the configuration of service
# To attempt to reload the service without 
# interrupting normal functionality
sudo systemctl reload nginx.service
# if reload is not available, restart instead
sudo systemctl reload-or-restart nginx.service

# status overview, you can see:
# unit file path
# drop-in
# enabled or disabled at vendor and custom
# up time
# Cgroup
systemctl status nginx.service

# find overridden config files for all units
# check unit drop in config snippets
sudo systemd-delta

# may need sudo
systemctl is-active nginx.service
systemctl is-enabled nginx.service
systemctl is-failed nginx.service

# list enabled units only
systemctl [list-units]

# list all of the units that systemd has loaded or attempted to load into memory
# include not currently active
systemctl list-units --all [--state=active|inactive|failed] [--type=service|target]
# list failed daemons, this is useful when reboot VM but daemons not
systemctl list-units --state failed

# show every available units installed on the system
systemctl list-unit-files

# after mask, cannot enable or start service
sudo systemctl mask nginx.service
sudo systemctl unmask nginx.service

# show unit file content and path
# if has overriding snippet, will show them as well
systemctl cat nginx.service

# list dependencies 
systemctl list-dependencies nginx.service [--all] [--reverse] [--before] [--after]

# low-level detail of unit
# all key=values
# -p: display a single property
systemctl show nginx.service [-p ExecStart]

# edit unit file
sudo systemctl edit --full nginx.service
# append unit file snippet
sudo systemctl edit nginx.service
# then reload to pick up changes
sodu systemctl daemon-reload

# all available targets
systemctl list-unit-files --type=target

# current default target
systemctl get-default

# set default target
sudo systemctl set-default multi-user.target
sudo systemctl set-default runlevel3.target

# see what units are tied to a target
systemctl list-dependencies multi-user.target

# show all active targets
systemctl list-units --type=target

# check units that will be kept alive
systemctl list-dependencies multi-user.target
# transition
sudo systemctl isolate multi-user.target

sudo systemctl poweroff
sudo systemctl reboot
# boot into rescue mode (single-user)
sudo systemctl rescue

lrwxrwxrwx. 1 root root          16 Jun  6  2020 halt -> ../bin/systemctl
lrwxrwxrwx. 1 root root          16 Jun  6  2020 poweroff -> ../bin/systemctl
lrwxrwxrwx. 1 root root          16 Jun  6  2020 reboot -> ../bin/systemctl
lrwxrwxrwx. 1 root root          16 Jun  6  2020 runlevel -> ../bin/systemctl
lrwxrwxrwx. 1 root root          16 Jun  6  2020 shutdown -> ../bin/systemctl
lrwxrwxrwx. 1 root root          16 Jun  6  2020 telinit -> ../bin/systemctl

# available time zone
timedatectl list-timezones
# set time zone
sudo timedatectl set-timezone America/Los_Angeles
# check
timedatectl status

# or using UTC
# --utc: time zone
journalctl --utc

# full log
journalctl -e

# -u: unit, but some log may be missing in ExecStartPre
# -b: limit to current boot
# -e: show ending logs
# -r: show in reverse order
# -f: tail log
journalctl -u nginx.service [--since] [--until] [-e]

# combine related units
# good for debug
journalctl -u nginx.service -u php-fpm.service --since today

# use `b` and `f` to move page
# -e: tail
# -x: more details
journalctl -ex

yum install -y cloud-init
# you can see these services
systemctl cat cloud-init-local
systemctl cat cloud-init
systemctl cat cloud-config.service
systemctl cat cloud-final.service

#cloud-config

Unhandled non-multipart (text/x-not-multipart) userdata ...

# -1: MD5
openssl passwd -1
# -salt: add salt
openssl passwd -1 -salt yoursalt
# from stdin
echo 'admin' | openssl passwd -1 -stdin -salt yoursalt

# fetch all remote tags
git fetch --tags
# list all tags
git tag [--list]
# delete tag
git tag -d <tag name>
# Delete a tag from the server with push tags
git push --delete origin <tag name>

# in your current branch tag latest commit
# create annotated tag, add more info
# will open editor
git tag -a 0.1.0

# tag specific commit
git tag -a 0.1.0 <commit hash>

# push tag to remote
git push origin <branch> --tags

## can see commit detail, tag annotation, and so on.
git show [tag|or anything]

# we have some hot fixes on release branch, then create a release tag on it for production
# finally merge into master
master ------ + ----- + -----+ --- + --------------- + ------>
               \                               /
                \       hotfix   hotfix       /
                 \-------- + ----- + --------/
      release branch           (release tag)

# on master branch
# checkout to a tag (detched HEAD)
git checkout v1.1
# create release branch on release tag
git checkout -b rb1.1

# or in one command to checkout a tag to a new branch
git checkout tags/v1.1 -b rb1.1

# make fix
<hot fix>

# commit
git commit -m "hotfix"
git tag -a v1.1.1 -m "hotfix"
# merge
git checkout master
# or rebase/squash or fast-forward
git merge rb1.1 -m "merged hotfix"
# delete branch, the tag is still there, because you merged to master
git branch -d rb1.1

vagrant up
# ssh to server
vagrant ssh [gitlab]
# ssh to worker
vagrant ssh runner
# destroy uses
vagrant destroy -f

# Update package manager and install prerequisites
sudo apt-get update
sudo apt-get install -y curl openssh-server ca-certificates
# we don't need email in this case, so skip it

# Set up gitlab apt repository
curl https://packages.gitlab.com/install/repositories/gitlab/gitlab-ce/script.deb.sh | sudo bash

# Install gitlab
# this is the IP address in vagrant file
# gitlab-ce is community edition
sudo EXTERNAL_URL="http://192.168.50.10" apt-get install gitlab-ce

sudo apt-get update

sudo apt-get install -y \
    apt-transport-https \
    ca-certificates \
    curl \
    gnupg-agent \
    software-properties-common

# add docker's official GPG key
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -

# add stable repository
sudo add-apt-repository \
   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
   $(lsb_release -cs) \
   stable"

# install docker
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io

# verify install good
sudo docker run hello-world

# name is gitlab-runner
# -v: will create folder automatically
sudo docker run -d --name gitlab-runner --restart always \
    -v /srv/gitlab-runner/config:/etc/gitlab-runner \
    -v /var/run/docker.sock:/var/run/docker.sock \
    gitlab/gitlab-runner:latest

# later gitlab-runner is command
# register is argument
sudo docker exec -it gitlab-runner gitlab-runner register

# command prompt:
Enter the GitLab instance URL (for example, https://gitlab.com/):
# from runner expand
http://192.168.50.10/
Enter the registration token:
# from runner expand
K5G9S5e5wmcdoANUGLF4
Enter a description for the runner:
[5922b65a9261]: docker
Enter tags for the runner (comma-separated):
# gitlab-ci will refer this tag
docker-tag
Registering runner... succeeded                     runner=K5G9S5e5
Enter an executor: docker, docker-ssh, virtualbox, docker+machine, docker-ssh+machine, custom, parallels, shell, ssh, kubernetes:
docker
Enter the default Docker image (for example, ruby:2.6):
# this can be overrided later
alpine:latest

var=
# 空
echo $var
# true
[[ -z $var ]]
# false
[[ -n $var ]]

set -u
unset var
# 报错
echo $var

# `w` can be literal or another variable

# if a is unset, value of w is used
a=${a-w}
# if a is unset or null(empty), value of w is used
# for example, used for positional parameter passed from outside
a=${a:-w}

# if a is unset or null(empty), value of w is assigned to a
# 不能用于positional parameters的赋值, 比如 ${3:=hello}
${a:=w}

# if a is unset or null(empty), w is written to stderr
# and script exits with err
a=${a:?w}

# return variable name starts with prefix
# these 2 are identical
${!prefix*} 
${!prefix@}
# return all BASH prefixed variables
echo ${!BASH*}

parameter="var"
var="hello"

# echo is hello
echo ${!parameter}

$ string=01234567890abcdefgh
# 7 is start index
$ echo ${string:7}
7890abcdefgh
# 0 is number of char to cut
$ echo ${string:7:0}

$ echo ${string:7:2}
78
$ echo ${string:7:-2}
7890abcdef
# 要空格, 防止和:- 混淆
$ echo ${string: -7}
bcdefgh
$ echo ${string: -7:0}

$ echo ${string: -7:2}
bc
$ echo ${string: -7:-2}
bcdef

# set $1 positional parameter
$ set -- 01234567890abcdefgh
$ echo ${1:7}
7890abcdefgh
$ echo ${1:7:0}

$ echo ${1:7:2}
78
$ echo ${1:7:-2}
7890abcdef
$ echo ${1: -7}
bcdefgh
$ echo ${1: -7:0}

$ echo ${1: -7:2}
bc
# -2:  start from end
$ echo ${1: -7:-2}
bcdef

# array
$ array[0]=01234567890abcdefgh
$ echo ${array[0]:7}
7890abcdefgh
$ echo ${array[0]:7:0}

$ echo ${array[0]:7:2}
78
$ echo ${array[0]:7:-2}
7890abcdef
$ echo ${array[0]: -7}
bcdefgh
$ echo ${array[0]: -7:0}

$ echo ${array[0]: -7:2}
bc
$ echo ${array[0]: -7:-2}
bcdef

# get string length
${#string}
# note, the number of positional parameters
${#@}

# 检查第一个位置参数是不是以-开头，扩展结果是删除最短的match部分
# 对 ${1} 的操作, for example ${1} is --verbose
# get: -verbose
${1#-} 
# 同上，但删除最长的match部分
# get: verbose
${1##-}

# get filename name from a download url
# \ is used to escape / in path
${1##*\/}
# get path of a url
${1%\/*}

# empty the whole string if substring target is inside
[ ! -z "${1##*target*}" ]

${1#+(-)}
${1##+(-)}

# remove leading space or blank
# note that double [[]] wrapper!
shopt -s extglob
${1##*([[:blank:]]|[[:space:]])}
# remove trailing space or blank
${1%%*([[:blank:]]|[[:space:]])}

# remove .tar.gz or .tgz suffix
${1%%(.tgz|.tar.gz)}

foo=JPG.JPG 
# replace first match
# jpg.JPG
echo ${foo/JPG/jpg} 
# replace all matches
# jpg.jpg
echo ${foo//JPG/jpg} 
# replace only start
# jpg.JPG
echo ${foo/#JPG/jpg} 
# replace only end
# JPG.jpg
echo ${foo/%JPG/jpg} 

ls *.txt
# ? is any one char
ls file?.txt
ls file??.txt

ls /usr/sbin/[A-Z]*
# 默认字典顺序 is actually in order
# aAbBcCdDeEfFgGhHiIjJkKlLmMnNoOpPqQrRsStTuUvVwWxXyYzZ
# 所以[A-Z] 不含a

echo $LANG
# usually is
en_US.UTF-8
# 所以对于上面[A-Z]*的正确写法是
ls /usr/sbin/[[:upper:]]*

# character set
# match one of them
ls [123abc]file.txt
ls file[0-9].txt
ls file[a-z9].txt

# 这个显示的结果和bash设置有关, 和上面提到的问题一样，不过这里更改了LC_COLLATE的值
# locale, In bash terminal, set LC_COLLATE=C (collation)
ls file[A-G].txt
# ! is inversion, not include a-z
ls file[!a-z].txt
# put at end to match !, it is a special char
ls file[a-z!].txt
ls file[az-].txt

# [:upper:] is the character class
# put char class in char set []
ls file[[:upper:]?].txt
ls file[[:lower:]?].txt
ls file[![:lower:][:space:]].txt

# numbers
[:digit:]
# upper and lower case
[:alpha:]
# upper and lower and numbers
[:alnum:]
# upper
[:upper:]
# lower
[:lower:]
# space, tab, carriage return, newline, vertical tab, and form feed.
# is superset of [:blank:]
[:space:]
# space and tab characters
[:blank:]

shopt | grep extglob
shopt -s extglob

touch file1.png photo.jpg photo photo.png file.png photo.png.jpg
rm -f file1.png photo.jpg photo photo.png file.png photo.png.jpg

# @(match): match one or others
# match photo.jpg
ls photo@(.jpg)
ls @(file)
# photo.jpg or photo.png
ls photo@(.jpg|.png)

# ?(match): match 0 or 1
ls photo?(.jpg|.png)

# +(match): match 1 or more
ls photo+(.jpg|.png)

# *(match): match 0 or more
ls photo*(.jpg|.png)

# !(match): invert match
# all files that do not have photo or file name and do not end with jpg or png
!(+(file|photo)*+(.jpg|.png))

# create file1.txt file2.txt file4.txt
touch file{1,2,4}.txt
touch file{1..1000}.txt

# expand from left to right, 两两组合
echo {a..c}{10..15}

# specify increase step
echo {1..100..2}
# can pad heading 0
echo {0001..10..2}
echo {10..0}

echo {a..z..2}

# can be nested
echo file-201{1..9}-{0{0..9},1{1..2}}-{1..30}.{tar,bak}.{tgz,bz2}
# create folder structure
mkdir -p 20{10..20}/{01-12}

# match before and after ,
# file file.bkp
cp -f a/long/path/file{,.bkp}

# match one char with zero to 3 occurances
.{,3}
# match one char with 3 to 7 occurances
.{3,7}
# match one char with 3 to more occurances
.{3,}

# \1 is (ss) pattern
(ss).*\1
(ss).*\1.*\1

# radar
# opapo
^(.)(.).\2\1$

[[ abcdef =~ b.d ]]
# the matched is bcd in BASH_REMATCH[0]
echo ${BASH_REMATCH[0]}
# if no match BASH_REMATCH[0] is null