shell編程系列10--文本處理三劍客之sed利用sed查詢特定內容

shell編程系列10--文本處理三劍客之sed利用sed查詢特定內容

利用sed查找文件內容:
    pattern種類：
        1、8p
        2、8,10p
        3、8,+5p
        4、/regexp/p
        5、8,/regexp/p
        6、/regexp/,8p

    1、打印/etc/passwd中第10行的內容
        sed -n '10p' /etc/passwd

    2、打印/etc/passwd中第8行開始，到第15行結束的內容
        sed -n '8,15p' /etc/passwd

    3、打印/etc/passwd中從第8行開始，而後+5行結束的內容
        sed -n '8,+5p' passwd

    4、打印/etc/passwd中開頭匹配nginx字符的內容
        sed -n '/^nginx/p' /etc/passwd

    5、打印/etc/passwd中開頭爲root的行開始，到ftp開頭的行結束的內容
        sed -n '/^root/,/^ftp/p' /etc/passwd
        
    6、打印/etc/passwd中第8行開始，到包含有/sbin/nologin的內容結束內容
        sed -n '8,/\/sbin\/nologin/p' /etc/passwd

    7、打印/etc/passwd中第一個包含/bin/bash內容的行開始，到第5行結束的內容
        sed -n '/\/bin\/bash/,5p' /etc/passwd 


    練習腳本1：
    需求描述：處理一個相似MYSQL配置文件my.cnf的文本，示例以下：
        編寫腳本實現如下功能：輸出文件有幾個段，而且針對每一個段能夠統計配置參數總個數

        預想輸出結果：
        1：client 2
        2: server 12
        3: mysqld 12
        4: mysqld_safe 7
        5: embedded 8
        6: mysqld-5.5 9

    function get_all_segment

    function count_items_in_segment

    # .表明任意一個字符，*表明0個或多個字符，.*合起來就是任意字符(包括空) ,-e 能夠連着使用屢次
    # sed -n '/\[.*\]/p' my.cnf | sed -e 's/\[//g' -e 's/\]//g'
    client
    mysqld
    mysqldump
    mysql
    myisamchk
    mysqld_safe
    mysqlhotcopy

    # 找到 [client] 段              過濾掉以 # 開頭的行和空行    而後過濾掉 [.*] 剩下的就是中間的內容了
    # sed -n '/\[client\]/,/\[.*\]/p' my.cnf |grep -v '^#' | grep -v '^$' | grep -v '\[.*\]'
    port                                                        = 3306
    socket                                                      = /tmp/mysql.sock

    # 具體腳本

    [root@localhost shell]# cat mysql_process.sh 
    #!/bin/bash
    #

    FILE_NAME=/data/shell/my.cnf
    # 獲取全部的片斷
    function get_all_segments
    {
        echo `sed -n '/\[.*\]/p' $FILE_NAME | sed -e 's/\[//g' -e 's/\]//g'`
    }

    #get_all_segments
    
    # 統計單個片斷 配置行數
    function count_items_in_segment
    {
        # 找到[xxx]而且下一個 [.*] 的項中間的內容就是一段配置
        #items=`sed -n '/\['$1'\]/,/\[.*\]/p' $FILE_NAME | grep -v "^#" | grep -v "^$" | grep -v "\[.*\]"`
        #sed -n '/\[client\]/,/\[.*\]/p' my.cnf |grep -v '^#' | grep -v '^$' | grep -v '\[.*\]'
        items=`sed -n '/\['$1'\]/,/\[.*\]/p' "$FILE_NAME" | grep -v "^$" | grep -v "^#" | grep -v "\[.*\]"| wc -l`
        echo $items

        # for 循環打印並計算my.cnf每一行的配置，適合配置項之間沒有空格的狀況
        #index=0
        #for item in $items;
        #do
        #    index=`expr $index + 1`
        #done

        #echo $index
    }


    number=0
    for segment in `get_all_segments`
    do
        number=`expr $number + 1`
        # 將每一個片斷當作參數傳進來計算
        items_count=`count_items_in_segment $segment`
        echo "$number: $segment $items_count"
    done

my.cnf 示例

[client]
port                                                        = 3306
socket                                                      = /tmp/mysql.sock

# The MySQL server
[mysqld]
user                                                        = mysql
port                                                        = 3306
bind-address                                                = 0.0.0.0
socket                                                      = /tmp/mysql.sock
datadir                                                     = /data/mysql_data
pid-file                                                    = /data/mysql_data/mysql.pid
basedir                                                     = /usr/local/mysql
tmpdir                                                      = /tmp

#此開關默認爲NULL，即不容許導入導出。
#secure-file-priv                                           = /opt/upload

#-------------------------------gobal variables------------------------#
#默認關閉，涉及到timestamp類型的列自動更新的問題
explicit_defaults_for_timestamp                 = 1
###transaction_write_set_extraction              = XXHASH64  #以便在server收集寫集合的同時將其記錄到二進制日誌。而且是行更改後的惟一標識此標識將用於檢測衝突。
###loose-group_replication_group_name            = 'ce9be252-2b71-11e6-b8f4-00212889f856' #組的名字能夠隨便起,但不能用主機的GTID
###loose-group_replication_start_on_boot         = off  #爲了不每次啓動自動引導具備相同名稱的第二個組,因此設置爲OFF。
###loose-group_replication_bootstrap_group       = off #同上
###loose-group_replication_local_address         = '192.168.1.88:33071' #寫本身主機所在IP
###loose-group_replication_group_seeds           ='192.168.1.88:33071,192.168.1.89:33071,192.168.1.90:33071'
###loose-group_replication_single_primary_mode   = off  #關閉單主模式的參數
###loose-group_replication_enforce_update_everywhere_checks = on #開啓多主模式的參數

skip-external-locking
skip-name-resolve
skip-ssl

#memory is 16G
key_buffer_size                                             = 32M
table_open_cache                                            = 2048
table_definition_cache                                      = 1024
sort_buffer_size                                            = 4M
net_buffer_length                                           = 32K
read_buffer_size                                            = 4M
read_rnd_buffer_size                                        = 16M

open_files_limit                                            = 10000
thread_cache_size                                           = 400
query_cache_type                                            = 0
query_cache_size                                            = 32M
max_write_lock_count                                        = 300
wait_timeout                                                = 120
interactive_timeout                                         = 120
net_read_timeout                                            = 120
net_write_timeout                                           = 120

max_connections                                             = 800
max_user_connections                                        = 750
max_connect_errors                                          = 10000
max_allowed_packet                                          = 512M
back_log                                                    = 2048
log_timestamps                                              = system
performance_schema                                          = OFF
character_set_server                                        = utf8mb4

##當連接數耗盡後，經過設置別用端口，讓root能夠登陸
extra_max_connections                                       = 2
extra_port                                                  = 13306

###讓mysql不區分大小寫敏感
lower_case_table_names                                      = 1

#explicit_defaults_for_timestamp                            = 1

#----------------Myisam--------------------------------#
myisam_recover_options                                      = DEFAULT
bulk_insert_buffer_size                                     = 64M
myisam_sort_buffer_size                                     = 128M
myisam_max_sort_file_size                                   = 512M
myisam_repair_threads                                       = 1

#if the query is exec time great than 2 seconds, the query will log to slow log if slowlog is enabled.
long_query_time                                             = 1
slow_query_log                                              = On
slow-query-log-file                                         = /data/mysql_data/slow.log
show_compatibility_56                                       = on

# Don't listen on a TCP/IP port at all. This can be a security enhancement,
# if all processes that need to connect to mysqld run on the same host.
# All interaction with mysqld must be made via Unix sockets or named pipes.
# Note that using this option without enabling named pipes on Windows
# (via the "enable-named-pipe" option) will render mysqld useless!
# 
#skip-networking

#----------------------------MySQL Log----------------#
# Replication Master Server (default)
# binary logging is required for replication
log-bin                                                     = mysql-bin
expire_logs_days                                            = 20
log_error                                                   = error.log
log_error_verbosity                                         = 1
log_warnings                                                = 1

# binary logging format - mixed recommended
binlog_format                                               = row
relay-log                                                   = mysql-relay-bin
relay-log-index                                             = relay.index
# required unique id between 1 and 2^32 - 1
server-id                                                   = 1
#sql-mode                                                    = "NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION"
#sql-mode                                                    = "NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION"
sync_binlog                                                 = 1
log_slave_updates                                           = 1
#binlog_checksum                                            = NONE

#------------------------------replicate--------------#
#排除不須要同步的庫表
#replicate-ignore-db                                        = mysql
#replicate-ignore-db                                        = sys
replicate-ignore-db                                         = information_schema
replicate-ignore-db                                         = performance_schema
replicate-ignore-db                                         = undolog
replicate-ignore-db                                         = for_nagios
replicate-ignore-db                                         = undolog

#replicate_wild_ignore_table                                = mysql.%
#replicate_wild_ignore_table                                = sys.%
replicate_wild_ignore_table                                 = information_schema.%
replicate_wild_ignore_table                                 = performance_schema.%
replicate_wild_ignore_table                                 = undolog.%
replicate_wild_ignore_table                                 = for_nagios.%
replicate_wild_ignore_table                                 = undolog.%

#主主複製須要開啓
#auto_increment_offset= 2
#auto_increment_increment= 2

#GTID模式複製，須要開啓以下
gtid_mode                                                  = ON
enforce_gtid_consistency                                   = ON

#併發複製
slave-parallel-type                                         = LOGICAL_CLOCK
slave-parallel-workers                                      = 2
master_info_repository                                      = TABLE
relay_log_info_repository                                   = TABLE
relay_log_recovery                                          = ON

#跳過slave進程啓動參數
skip-slave-start

#若是實例爲從庫，則須要設置爲on
#read_only                                                   = on

#skip-grant-tables

#--------------------------------------------------------innoDB------------#
innodb_rollback_on_timeout
# Uncomment the following if you are using InnoDB tables
innodb_data_home_dir                                        = /data/mysql_data
innodb_data_file_path                                       = ibdata1:1G;ibdata2:1G:autoextend
innodb_log_group_home_dir                                   = /data/mysql_data
innodb_undo_directory                                       = /data/mysql_data/undolog/
innodb_undo_logs                                            = 128
innodb_undo_tablespaces                                     = 3

# You can set .._buffer_pool_size up to 50 - 80 %
#innodb_use_sys_malloc = 0 
#innodb_page_size = 8192
innodb_buffer_pool_size                                     = 3G
innodb_buffer_pool_instances                                = 1
#innodb_additional_mem_pool_size = 8M

# Set .._log_file_size to 25 % of buffer pool size
innodb_log_file_size                                        = 256M
innodb_log_buffer_size                                      = 64M
innodb_log_files_in_group                                   = 3
#每次事務提交時MySQL都會把log buffer的數據寫入log file，而且flush(刷到磁盤)中去，該模式爲系統默認
innodb_flush_log_at_trx_commit                              = 1
innodb_lock_wait_timeout                                    = 120
#啓用獨立表空間
innodb_file_per_table                                       = 1

#CPU是1顆8核的，那麼能夠設置
innodb_read_io_threads                                      = 8
innodb_write_io_threads                                     = 8

#默認是0，則表示沒有併發線程數限制，全部請求都會直接請求線程執行，當併發用戶線程數量小於64，建議設置innodb_thread_concurrency=0，在大多數狀況下，最佳的值是小於並接近虛擬CPU的個數
innodb_thread_concurrency                                   = 12
innodb_max_dirty_pages_pct                                  = 75
innodb_flush_method                                         = O_DIRECT

innodb_purge_threads                                        = 10
innodb_large_prefix                                         = 1

#參數待測試
#innodb_io_capacity                                         = 20000
#innodb_io_capacity_max                                     = 40000

#根據CPU核心數來設定
thread_pool_size                                            = 8
#thread_handling = pool-of-threads
thread_pool_oversubscribe                                   = 24

#thread_handling                             = pool-of-threads
thread_pool_stall_limit                                     = 100
thread_pool_max_threads                                     = 30

#解釋： 在啓動時把熱數據加載到內存。
innodb_buffer_pool_load_at_startup                          = 1
#解釋： 在關閉時把熱數據dump到本地磁盤
innodb_buffer_pool_dump_at_shutdown                         = 1

##默認是8M, 若是一次insert數據量比較多的話, 能夠適當增長
innodb_autoextend_increment                                 = 32

[mysqldump]
quick
max_allowed_packet                                          = 512M

[mysql]
no-auto-rehash
# Remove the next comment character if you are not familiar with SQL
#safe-updates

[myisamchk]
#key_buffer_size = 20M
#sort_buffer_size = 20M
key_buffer_size                                             = 200M
sort_buffer_size                                            = 200M
read_buffer                                                 = 2M
write_buffer                                                = 2M

[mysqld_safe]
#控制文件打開數的show global status like 'open%file%';比較合適的設置：Open_files / open_files_limit * 100% <= 75%
open-files-limit                                            = 65535
log-error                                                   = /data/mysql_data/error.log

[mysqlhotcopy]
interactive-timeout