Differences between revisions 1 and 46 (spanning 45 versions)

Filter unique values from third column, separated by space

   1 cat datax.log |  awk -F ' ' '/IMEI/{print $3}' | sort | uniq | wc -l

Filter unique values from third column, separated by :

   1 cat datax.log |  awk -F ':' '/IMEI/{print $3}' | sort | uniq | wc -l

Search for text in files recursively

   1 cd startFolder
   2 grep -H "textToSearch" * -R

Copy files to other destination (Win32)

@echo off
REM Mount drive
echo Mounting remote drive
net use z: \\192.168.1.1\backs password /USER:userx

REM copy current day files
echo Copying current day files
forfiles -p "I:" -s -m *FULL*.sqb -d +0 -c "cmd /c copy @path Z:\BACKUPS\@file"

echo Deleting files older than 7 days
REM delete files older than 7 days
forfiles -p "Z:\BACKUPS" -m *.sqb -d 7 -c "cmd /c del @path"

Gzip files that where last modified in 30 or more days (Win32)

forfiles /p "c:\logs" /d -30 /m *.log /c "cmd /c gzip @path "

Run task every day at 01:00 (Win32)

at 01:00 /every:M,T,W,Th,F,S,Su c:\task.bat

Block responses to pings

   1 #block responses to pings
   2 iptables -A INPUT -p icmp -m icmp --icmp-type echo-request -j DROP
   3 iptables -L
   4 #restore responses to pings
   5 iptables -F
   6 iptables -L

Zip SVN modified Java files

   1 #!/bin/sh
   2 # chmod 755 /usr/local/bin/modifiedJavaSvn.sh # on CygWin
   3 svn status | awk '/.java/{ print( gensub( /\\/,"/","g",$2) ); }' | xargs -i zip modifiedJavaFiles.zip {}
   4 unzip -v modifiedJavaFiles.zip

Deploy EAR script

   1 #!/usr/bin/bash
   2 # deploy EAR in remote JBoss . 
   3 # Must have pub SSH keys (authorized_keys) deployed on remote server
   4 # Can be used with cygwin
   5 FILEX=sample.ear
   6 DEST_DIR=/opt/jboss/server/default/deploy/
   7 ECLIPSE_PROJECT=/home/userx/workspace/sample-ear/
   8 
   9 deploy()
  10 {
  11   SERVER=$1
  12   USER=$2
  13   echo "Deploying in $SERVER ..."
  14 
  15   cd $PROJECT
  16   scp target/$FILEX $USER@$SERVER:/tmp
  17   ssh $USER@$SERVER -C "mv /tmp/$FILEX $DEST_DIR/$FILEX"
  18   ssh $USER@$SERVER -C "ls -lah $DEST_DIR"
  19   ssh $USER@$SERVER -C "chmod 666  $DEST_DIR/$FILEX"
  20   ssh $USER@$SERVER -C "/sbin/service jboss abort"
  21   ssh $USER@$SERVER -C "sudo /sbin/service jboss start"
  22 }
  23 
  24 deploy "example-server" "userx"

List JAR contents to file

cd ~/.m2/repository
find . -name "*.jar" | xargs -i jar -tf {} > /tmp/explodedJars.txt
find . -name "*.jar" | xargs -i sh -c "echo {} ; jar -tf {}" > /tmp/explodedJars.txt

Size of each directory inside current directory

find . -maxdepth 1 -type d | xargs -i du {} -hs

Backup folders in HOME

   1 #!/bin/bash
   2 # cd ~
   3 # ln -s scripts/backup.sh backup.sh
   4 # sh backup.sh
   5 DATE=`date '+%Y%m%d%H%M%S'`
   6 FILENAME=~/backups/backup_$DATE.tar.gz
   7 cd ~
   8 mkdir -p ~/backups
   9 rm $FILENAME
  10 tar cvzf $FILENAME Documents GNUstep ThunderbirdMail Desktop .Skype scripts moin-1.9.7
  11 # tar tvzf backup_20150108103242.tar.gz
  12

Get installed updates on Windows

wmic qfe list brief /format:texttablewsys > updates.txt

Find files modified less than an hour ago

find . -mtime -1h -name "*.cfg"

Lock file to allow only one instance of a script to run

   1 #!/bin/sh
   2 LOCK_FILE=$(echo "/var/tmp/lockTest.lck")
   3 PID=$$
   4 LOCKED=0
   5 COUNT=0
   6 echo "Trying to get lock to $PID in $LOCK_FILE"
   7 
   8 while [ $LOCKED -eq "0" ] && [ $COUNT -le "3" ]; do
   9     if ( set -o noclobber; echo "$PID" > "$LOCK_FILE" ) 2> /dev/null; then
  10         LOCKED=1
  11     else        
  12         sleep 1
  13         COUNT=$(($COUNT+1))
  14         echo "Trying to get lock to $PID in $LOCK_FILE, retry $COUNT"
  15     fi    
  16 done
  17 
  18 if [ $LOCKED -eq "0" ]; then
  19     echo "Unable to get lock to $PID with $COUNT retries"
  20     exit 1
  21 else
  22     echo "Lock acquired to $PID"    
  23 fi   
  24 #####################################
  25 sleep 10
  26 #####################################
  27 rm -f "$LOCK_FILE"
  28 LOCKED=0
  29 echo "Released lock to $PID"
  30 exit 0

awk , load file into array

https://magvar.wordpress.com/2011/05/18/awking-it-how-to-load-a-file-into-an-array-in-awk/

   1 #! /bin/bash
   2 INPUTFILE="inputfile.csv"
   3 DATAFILE="data.csv"
   4 OUTFILE="output.csv"
   5 
   6 awk 'BEGIN {
   7 while (getline < "'"$INPUTFILE"'")
   8 {
   9 split($0,ft,",");
  10 id=ft[1];
  11 name=ft[2];
  12 
  13 key=id;
  14 data=name;
  15 nameArr[key]=data;
  16 }
  17 close("'"$INPUTFILE"'");
  18 
  19 while (getline < "'"$DATAFILE"'")
  20 {
  21 split($0,ft,",");
  22 id=ft[1]; # Id is the first column
  23 phone=ft[2]; # Phonenumber is the second
  24 name=nameArr[id]; # Get the name from the nameArr, using "id" as key
  25 print id","name","phone > "'"$OUTFILE"'"; # Print directly to the outputfile
  26 }
  27 }'

Shell script sample

   1 #!/bin/sh
   2 function replaceString {
   3   RES=$(echo $1 | sed "s/$2/$3/g")
   4   echo $RES
   5 }
   6 
   7 function add {
   8   echo $( echo "$1 + $2" | bc )    
   9 }
  10 
  11 function inc {
  12   echo $( expr $1 + 1 )  
  13 }
  14 ####################################
  15 echo $(add "1.5" "3.2" )
  16 
  17 TEXT1="AAA BBB CCC"
  18 
  19 for WORD in $TEXT1
  20 do
  21   echo ${WORD}
  22 done
  23 
  24 WITHOUTSPACES=$(replaceString "AAA BBB XXX" " " "" )
  25 echo $WITHOUTSPACES
  26 
  27 if [ $WITHOUTSPACES == "AAABBBXXX" ]
  28 then
  29   echo "Equal"
  30 fi
  31 
  32 if [ $WITHOUTSPACES != "AAA BBB XXX" ]
  33 then
  34   echo "Diff"
  35 fi
  36 
  37 A=$(ls / | wc -l)
  38 A=$(expr $A + 1)
  39 if [ $A -eq 22 ]
  40 then
  41   echo $A
  42 fi
  43 
  44 B="222"
  45 B=$(expr $B)
  46 if [ $B -eq 222 ]
  47 then
  48   echo $B
  49 fi
  50 
  51 COUNT="0"
  52 while [ $COUNT -lt 10 ]
  53 do
  54   COUNT=$( inc $COUNT )
  55 done
  56 
  57 $(test 0 -ge 0)
  58 RES=$(echo $?)
  59 echo $RES

Change TIME_WAIT (time wait) tcp/ip

Linux

echo 1 > /proc/sys/net/ipv4/tcp_fin_timeout
echo 1 > /proc/sys/net/ipv4/tcp_tw_recycle
echo 1 > /proc/sys/net/ipv4/tcp_tw_reuse

Freebsd

sysctl net.inet.tcp.msl=1500

Show root folders occupied space

   1 cd /tmp
   2 ls --color=never | xargs -i du {} -hs
   3 find . -type d -maxdepth 1 | xargs -i du {} -hs
   4 
   5 #~/.bashrc
   6 alias folderUsedSpace='find . -type d -maxdepth 1 | xargs -i du {} -hs'

Add swapfile 512MB

   1 df -h
   2 dd if=/dev/zero of=/swapfile1 bs=1024 count=524288
   3 chown root:root /swapfile1 
   4 chmod 0600 /swapfile1 
   5 mkswap /swapfile1 
   6 swapon /swapfile1
   7 free -m
   8 vi /etc/fstab
   9 # /swapfile1 none swap sw 0 0
  10 free -m
  11 htop

Alias current date

   1 alias currdate="date '+%Y%m%dT%H%M%S'"

Alias history without line number

   1 alias hist='history | cut -c 8-'

Calculator using bc

   1 #!/bin/sh
   2 echo "scale=3;$1" | bc

Get slackbuild

getslackbuild.sh

   1 #!/bin/sh
   2 PACKAGENAME=$1
   3 PAGE=/tmp/page.html
   4 HEADERS=/tmp/headers.html
   5 RES=/tmp/slackbuildres.txt
   6 curl https://slackbuilds.org/result/?search=$PACKAGENAME -D $HEADERS > $PAGE 2>/dev/null
   7 LOCATION=$(cat $HEADERS | grep -i Location | sed  's/Location: //g' )
   8 echo "Location: $LOCATION"
   9 POS=$(echo $LOCATION | grep -b -o "?search" | cut -d: -f1 )
  10 LOCATION_FIXED=$( echo $LOCATION | cut -c1-"$POS" | sed  "s/$PACKAGENAME\///g"   )
  11 INFO_FIXED=$( echo $LOCATION | cut -c1-"$POS"  )
  12 PACKAGE=$(echo $LOCATION_FIXED$1.tar.gz | sed 's/http:/https:/g' | sed  's/repository\//slackbuilds\//g')
  13 INFO=$(echo $INFO_FIXED$1.info | sed 's/http:/https:/g'  | sed  's/repository\//slackbuilds\//g'  )
  14 echo "Package: $PACKAGE"
  15 echo "Info: $INFO"
  16 cd /tmp
  17 rm -rf $PACKAGENAME/
  18 rm -f $PACKAGENAME.tar.gz*
  19 echo "Getting package"
  20 wget $PACKAGE
  21 tar xvzf $PACKAGENAME.tar.gz
  22 cd $PACKAGENAME
  23 echo "Getting info"
  24 wget $INFO
  25 DOWNLOAD=$(cat $PACKAGENAME.info | grep "DOWNLOAD=" | sed 's/"//g' | sed 's/DOWNLOAD=//g' )
  26 echo "Download source code"
  27 wget $DOWNLOAD
  28 echo "Run slackbuild"
  29 ./$PACKAGENAME.SlackBuild > $RES 2>&1
  30 SBO=$( cat $RES | grep "Slackware package " | grep " created" | sed 's/Slackware package //g' | sed 's/ created\.//g'  )
  31 echo "SlackBuild package: $SBO"

Prompt PS1 without current folder

   1 # prompt without current folder
   2 #PS1="$(date +'%Y-%m-%dT%H:%M:%S') $(whoami)@${HOST}\$ "
   3 #PS1="$(date +'%Y-%m-%dT%H:%M:%S') $(whoami)@$(hostname -a)\$ "
   4 PS1="\D{%Y-%m-%d}T\A \u@\H\r\n\$ "

Prompt PS1 with folder

   1 # PS1="$(date +'%Y-%m-%dT%H:%M:%S') $(whoami)@$(hostname -a)\n${PWD}\n\$ "
   2 PS1="\D{%Y-%m-%d}T\A \u@\H\n\w\r\n\$ "

PowerShell commands

Get-ChildItem c:\windows\system32\*.txt -Recurse | Select-String -Pattern "Microsoft" -CaseSensitive
sls "searchstring" file

Search using GNU grep

   1 #!/bin/sh
   2 echo "Searching $1 in $(pwd)"
   3 grep -H -n "$1" * -R

-  ⇤ ← Revision 1 as of 2014-01-24 12:46:36 → 
  Size: 143
  Editor: bl13-10-133
  Comment:
+   ← Revision 46 as of 2018-09-12 22:03:39 → ⇥
  Size: 8775
  Editor: localhost
  Comment:
-Deletions are marked like this.
+Additions are marked like this.
 Line 1:
-Line 3:
+Line 2:
-{{{
#
cat datax.log |  awk '/IMEI/{print $3}' | sort | uniq | wc -l
}}}
+{{{#!highlight bash
cat datax.log |  awk -F ' ' '/IMEI/{print $3}' | sort | uniq | wc -l
}}}

= Filter unique values from third column, separated by :  =
{{{#!highlight bash
cat datax.log |  awk -F ':' '/IMEI/{print $3}' | sort | uniq | wc -l
}}}


= Search for text in files recursively  =
{{{#!highlight bash
cd startFolder
grep -H "textToSearch" * -R
}}}

= Copy files to other destination (Win32) =
{{{
@echo off
REM Mount drive
echo Mounting remote drive
net use z: \\192.168.1.1\backs password /USER:userx

REM copy current day files
echo Copying current day files
forfiles -p "I:" -s -m *FULL*.sqb -d +0 -c "cmd /c copy @path Z:\BACKUPS\@file"

echo Deleting files older than 7 days
REM delete files older than 7 days
forfiles -p "Z:\BACKUPS" -m *.sqb -d 7 -c "cmd /c del @path"
}}}

= Gzip files that where last modified in 30 or more days (Win32) =
{{{
forfiles /p "c:\logs" /d -30 /m *.log /c "cmd /c gzip @path "
}}}

= Run task every day at 01:00 (Win32) =
{{{
at 01:00 /every:M,T,W,Th,F,S,Su c:\task.bat
}}}

= Block responses to pings =
{{{#!highlight bash
#block responses to pings
iptables -A INPUT -p icmp -m icmp --icmp-type echo-request -j DROP
iptables -L
#restore responses to pings
iptables -F
iptables -L
}}}

= Zip SVN modified Java files =
{{{#!highlight bash
#!/bin/sh
# chmod 755 /usr/local/bin/modifiedJavaSvn.sh # on CygWin
svn status | awk '/.java/{ print( gensub( /\\/,"/","g",$2) ); }' | xargs -i zip modifiedJavaFiles.zip {}
unzip -v modifiedJavaFiles.zip
}}}

== Deploy EAR script ==
{{{#!highlight bash
#!/usr/bin/bash
# deploy EAR in remote JBoss . 
# Must have pub SSH keys (authorized_keys) deployed on remote server
# Can be used with cygwin
FILEX=sample.ear
DEST_DIR=/opt/jboss/server/default/deploy/
ECLIPSE_PROJECT=/home/userx/workspace/sample-ear/

deploy()
{
  SERVER=$1
  USER=$2
  echo "Deploying in $SERVER ..."

  cd $PROJECT
  scp target/$FILEX $USER@$SERVER:/tmp
  ssh $USER@$SERVER -C "mv /tmp/$FILEX $DEST_DIR/$FILEX"
  ssh $USER@$SERVER -C "ls -lah $DEST_DIR"
  ssh $USER@$SERVER -C "chmod 666  $DEST_DIR/$FILEX"
  ssh $USER@$SERVER -C "/sbin/service jboss abort"
  ssh $USER@$SERVER -C "sudo /sbin/service jboss start"
}

deploy "example-server" "userx"
}}}

== List JAR contents to file ==
 * cd ~/.m2/repository
 * find . -name "*.jar" | xargs -i jar -tf {} > /tmp/explodedJars.txt
 * find . -name "*.jar" | xargs -i sh -c "echo {} ; jar -tf {}" > /tmp/explodedJars.txt

== Size of each directory inside current directory ==
 * find . -maxdepth 1 -type d | xargs -i du {} -hs

== Backup folders in HOME ==
{{{#!highlight bash
#!/bin/bash
# cd ~
# ln -s scripts/backup.sh backup.sh
# sh backup.sh
DATE=`date '+%Y%m%d%H%M%S'`
FILENAME=~/backups/backup_$DATE.tar.gz
cd ~
mkdir -p ~/backups
rm $FILENAME
tar cvzf $FILENAME Documents GNUstep ThunderbirdMail Desktop .Skype scripts moin-1.9.7
# tar tvzf backup_20150108103242.tar.gz
}}}

== Get installed updates on Windows ==
{{{
wmic qfe list brief /format:texttablewsys > updates.txt
}}}

== Find files modified less than an hour ago ==
{{{
find . -mtime -1h -name "*.cfg"
}}}

== Lock file to allow only one instance of a script to run ==
{{{#!highlight bash
#!/bin/sh
LOCK_FILE=$(echo "/var/tmp/lockTest.lck")
PID=$$
LOCKED=0
COUNT=0
echo "Trying to get lock to $PID in $LOCK_FILE"

while [ $LOCKED -eq "0" ] && [ $COUNT -le "3" ]; do
    if ( set -o noclobber; echo "$PID" > "$LOCK_FILE" ) 2> /dev/null; then
        LOCKED=1
    else        
        sleep 1
        COUNT=$(($COUNT+1))
        echo "Trying to get lock to $PID in $LOCK_FILE, retry $COUNT"
    fi    
done

if [ $LOCKED -eq "0" ]; then
    echo "Unable to get lock to $PID with $COUNT retries"
    exit 1
else
    echo "Lock acquired to $PID"    
fi   
#####################################
sleep 10
#####################################
rm -f "$LOCK_FILE"
LOCKED=0
echo "Released lock to $PID"
exit 0
}}}

== awk , load file into array ==
 * https://magvar.wordpress.com/2011/05/18/awking-it-how-to-load-a-file-into-an-array-in-awk/

{{{#!highlight sh
#! /bin/bash
INPUTFILE="inputfile.csv"
DATAFILE="data.csv"
OUTFILE="output.csv"

awk 'BEGIN {
while (getline < "'"$INPUTFILE"'")
{
split($0,ft,",");
id=ft[1];
name=ft[2];

key=id;
data=name;
nameArr[key]=data;
}
close("'"$INPUTFILE"'");

while (getline < "'"$DATAFILE"'")
{
split($0,ft,",");
id=ft[1]; # Id is the first column
phone=ft[2]; # Phonenumber is the second
name=nameArr[id]; # Get the name from the nameArr, using "id" as key
print id","name","phone > "'"$OUTFILE"'"; # Print directly to the outputfile
}
}'

}}}

== Shell script sample ==
{{{#!highlight sh
#!/bin/sh
function replaceString {
  RES=$(echo $1 | sed "s/$2/$3/g")
  echo $RES
}

function add {
  echo $( echo "$1 + $2" | bc )    
}

function inc {
  echo $( expr $1 + 1 )  
}
####################################
echo $(add "1.5" "3.2" )

TEXT1="AAA BBB CCC"

for WORD in $TEXT1
do
  echo ${WORD}
done

WITHOUTSPACES=$(replaceString "AAA BBB XXX" " " "" )
echo $WITHOUTSPACES

if [ $WITHOUTSPACES == "AAABBBXXX" ]
then
  echo "Equal"
fi

if [ $WITHOUTSPACES != "AAA BBB XXX" ]
then
  echo "Diff"
fi

A=$(ls / | wc -l)
A=$(expr $A + 1)
if [ $A -eq 22 ]
then
  echo $A
fi

B="222"
B=$(expr $B)
if [ $B -eq 222 ]
then
  echo $B
fi

COUNT="0"
while [ $COUNT -lt 10 ]
do
  COUNT=$( inc $COUNT )
done

$(test 0 -ge 0)
RES=$(echo $?)
echo $RES
}}}

== Change TIME_WAIT (time wait) tcp/ip ==
Linux
{{{
echo 1 > /proc/sys/net/ipv4/tcp_fin_timeout
echo 1 > /proc/sys/net/ipv4/tcp_tw_recycle
echo 1 > /proc/sys/net/ipv4/tcp_tw_reuse
}}}

Freebsd
 * sysctl net.inet.tcp.msl=1500

== Show root folders occupied space ==
{{{#!highlight bash
cd /tmp
ls --color=never | xargs -i du {} -hs
find . -type d -maxdepth 1 | xargs -i du {} -hs

#~/.bashrc
alias folderUsedSpace='find . -type d -maxdepth 1 | xargs -i du {} -hs'
}}}

== Add swapfile 512MB ==
{{{#!highlight sh
df -h
dd if=/dev/zero of=/swapfile1 bs=1024 count=524288
chown root:root /swapfile1 
chmod 0600 /swapfile1 
mkswap /swapfile1 
swapon /swapfile1
free -m
vi /etc/fstab
# /swapfile1 none swap sw 0 0
free -m
htop
}}}

== Alias current date ==
{{{#!highlight sh
alias currdate="date '+%Y%m%dT%H%M%S'"
}}}

== Alias history without line number ==
{{{#!highlight sh
alias hist='history | cut -c 8-'
}}}

== Calculator using bc ==
{{{#!highlight sh
#!/bin/sh
echo "scale=3;$1" | bc 
}}}

== Get slackbuild ==
'''getslackbuild.sh'''
{{{#!highlight bash
#!/bin/sh
PACKAGENAME=$1
PAGE=/tmp/page.html
HEADERS=/tmp/headers.html
RES=/tmp/slackbuildres.txt
curl https://slackbuilds.org/result/?search=$PACKAGENAME -D $HEADERS > $PAGE 2>/dev/null
LOCATION=$(cat $HEADERS | grep -i Location | sed  's/Location: //g' )
echo "Location: $LOCATION"
POS=$(echo $LOCATION | grep -b -o "?search" | cut -d: -f1 )
LOCATION_FIXED=$( echo $LOCATION | cut -c1-"$POS" | sed  "s/$PACKAGENAME\///g"   )
INFO_FIXED=$( echo $LOCATION | cut -c1-"$POS"  )
PACKAGE=$(echo $LOCATION_FIXED$1.tar.gz | sed 's/http:/https:/g' | sed  's/repository\//slackbuilds\//g')
INFO=$(echo $INFO_FIXED$1.info | sed 's/http:/https:/g'  | sed  's/repository\//slackbuilds\//g'  )
echo "Package: $PACKAGE"
echo "Info: $INFO"
cd /tmp
rm -rf $PACKAGENAME/
rm -f $PACKAGENAME.tar.gz*
echo "Getting package"
wget $PACKAGE
tar xvzf $PACKAGENAME.tar.gz
cd $PACKAGENAME
echo "Getting info"
wget $INFO
DOWNLOAD=$(cat $PACKAGENAME.info | grep "DOWNLOAD=" | sed 's/"//g' | sed 's/DOWNLOAD=//g' )
echo "Download source code"
wget $DOWNLOAD
echo "Run slackbuild"
./$PACKAGENAME.SlackBuild > $RES 2>&1
SBO=$( cat $RES | grep "Slackware package " | grep " created" | sed 's/Slackware package //g' | sed 's/ created\.//g'  )
echo "SlackBuild package: $SBO"
}}}

== Prompt PS1 without current folder ==
{{{#!highlight bash
# prompt without current folder
#PS1="$(date +'%Y-%m-%dT%H:%M:%S') $(whoami)@${HOST}\$ "
#PS1="$(date +'%Y-%m-%dT%H:%M:%S') $(whoami)@$(hostname -a)\$ "
PS1="\D{%Y-%m-%d}T\A \u@\H\r\n\$ "
}}}

== Prompt PS1 with folder ==
{{{#!highlight bash
# PS1="$(date +'%Y-%m-%dT%H:%M:%S') $(whoami)@$(hostname -a)\n${PWD}\n\$ "
PS1="\D{%Y-%m-%d}T\A \u@\H\n\w\r\n\$ "
}}}

== PowerShell commands ==
{{{
Get-ChildItem c:\windows\system32\*.txt -Recurse | Select-String -Pattern "Microsoft" -CaseSensitive
sls "searchstring" file 
}}}


== Search using GNU grep ==
{{{#!highlight bash
#!/bin/sh
echo "Searching $1 in $(pwd)"
grep -H -n "$1" * -R
}}}