I have a script to move XML files from Server 1 to Server 2 and then, based on file name create Update SQL query, which is stored in a file. It took 5 hours to copy 55000 files from Server 1 to Server 2, I have around 2 million files.
Its there any faster way to perform this operation quickly. Shell script is as below
#!/usr/bin/env bash
export ORACLE_HOME=/u01/app/oracle/product/19.3.0/db
export LD_LIBRARY_PATH=/u01/app/oracle/product/19.3.0/db/lib
export PATH=$PATH:$ORACLE_HOME/bin
username="eip"
origin="/flexsync"
backup="/flexsync/processed/processedAssets"
destination="/home/eip2/data/synchronization/flexsync/in"
ip="abc.def.xyz"
dt=$(date +%Y%m%d%H%M%S)
echo "Oracle Home: "$ORACLE_HOME
echo "Oracle Library: "$LD_LIBRARY_PATH
echo "Oracle Path: "$PATH
echo "Source Directory: "$origin
echo "Source Backup directory: "$backup
echo "Target Directory: "$destination
echo "Target System: "$ip
echo "Uploading files to remote server..."
for file in "$origin"/*.xml
do
[ -f "$file" ] || continue
name=${file%.*}
name=${name##*/}
save="${name}_$dt.xml"
fname="${name}.xml"
#echo "Copying file from Server A to Server B"
scp -C -- "$file" "$username@$ip:$destination"
#echo "Moving file from Server A $origin to $backup and appending timestamp before file type"
mv -f -- "$file" "$backup/$save"
#echo "Update SQL "
cat >> /flexsync/update.sql <<EOF
update TRC_ISTOILOVA.X_EISERVER_ASSETS_CURR_10_24_2023 set FLEXSYNC='YES' where FILE_NAME='$fname';
update TRC_ISTOILOVA.X_EISERVER_ASSETS_CURR_10_24_2023 set FLEXSYNC_DT=SYSTIMESTAMP where FILE_NAME='$fname';
update TRC_ISTOILOVA.X_EISERVER_ASSETS_CURR_10_24_2023 set LAST_UPDATE_TIME=SYSTIMESTAMP where FILE_NAME='$fname';
commit;
EOF
#echo "File Updated"
done
echo " Updating table"
sql_file="/flexsync/update.sql"
/u01/app/oracle/product/19.3.0/db/bin/sqlplus abc/xyz@def:2222/def_svc @"$sql_file"
exit;
echo "Table Updated"
echo 'File upload to remote server completed! ;)'