From 8374b038340386a2bede53e982679891a52f279f Mon Sep 17 00:00:00 2001 From: Frank Bischof Date: Thu, 8 Jun 2023 09:28:26 +0200 Subject: [PATCH] v3.0.1 --- 7zip_archiver_v2.ps1 | 166 +++++++++++++++++++++++++------------------ CHANGELOG | 3 + README.md | 18 ++++- 3 files changed, 114 insertions(+), 73 deletions(-) diff --git a/7zip_archiver_v2.ps1 b/7zip_archiver_v2.ps1 index f8f84e5..22a9215 100644 --- a/7zip_archiver_v2.ps1 +++ b/7zip_archiver_v2.ps1 @@ -1,8 +1,9 @@ -# 7zip_to_netapp_v2.ps1 +# 7zip_to_netapp.ps1 # Author: F. Bischof (info@meer-web.nl) -# Version 2.6.0 +# Version 3.0.1 # Set global vars +$MAX_FOLDER_SIZE = "500GB" $TIMESTAMP = Get-Date -Format "yyyyMMddHHmm" $TARGET_FILENAME = "${TIMESTAMP}-archive.7z" $CSVFILE = "C:\Scripts\movethis.csv" @@ -66,81 +67,106 @@ foreach ($SOURCE_ENTRY in $CSVFILE) { WRITELOG "TARGET variable is empty! Please check the CSV file. Exiting..." exit } + WRITELOG "Archiving: $COUNTER - $SOURCE" Write-Output "Archiving $SOURCE" - # Create stagefolders which give the state of the source folder - Write-Output "Creating stage folder" - $TEMPFOLDER_RUNNING = ${SOURCE} + "_being_archived_by_solvinity" - $TEMPFOLDER_DONE = ${SOURCE} + "_is_archived_by_solvinity" - $TEMPFOLDER_FAILED = ${SOURCE} + "_archived_failed_by_solvinity" - WRITELOG "Creating stage folder $TEMPFOLDER_RUNNING" - if (!(test-path $TEMPFOLDER_RUNNING)){ - New-Item -ItemType Directory $TEMPFOLDER_RUNNING + WRITELOG "Calculating folder size of $SOURCE" + Write-Output "Calculating folder size of $SOURCE" + $SOURCE_SIZE = (Get-ChildItem ${SOURCE} -Recurse| Measure-Object -Property Length -sum).Sum + $SOURCE_SIZE_FACTOR = [int]($SOURCE_SIZE / $MAX_FOLDER_SIZE) + if ($SOURCE_SIZE_FACTOR -ge 1) { + WRITELOG "$SOURCE is too big to compress using 7zip, switching to robocopy." + WRITELOG "Mirroring $SOURCE using robocopy" + robocopy /MIR ${SOURCE} ${TARGET} + $TARGET_SIZE = (Get-ChildItem ${TARGET} -Recurse| Measure-Object -Property Length -sum).Sum + if (${SOURCE_SIZE} -ne ${TARGET_SIZE}) { + WRITELOG "Source and destination sizes do not match!"; exit + } else { + WRITELOG "Source and destination sizes match" + } + WRITELOG "empty $SOURCE" + mkdir empty + robocopy /MIR empty ${SOURCE} + WRITELOG "Rename $SOURCE to " + $TEMPFOLDER_DONE = ${SOURCE} + "_is_archived_by_solvinity" + Rename-Item ${SOURCE} ${TEMPFOLDER_DONE} + WRITELOG "Temp quit on ${SOURCE}" + exit } else { - WRITELOG "$TEMPFOLDER_RUNNING already exists! Please check or delete this folder! Exiting..." - exit - } - - # Adjust ACL so that users cannot access the archived folder anymore - WRITELOG "Locking ACL on $SOURCE" - $ACL = get-acl -Path $SOURCE - $ACL.SetAccessRuleProtection($True, $False) - $LOCALADMIN_FULLCONTROL = New-Object system.security.accesscontrol.filesystemaccessrule("builtin\Administrators", "FullControl", "ContainerInherit,ObjectInherit", "none", "Allow") - $ACL.SetAccessRule($LOCALADMIN_FULLCONTROL) - Set-Acl -Path $SOURCE -AclObject $ACL - - # Close down open files - WRITELOG "Closing OpenFiles on $SOURCE" - $OPENFILES = Get-SmbOpenFile | Where-Object -Property path -like *$SOURCE* - $OPENFILES_COUNT = $OPENFILES.count - WRITELOG "$OPENFILES_COUNT files closed" - $OPENFILES | Close-SmbOpenFile -Force - - # Start archiving - WRITELOG "7ZIP $SOURCE" - 7z a -mx3 -t7z -r "${TARGET}\${TARGET_FILENAME}" "$SOURCE\*" - - # Validate number of files in archive and DFS - WRITELOG "Compare the number of files in DFS and archive" - ## 7zip - WRITELOG "Counting files in 7zip archive" - $7ZIP_FILECOUNT = '0' - 7z l $TARGET\$TARGET_FILENAME | Select-Object -Last 1 | - Select-String '([0-9]+) files(?:, ([0-9]+) folders)?' | - - ForEach-Object { - $7ZIP_FILECOUNT = [Int] $_.Matches[0].Groups[1].Value - if ($7ZIP_FILECOUNT -eq $null) { - # Ignore counter if null + # Create stagefolders which give the state of the source folder + Write-Output "Creating stage folder" + $TEMPFOLDER_RUNNING = ${SOURCE} + "_being_archived" + $TEMPFOLDER_DONE = ${SOURCE} + "_is_archived" + $TEMPFOLDER_FAILED = ${SOURCE} + "_archived_failed" + WRITELOG "Creating stage folder $TEMPFOLDER_RUNNING" + if (!(test-path $TEMPFOLDER_RUNNING)){ + New-Item -ItemType Directory $TEMPFOLDER_RUNNING + } else { + WRITELOG "$TEMPFOLDER_RUNNING already exists! Please check or delete this folder! Exiting..." + exit } - } - - ## DFS - WRITELOG "Counting files in source path" - $dirandfilelist = Get-ChildItem -Recurse -force $SOURCE - $DFS_FILECOUNT = ($dirandfilelist | Where-Object { ! $_.PSIsContainer }).count - - WRITELOG "${SOURCE}: ${7ZIP_FILECOUNT} / ${DFS_FILECOUNT}" - WRITELOG "Comparing source and target files" - if ($7ZIP_FILECOUNT -eq $DFS_FILECOUNT) { - # Compare OK - WRITELOG "Number of files matching! Cleaning source folder" - Write-Output "Cleaning up $SOURCE" - robocopy /MIR $EMPTY_FOLDER $SOURCE - remove-item -Force -Recurse $SOURCE - WRITELOG "Source has been archived to $TARGET!" - if (!(test-path $TEMPFOLDER_DONE)){ - WRITELOG "Renaming stage folder to $TEMPFOLDER_DONE" - Rename-Item -Path $TEMPFOLDER_RUNNING -NewName $TEMPFOLDER_DONE + # Adjust ACL so that users cannot access the archived folder anymore + WRITELOG "Locking ACL on $SOURCE" + $ACL = get-acl -Path $SOURCE + $ACL.SetAccessRuleProtection($True, $False) + $LOCALADMIN_FULLCONTROL = New-Object system.security.accesscontrol.filesystemaccessrule("builtin\Administrators", "FullControl", "ContainerInherit,ObjectInherit", "none", "Allow") + $ACL.SetAccessRule($LOCALADMIN_FULLCONTROL) + Set-Acl -Path $SOURCE -AclObject $ACL + + # Close down open files + WRITELOG "Closing OpenFiles on $SOURCE" + $OPENFILES = Get-SmbOpenFile | Where-Object -Property path -like *$SOURCE* + $OPENFILES_COUNT = $OPENFILES.count + WRITELOG "$OPENFILES_COUNT files closed" + $OPENFILES | Close-SmbOpenFile -Force + + # Start archiving + WRITELOG "7ZIP $SOURCE" + 7z a -mx3 -t7z -r "${TARGET}\${TARGET_FILENAME}" "$SOURCE\*" + + # Validate number of files in archive and DFS + WRITELOG "Compare the number of files in DFS and archive" + ## 7zip + WRITELOG "Counting files in 7zip archive" + $7ZIP_FILECOUNT = '0' + 7z l $TARGET\$TARGET_FILENAME | Select-Object -Last 1 | + Select-String '([0-9]+) files(?:, ([0-9]+) folders)?' | + + ForEach-Object { + $7ZIP_FILECOUNT = [Int] $_.Matches[0].Groups[1].Value + if ($7ZIP_FILECOUNT -eq $null) { + # Ignore counter if null + } + } - } else { - # Compare mismatch - WRITELOG "CRITICAL - Compare failed, the number of files are not matching. Counted: $7ZIP_FILECOUNT / $DFS_FILECOUNT" - if (!(test-path $TEMPFOLDER_FAILED)){ - WRITELOG "CRITICAL - Renaming stage folder to $TEMPFOLDER_FAILED" - Rename-Item -Path $TEMPFOLDER_RUNNING -NewName $TEMPFOLDER_FAILED + + ## DFS + WRITELOG "Counting files in source path" + $dirandfilelist = Get-ChildItem -Recurse -force $SOURCE + $DFS_FILECOUNT = ($dirandfilelist | Where-Object { ! $_.PSIsContainer }).count + + WRITELOG "${SOURCE}: ${7ZIP_FILECOUNT} / ${DFS_FILECOUNT}" + WRITELOG "Comparing source and target files" + if ($7ZIP_FILECOUNT -eq $DFS_FILECOUNT) { + # Compare OK + WRITELOG "Number of files matching! Cleaning source folder" + Write-Output "Cleaning up $SOURCE" + robocopy /MIR $EMPTY_FOLDER $SOURCE + remove-item -Force -Recurse $SOURCE + WRITELOG "Source has been archived to $TARGET!" + if (!(test-path $TEMPFOLDER_DONE)){ + WRITELOG "Renaming stage folder to $TEMPFOLDER_DONE" + Rename-Item -Path $TEMPFOLDER_RUNNING -NewName $TEMPFOLDER_DONE + } + } else { + # Compare mismatch + WRITELOG "CRITICAL - Compare failed, the number of files are not matching. Counted: $7ZIP_FILECOUNT / $DFS_FILECOUNT" + if (!(test-path $TEMPFOLDER_FAILED)){ + WRITELOG "CRITICAL - Renaming stage folder to $TEMPFOLDER_FAILED" + Rename-Item -Path $TEMPFOLDER_RUNNING -NewName $TEMPFOLDER_FAILED + } } } } diff --git a/CHANGELOG b/CHANGELOG index f66d404..b2fd75c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,6 @@ +## 3.0.1 +- Added a $MAX_FOLDER_SIZE where if reached it switches to robocopy instead. + ## 2.6.0 - Bugfixed CSVFile modification diff --git a/README.md b/README.md index 3bce196..e4c88c6 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,20 @@ This script is used for archiving major data. ## Requirements -The script only needs to know where to find the CSV file which needs to be loaded. +The script needs to know where to find the CSV file which needs to be loaded. This CSV path can be set in the global vars part of the script. +- 7zip needs to be installed +- robocopy needs to be installed + +## Configuration +$MAX_FOLDER_SIZE = Max folder size to zip, otherwise switch to robocopy. +$TIMESTAMP = Timeformat to use for the logfile +$TARGET_FILENAME = Target archive name +$CSVFILE = Source CSV file +$LOGFILE = Log file +$TEMPFOLDER_RUNNING = Source folder name for when archive is running +$TEMPFOLDER_DONE = Source folder name for when archive is done +$TEMPFOLDER_FAILED = Source folder name for when archive is failed ### CSV template Create a CSV file containing the following setup: @@ -22,6 +34,6 @@ C:\temp\source_folder,D:\archive\target_folder ``` ## Run script -> .\7zip_to_netapp_v2.ps1 +> .\7zip_to_netapp.ps1 -Just sit back and relax... +Just sit back and relax... \ No newline at end of file