This commit is contained in:
Frank Bischof 2023-06-08 09:28:26 +02:00
parent bc62d99054
commit 8374b03834
3 changed files with 114 additions and 73 deletions

View File

@ -1,8 +1,9 @@
# 7zip_to_netapp_v2.ps1
# 7zip_to_netapp.ps1
# Author: F. Bischof (info@meer-web.nl)
# Version 2.6.0
# Version 3.0.1
# Set global vars
$MAX_FOLDER_SIZE = "500GB"
$TIMESTAMP = Get-Date -Format "yyyyMMddHHmm"
$TARGET_FILENAME = "${TIMESTAMP}-archive.7z"
$CSVFILE = "C:\Scripts\movethis.csv"
@ -66,81 +67,106 @@ foreach ($SOURCE_ENTRY in $CSVFILE) {
WRITELOG "TARGET variable is empty! Please check the CSV file. Exiting..."
exit
}
WRITELOG "Archiving: $COUNTER - $SOURCE"
Write-Output "Archiving $SOURCE"
# Create stagefolders which give the state of the source folder
Write-Output "Creating stage folder"
$TEMPFOLDER_RUNNING = ${SOURCE} + "_being_archived_by_solvinity"
$TEMPFOLDER_DONE = ${SOURCE} + "_is_archived_by_solvinity"
$TEMPFOLDER_FAILED = ${SOURCE} + "_archived_failed_by_solvinity"
WRITELOG "Creating stage folder $TEMPFOLDER_RUNNING"
if (!(test-path $TEMPFOLDER_RUNNING)){
New-Item -ItemType Directory $TEMPFOLDER_RUNNING
WRITELOG "Calculating folder size of $SOURCE"
Write-Output "Calculating folder size of $SOURCE"
$SOURCE_SIZE = (Get-ChildItem ${SOURCE} -Recurse| Measure-Object -Property Length -sum).Sum
$SOURCE_SIZE_FACTOR = [int]($SOURCE_SIZE / $MAX_FOLDER_SIZE)
if ($SOURCE_SIZE_FACTOR -ge 1) {
WRITELOG "$SOURCE is too big to compress using 7zip, switching to robocopy."
WRITELOG "Mirroring $SOURCE using robocopy"
robocopy /MIR ${SOURCE} ${TARGET}
$TARGET_SIZE = (Get-ChildItem ${TARGET} -Recurse| Measure-Object -Property Length -sum).Sum
if (${SOURCE_SIZE} -ne ${TARGET_SIZE}) {
WRITELOG "Source and destination sizes do not match!"; exit
} else {
WRITELOG "Source and destination sizes match"
}
WRITELOG "empty $SOURCE"
mkdir empty
robocopy /MIR empty ${SOURCE}
WRITELOG "Rename $SOURCE to "
$TEMPFOLDER_DONE = ${SOURCE} + "_is_archived_by_solvinity"
Rename-Item ${SOURCE} ${TEMPFOLDER_DONE}
WRITELOG "Temp quit on ${SOURCE}"
exit
} else {
WRITELOG "$TEMPFOLDER_RUNNING already exists! Please check or delete this folder! Exiting..."
exit
}
# Adjust ACL so that users cannot access the archived folder anymore
WRITELOG "Locking ACL on $SOURCE"
$ACL = get-acl -Path $SOURCE
$ACL.SetAccessRuleProtection($True, $False)
$LOCALADMIN_FULLCONTROL = New-Object system.security.accesscontrol.filesystemaccessrule("builtin\Administrators", "FullControl", "ContainerInherit,ObjectInherit", "none", "Allow")
$ACL.SetAccessRule($LOCALADMIN_FULLCONTROL)
Set-Acl -Path $SOURCE -AclObject $ACL
# Close down open files
WRITELOG "Closing OpenFiles on $SOURCE"
$OPENFILES = Get-SmbOpenFile | Where-Object -Property path -like *$SOURCE*
$OPENFILES_COUNT = $OPENFILES.count
WRITELOG "$OPENFILES_COUNT files closed"
$OPENFILES | Close-SmbOpenFile -Force
# Start archiving
WRITELOG "7ZIP $SOURCE"
7z a -mx3 -t7z -r "${TARGET}\${TARGET_FILENAME}" "$SOURCE\*"
# Validate number of files in archive and DFS
WRITELOG "Compare the number of files in DFS and archive"
## 7zip
WRITELOG "Counting files in 7zip archive"
$7ZIP_FILECOUNT = '0'
7z l $TARGET\$TARGET_FILENAME | Select-Object -Last 1 |
Select-String '([0-9]+) files(?:, ([0-9]+) folders)?' |
ForEach-Object {
$7ZIP_FILECOUNT = [Int] $_.Matches[0].Groups[1].Value
if ($7ZIP_FILECOUNT -eq $null) {
# Ignore counter if null
# Create stagefolders which give the state of the source folder
Write-Output "Creating stage folder"
$TEMPFOLDER_RUNNING = ${SOURCE} + "_being_archived"
$TEMPFOLDER_DONE = ${SOURCE} + "_is_archived"
$TEMPFOLDER_FAILED = ${SOURCE} + "_archived_failed"
WRITELOG "Creating stage folder $TEMPFOLDER_RUNNING"
if (!(test-path $TEMPFOLDER_RUNNING)){
New-Item -ItemType Directory $TEMPFOLDER_RUNNING
} else {
WRITELOG "$TEMPFOLDER_RUNNING already exists! Please check or delete this folder! Exiting..."
exit
}
}
# Adjust ACL so that users cannot access the archived folder anymore
WRITELOG "Locking ACL on $SOURCE"
$ACL = get-acl -Path $SOURCE
$ACL.SetAccessRuleProtection($True, $False)
$LOCALADMIN_FULLCONTROL = New-Object system.security.accesscontrol.filesystemaccessrule("builtin\Administrators", "FullControl", "ContainerInherit,ObjectInherit", "none", "Allow")
$ACL.SetAccessRule($LOCALADMIN_FULLCONTROL)
Set-Acl -Path $SOURCE -AclObject $ACL
## DFS
WRITELOG "Counting files in source path"
$dirandfilelist = Get-ChildItem -Recurse -force $SOURCE
$DFS_FILECOUNT = ($dirandfilelist | Where-Object { ! $_.PSIsContainer }).count
# Close down open files
WRITELOG "Closing OpenFiles on $SOURCE"
$OPENFILES = Get-SmbOpenFile | Where-Object -Property path -like *$SOURCE*
$OPENFILES_COUNT = $OPENFILES.count
WRITELOG "$OPENFILES_COUNT files closed"
$OPENFILES | Close-SmbOpenFile -Force
# Start archiving
WRITELOG "7ZIP $SOURCE"
7z a -mx3 -t7z -r "${TARGET}\${TARGET_FILENAME}" "$SOURCE\*"
# Validate number of files in archive and DFS
WRITELOG "Compare the number of files in DFS and archive"
## 7zip
WRITELOG "Counting files in 7zip archive"
$7ZIP_FILECOUNT = '0'
7z l $TARGET\$TARGET_FILENAME | Select-Object -Last 1 |
Select-String '([0-9]+) files(?:, ([0-9]+) folders)?' |
ForEach-Object {
$7ZIP_FILECOUNT = [Int] $_.Matches[0].Groups[1].Value
if ($7ZIP_FILECOUNT -eq $null) {
# Ignore counter if null
}
WRITELOG "${SOURCE}: ${7ZIP_FILECOUNT} / ${DFS_FILECOUNT}"
WRITELOG "Comparing source and target files"
if ($7ZIP_FILECOUNT -eq $DFS_FILECOUNT) {
# Compare OK
WRITELOG "Number of files matching! Cleaning source folder"
Write-Output "Cleaning up $SOURCE"
robocopy /MIR $EMPTY_FOLDER $SOURCE
remove-item -Force -Recurse $SOURCE
WRITELOG "Source has been archived to $TARGET!"
if (!(test-path $TEMPFOLDER_DONE)){
WRITELOG "Renaming stage folder to $TEMPFOLDER_DONE"
Rename-Item -Path $TEMPFOLDER_RUNNING -NewName $TEMPFOLDER_DONE
}
} else {
# Compare mismatch
WRITELOG "CRITICAL - Compare failed, the number of files are not matching. Counted: $7ZIP_FILECOUNT / $DFS_FILECOUNT"
if (!(test-path $TEMPFOLDER_FAILED)){
WRITELOG "CRITICAL - Renaming stage folder to $TEMPFOLDER_FAILED"
Rename-Item -Path $TEMPFOLDER_RUNNING -NewName $TEMPFOLDER_FAILED
## DFS
WRITELOG "Counting files in source path"
$dirandfilelist = Get-ChildItem -Recurse -force $SOURCE
$DFS_FILECOUNT = ($dirandfilelist | Where-Object { ! $_.PSIsContainer }).count
WRITELOG "${SOURCE}: ${7ZIP_FILECOUNT} / ${DFS_FILECOUNT}"
WRITELOG "Comparing source and target files"
if ($7ZIP_FILECOUNT -eq $DFS_FILECOUNT) {
# Compare OK
WRITELOG "Number of files matching! Cleaning source folder"
Write-Output "Cleaning up $SOURCE"
robocopy /MIR $EMPTY_FOLDER $SOURCE
remove-item -Force -Recurse $SOURCE
WRITELOG "Source has been archived to $TARGET!"
if (!(test-path $TEMPFOLDER_DONE)){
WRITELOG "Renaming stage folder to $TEMPFOLDER_DONE"
Rename-Item -Path $TEMPFOLDER_RUNNING -NewName $TEMPFOLDER_DONE
}
} else {
# Compare mismatch
WRITELOG "CRITICAL - Compare failed, the number of files are not matching. Counted: $7ZIP_FILECOUNT / $DFS_FILECOUNT"
if (!(test-path $TEMPFOLDER_FAILED)){
WRITELOG "CRITICAL - Renaming stage folder to $TEMPFOLDER_FAILED"
Rename-Item -Path $TEMPFOLDER_RUNNING -NewName $TEMPFOLDER_FAILED
}
}
}
}

View File

@ -1,3 +1,6 @@
## 3.0.1
- Added a $MAX_FOLDER_SIZE where if reached it switches to robocopy instead.
## 2.6.0
- Bugfixed CSVFile modification

View File

@ -3,8 +3,20 @@
This script is used for archiving major data.
## Requirements
The script only needs to know where to find the CSV file which needs to be loaded.
The script needs to know where to find the CSV file which needs to be loaded.
This CSV path can be set in the global vars part of the script.
- 7zip needs to be installed
- robocopy needs to be installed
## Configuration
$MAX_FOLDER_SIZE = Max folder size to zip, otherwise switch to robocopy.
$TIMESTAMP = Timeformat to use for the logfile
$TARGET_FILENAME = Target archive name
$CSVFILE = Source CSV file
$LOGFILE = Log file
$TEMPFOLDER_RUNNING = Source folder name for when archive is running
$TEMPFOLDER_DONE = Source folder name for when archive is done
$TEMPFOLDER_FAILED = Source folder name for when archive is failed
### CSV template
Create a CSV file containing the following setup:
@ -22,6 +34,6 @@ C:\temp\source_folder,D:\archive\target_folder
```
## Run script
> .\7zip_to_netapp_v2.ps1
> .\7zip_to_netapp.ps1
Just sit back and relax...