From 021c6ecc8b776695e41c0512411d74978d1fa299 Mon Sep 17 00:00:00 2001 From: IRBorisov <8611739+IRBorisov@users.noreply.github.com> Date: Fri, 1 Sep 2023 20:45:15 +0300 Subject: [PATCH] Refactor scripts and add Windows backup scripts --- .vscode/launch.json | 10 +-- rsconcept/PopulateDevData.ps1 | 13 ---- rsconcept/RunCoverage.ps1 | 12 ---- rsconcept/RunLint.ps1 | 8 --- rsconcept/RunTests.ps1 | 12 ---- rsconcept/backend/Dockerfile | 6 +- rsconcept/backend/LocalEnvSetup.ps1 | 24 ------- .../frontend/src/context/ThemeContext.tsx | 8 +-- .../src/pages/RSFormPage/EditorTermGraph.tsx | 2 +- scripts/dev/LocalDevSetup.ps1 | 55 ++++++++++++++ scripts/dev/PopulateDevData.ps1 | 27 +++++++ scripts/dev/RunCoverage.ps1 | 23 ++++++ scripts/dev/RunLint.ps1 | 17 +++++ {rsconcept => scripts/dev}/RunServer.ps1 | 23 +++--- scripts/dev/RunTests.ps1 | 25 +++++++ scripts/prod/CreateBackup.ps1 | 71 +++++++++++++++++++ scripts/prod/LoadPostgreBackup.ps1 | 27 +++++++ 17 files changed, 272 insertions(+), 91 deletions(-) delete mode 100644 rsconcept/PopulateDevData.ps1 delete mode 100644 rsconcept/RunCoverage.ps1 delete mode 100644 rsconcept/RunLint.ps1 delete mode 100644 rsconcept/RunTests.ps1 delete mode 100644 rsconcept/backend/LocalEnvSetup.ps1 create mode 100644 scripts/dev/LocalDevSetup.ps1 create mode 100644 scripts/dev/PopulateDevData.ps1 create mode 100644 scripts/dev/RunCoverage.ps1 create mode 100644 scripts/dev/RunLint.ps1 rename {rsconcept => scripts/dev}/RunServer.ps1 (78%) create mode 100644 scripts/dev/RunTests.ps1 create mode 100644 scripts/prod/CreateBackup.ps1 create mode 100644 scripts/prod/LoadPostgreBackup.ps1 diff --git a/.vscode/launch.json b/.vscode/launch.json index f677f73c..5107751f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,28 +8,28 @@ "name": "Run", "type": "PowerShell", "request": "launch", - "script": "${workspaceFolder}/rsconcept/RunServer.ps1", + "script": "${workspaceFolder}/scripts/dev/RunServer.ps1", "args": [] }, { "name": "Lint", "type": "PowerShell", "request": "launch", - "script": "${workspaceFolder}/rsconcept/RunLint.ps1", + "script": "${workspaceFolder}/scripts/dev/RunLint.ps1", "args": [] }, { "name": "Test", "type": "PowerShell", "request": "launch", - "script": "${workspaceFolder}/rsconcept/RunTests.ps1", + "script": "${workspaceFolder}/scripts/dev/RunTests.ps1", "args": [] }, { "name": "BE-Coverage", "type": "PowerShell", "request": "launch", - "script": "${workspaceFolder}/rsconcept/RunCoverage.ps1", + "script": "${workspaceFolder}/scripts/dev/RunCoverage.ps1", "args": [] }, { @@ -57,7 +57,7 @@ "name": "Restart", "type": "PowerShell", "request": "launch", - "script": "${workspaceFolder}/rsconcept/RunServer.ps1", + "script": "${workspaceFolder}/scripts/dev/RunServer.ps1", "args": ["-freshStart"] }, { diff --git a/rsconcept/PopulateDevData.ps1 b/rsconcept/PopulateDevData.ps1 deleted file mode 100644 index d7711d6b..00000000 --- a/rsconcept/PopulateDevData.ps1 +++ /dev/null @@ -1,13 +0,0 @@ -# Initialize database ! -# FOR DEVELOPEMENT BUILDS ONLY! -$container= Read-Host -Prompt "Enter backend container name: " - -docker exec -it $container python manage.py loaddata fixtures/InitialData.json - -docker exec ` - -e DJANGO_SUPERUSER_USERNAME=admin ` - -e DJANGO_SUPERUSER_PASSWORD=1234 ` - -e DJANGO_SUPERUSER_EMAIL=admin@admin.com ` - -it $container python manage.py createsuperuser --noinput - -pause \ No newline at end of file diff --git a/rsconcept/RunCoverage.ps1 b/rsconcept/RunCoverage.ps1 deleted file mode 100644 index 15dc7461..00000000 --- a/rsconcept/RunCoverage.ps1 +++ /dev/null @@ -1,12 +0,0 @@ - # Run coverage analysis -Set-Location $PSScriptRoot\backend - -$coverageExec = "$PSScriptRoot\backend\venv\Scripts\coverage.exe" -$djangoSrc = "$PSScriptRoot\backend\manage.py" -$exclude = '*/venv/*,*/tests/*,*/migrations/*,*__init__.py,manage.py,apps.py,urls.py,settings.py' - -& $coverageExec run --omit=$exclude $djangoSrc test -& $coverageExec report -& $coverageExec html - -Start-Process "file:///$PSScriptRoot\backend\htmlcov\index.html" \ No newline at end of file diff --git a/rsconcept/RunLint.ps1 b/rsconcept/RunLint.ps1 deleted file mode 100644 index c47debec..00000000 --- a/rsconcept/RunLint.ps1 +++ /dev/null @@ -1,8 +0,0 @@ - # Run coverage analysis -Set-Location $PSScriptRoot\backend - -$pylint = "$PSScriptRoot\backend\venv\Scripts\pylint.exe" -$mypy = "$PSScriptRoot\backend\venv\Scripts\mypy.exe" - -& $pylint cctext project apps -& $mypy cctext project apps \ No newline at end of file diff --git a/rsconcept/RunTests.ps1 b/rsconcept/RunTests.ps1 deleted file mode 100644 index 571d647f..00000000 --- a/rsconcept/RunTests.ps1 +++ /dev/null @@ -1,12 +0,0 @@ - # Run tests -Set-Location $PSScriptRoot\backend - -$pyExec = "$PSScriptRoot\backend\venv\Scripts\python.exe" -$djangoSrc = "$PSScriptRoot\backend\manage.py" - -& $pyExec $djangoSrc check -& $pyExec $djangoSrc test - -Set-Location $PSScriptRoot\frontend - -& npm test \ No newline at end of file diff --git a/rsconcept/backend/Dockerfile b/rsconcept/backend/Dockerfile index 8b0a9661..db53d958 100644 --- a/rsconcept/backend/Dockerfile +++ b/rsconcept/backend/Dockerfile @@ -46,6 +46,7 @@ RUN mkdir -p $USER_HOME && \ mkdir -p $APP_HOME && \ mkdir -p $APP_HOME/static && \ mkdir -p $APP_HOME/media && \ + mkdir -p $APP_HOME/backup && \ adduser --system --group app # Install python dependencies @@ -61,10 +62,11 @@ COPY project/ ./project COPY fixtures/ ./fixtures COPY manage.py entrypoint.sh ./ RUN sed -i 's/\r$//g' $APP_HOME/entrypoint.sh && \ - chmod +x $APP_HOME/entrypoint.sh && \ + chmod +x $APP_HOME/entrypoint.sh && \ chown -R app:app $APP_HOME && \ chmod -R a+rwx $APP_HOME/static && \ - chmod -R a+rwx $APP_HOME/media + chmod -R a+rwx $APP_HOME/media && \ + chmod -R a+rwx $APP_HOME/backup RUN diff --git a/rsconcept/backend/LocalEnvSetup.ps1 b/rsconcept/backend/LocalEnvSetup.ps1 deleted file mode 100644 index d33f6530..00000000 --- a/rsconcept/backend/LocalEnvSetup.ps1 +++ /dev/null @@ -1,24 +0,0 @@ -# Script creates venv and installs dependencies + imports -Set-Location $PSScriptRoot - -$envPath = "$PSScriptRoot\venv" -$python = "$envPath\Scripts\python.exe" - -if (Test-Path -Path $envPath) { - Write-Host "Removing previous env: $envPath`n" -ForegroundColor DarkGreen - Remove-Item $envPath -Recurse -Force -} - -Write-Host "Creating python env: $envPath`n" -ForegroundColor DarkGreen -& 'python' -m venv $envPath -& $python -m pip install --upgrade pip -& $python -m pip install -r requirements_dev.txt - -$wheel = Get-Childitem -Path import\*win*.whl -Name -if (-not $wheel) { - Write-Error 'Missing import wheel' - Exit 1 -} - -Write-Host "Installing wheel: $wheel`n" -ForegroundColor DarkGreen -& $python -m pip install -I import\$wheel \ No newline at end of file diff --git a/rsconcept/frontend/src/context/ThemeContext.tsx b/rsconcept/frontend/src/context/ThemeContext.tsx index 2e3a1655..2f0dde1f 100644 --- a/rsconcept/frontend/src/context/ThemeContext.tsx +++ b/rsconcept/frontend/src/context/ThemeContext.tsx @@ -57,16 +57,16 @@ export const ThemeState = ({ children }: ThemeStateProps) => { const mainHeight = useMemo( () => { - return !noNavigation ? + return !noNavigation ? 'calc(100vh - 7rem - 2px)' - : '100vh'; + : '100vh'; }, [noNavigation]); const viewportHeight = useMemo( () => { - return !noNavigation ? + return !noNavigation ? 'calc(100vh - 3rem - 2px)' - : '100vh'; + : '100vh'; }, [noNavigation]); return ( diff --git a/rsconcept/frontend/src/pages/RSFormPage/EditorTermGraph.tsx b/rsconcept/frontend/src/pages/RSFormPage/EditorTermGraph.tsx index 2a8539af..46d908aa 100644 --- a/rsconcept/frontend/src/pages/RSFormPage/EditorTermGraph.tsx +++ b/rsconcept/frontend/src/pages/RSFormPage/EditorTermGraph.tsx @@ -358,7 +358,7 @@ function EditorTermGraph({ onOpenEdit, onCreateCst, onDeleteCst }: EditorTermGra
{hoverCst &&
- diff --git a/scripts/dev/LocalDevSetup.ps1 b/scripts/dev/LocalDevSetup.ps1 new file mode 100644 index 00000000..f34dc146 --- /dev/null +++ b/scripts/dev/LocalDevSetup.ps1 @@ -0,0 +1,55 @@ +# Create venv and install dependencies + imports + +$backend = Resolve-Path -Path "$PSScriptRoot\..\..\rsconcept\backend" +$frontend = Resolve-Path -Path "$PSScriptRoot\..\..\rsconcept\fronted" +$envPath = "$backend\venv" +$python = "$envPath\Scripts\python.exe" + +function LocalDevelopmentSetup() { + FrontendSetup + BackendSetup +} + +function FrontendSetup() { + Set-Location $frontend + & npm install +} + +function BackendSetup() { + Set-Location $backend + + ClearPrevious + CreateEnv + InstallPips + InstallImports +} + +function ClearPrevious() { + if (Test-Path -Path $envPath) { + Write-Host "Removing previous env: $envPath`n" -ForegroundColor DarkGreen + Remove-Item $envPath -Recurse -Force + } +} + +function CreateEnv() { + Write-Host "Creating python env: $envPath`n" -ForegroundColor DarkGreen + & 'python' -m venv $envPath +} + +function InstallPips() { + & $python -m pip install --upgrade pip + & $python -m pip install -r requirements_dev.txt +} + +function InstallImports() { + $wheel = Get-Childitem -Path import\*win*.whl -Name + if (-not $wheel) { + Write-Error 'Missing import wheel' + Exit 1 + } + + Write-Host "Installing wheel: $wheel`n" -ForegroundColor DarkGreen + & $python -m pip install -I import\$wheel +} + +LocalDevelopmentSetup \ No newline at end of file diff --git a/scripts/dev/PopulateDevData.ps1 b/scripts/dev/PopulateDevData.ps1 new file mode 100644 index 00000000..c2e550a0 --- /dev/null +++ b/scripts/dev/PopulateDevData.ps1 @@ -0,0 +1,27 @@ +# Initialize database ! +# FOR DEVELOPEMENT BUILDS ONLY! +$container= Read-Host -Prompt "Enter backend container name: " + +$backend = Resolve-Path -Path "$PSScriptRoot\..\..\rsconcept\backend" + +function PopulateDevData() { + ImportInitialData + CreateAdmin +} + +function ImportInitialData() { + docker exec ` + -it $container ` + python manage.py loaddata $backend\fixtures\InitialData.json +} + +function CreateAdmin() { + docker exec ` + -e DJANGO_SUPERUSER_USERNAME=admin ` + -e DJANGO_SUPERUSER_PASSWORD=1234 ` + -e DJANGO_SUPERUSER_EMAIL=admin@admin.com ` + -it $container python manage.py createsuperuser --noinput +} + +PopulateDevData +pause \ No newline at end of file diff --git a/scripts/dev/RunCoverage.ps1 b/scripts/dev/RunCoverage.ps1 new file mode 100644 index 00000000..8f61f676 --- /dev/null +++ b/scripts/dev/RunCoverage.ps1 @@ -0,0 +1,23 @@ +# Run coverage analysis + +$backend = Resolve-Path -Path "$PSScriptRoot\..\..\rsconcept\backend" + +function RunLinters() { + BackendCoverage +} + +function BackendCoverage() { + Set-Location $backend + + $coverageExec = "$backend\venv\Scripts\coverage.exe" + $djangoSrc = "$backend\manage.py" + $exclude = '*/venv/*,*/tests/*,*/migrations/*,*__init__.py,manage.py,apps.py,urls.py,settings.py' + + & $coverageExec run --omit=$exclude $djangoSrc test + & $coverageExec report + & $coverageExec html + + Start-Process "$backend\htmlcov\index.html" +} + +RunLinters \ No newline at end of file diff --git a/scripts/dev/RunLint.ps1 b/scripts/dev/RunLint.ps1 new file mode 100644 index 00000000..2126cf1a --- /dev/null +++ b/scripts/dev/RunLint.ps1 @@ -0,0 +1,17 @@ +# Run coverage analysis +$backend = Resolve-Path -Path "$PSScriptRoot\..\..\rsconcept\backend" + +function RunLinters() { + BackendLint +} + +function BackendLint() { + $pylint = "$backend\venv\Scripts\pylint.exe" + $mypy = "$backend\venv\Scripts\mypy.exe" + + Set-Location $backend + & $pylint cctext project apps + & $mypy cctext project apps +} + +RunLinters \ No newline at end of file diff --git a/rsconcept/RunServer.ps1 b/scripts/dev/RunServer.ps1 similarity index 78% rename from rsconcept/RunServer.ps1 rename to scripts/dev/RunServer.ps1 index 7b51b6b5..d98b949a 100644 --- a/rsconcept/RunServer.ps1 +++ b/scripts/dev/RunServer.ps1 @@ -1,22 +1,25 @@ - # Run local server +# Run local server Param( [switch] $freshStart ) -$pyExec = "$PSScriptRoot\backend\venv\Scripts\python.exe" -$djangoSrc = "$PSScriptRoot\backend\manage.py" +$backend = Resolve-Path -Path "$PSScriptRoot\..\..\rsconcept\backend" +$frontend = Resolve-Path -Path "$PSScriptRoot\..\..\rsconcept\frontend" + +$pyExec = "$backend\venv\Scripts\python.exe" +$djangoSrc = "$backend\manage.py" $initialData = "fixtures/InitialData.json" function RunServer() { - RunBackend - RunFrontend + BackendRun + FrontendRun Start-Sleep -Seconds 1 Start-Process "http://localhost:8000/" Start-Process "http://localhost:3000/" } -function RunBackend() { - Set-Location $PSScriptRoot\backend +function BackendRun() { + Set-Location $backend if ($freshStart) { FlushData DoMigrations @@ -30,15 +33,15 @@ function RunBackend() { Invoke-Expression "cmd /c start powershell -Command { `$Host.UI.RawUI.WindowTitle = 'django'; & $pyExec $djangoSrc runserver }" } -function RunFrontend() { - Set-Location $PSScriptRoot\frontend +function FrontendRun() { + Set-Location $frontend & npm install Invoke-Expression "cmd /c start powershell -Command { `$Host.UI.RawUI.WindowTitle = 'react'; & npm run dev }" } function FlushData { & $pyExec $djangoSrc flush --noinput - $dbPath = "$PSScriptRoot\backend\db.sqlite3" + $dbPath = "$backend\db.sqlite3" if (Test-Path -Path $dbPath -PathType Leaf) { Remove-Item $dbPath } diff --git a/scripts/dev/RunTests.ps1 b/scripts/dev/RunTests.ps1 new file mode 100644 index 00000000..267dd277 --- /dev/null +++ b/scripts/dev/RunTests.ps1 @@ -0,0 +1,25 @@ +# Run tests + +$backend = Resolve-Path -Path "$PSScriptRoot\..\..\rsconcept\backend" +$frontend = Resolve-Path -Path "$PSScriptRoot\..\..\rsconcept\frontend" + +function RunTests() { + TestBackend + TestFrontend +} + +function TestBackend() { + $pyExec = "$backend\venv\Scripts\python.exe" + $djangoSrc = "$backend\manage.py" + + Set-Location $backend + & $pyExec $djangoSrc check + & $pyExec $djangoSrc test +} + +function TestFrontend() { + Set-Location $frontend + & npm test +} + +RunTests \ No newline at end of file diff --git a/scripts/prod/CreateBackup.ps1 b/scripts/prod/CreateBackup.ps1 new file mode 100644 index 00000000..e5f50597 --- /dev/null +++ b/scripts/prod/CreateBackup.ps1 @@ -0,0 +1,71 @@ +# ====== Create database backup ========== +# WARNING! DO NOT RUN THIS FILE AUTOMATICALLY FROM REPOSITORY LOCATION! +# Create a copy in secure location @production host. Update backup scripts from repository manually +# ======================================== + +# Input params +$backupLocation = "D:\DEV\backup\portal" + +$containerDB = "dev-portal-db" +$containerBackend = "dev-portal-backend" +$pgUser = "portal-admin" +$pgDB = "portal-db" + +# Internal params +$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8' +$_date = Get-Date +$_formatDate = $_date.ToString("yyyy-MM-dd") + +$destination = "{0}\{1}" -f $backupLocation, $_formatDate + +function CreateBackup() { + EnsureLocationIsReady + PostgreDump + DjangoDump + Write-Host "Backup saved to $destination" -ForegroundColor DarkGreen +} + +function EnsureLocationIsReady() { + if (Test-Path -Path $destination) { + Write-Host "Removing previous unfinished backup: $destination`n" -ForegroundColor DarkRed + Remove-Item $destination -Recurse -Force + } + New-Item -ItemType Directory -Path $destination | Out-Null + if (Test-Path -Path $archive -PathType Leaf) { + Write-Host "Removing previous backup: $archive`n" -ForegroundColor DarkRed + } +} + +function PostgreDump() { + $host_dbDump = "$destination\$_formatDate-db.dump" + $local_dbDump = "/home/$_formatDate-db.dump" + & docker exec $containerDB ` + pg_dump ` + --username=$pgUser ` + --exclude-table=django_migrations ` + --format=custom ` + --dbname=$pgDB ` + --file=$local_dbDump + & docker cp ${containerDB}:${local_dbDump} $host_dbDump + & docker exec $containerDB rm $local_dbDump +} + +function DjangoDump() { + $host_dataDump = "$destination\$_formatDate-data.json.gz" + $local_dataDump = "/home/app/web/backup/$_formatDate-data.json" + $local_archiveDump = "/home/app/web/backup/$_formatDate-data.json.gz" + & docker exec $containerBackend ` + python manage.py dumpdata ` + --indent=2 ` + --exclude=admin.LogEntry ` + --exclude=sessions ` + --exclude=contenttypes ` + --exclude=auth.permission ` + --output=$local_dataDump + & docker exec $containerBackend gzip --force $local_dataDump + & docker cp ${containerBackend}:${local_archiveDump} $host_dataDump + & docker exec $containerBackend rm $local_archiveDump +} + + +CreateBackup \ No newline at end of file diff --git a/scripts/prod/LoadPostgreBackup.ps1 b/scripts/prod/LoadPostgreBackup.ps1 new file mode 100644 index 00000000..38d751cd --- /dev/null +++ b/scripts/prod/LoadPostgreBackup.ps1 @@ -0,0 +1,27 @@ +# ====== Load database backup ========== +# WARNING! DO NOT RUN THIS FILE AUTOMATICALLY FROM REPOSITORY LOCATION! +# ======================================== + +# Input params +$dataDump = "D:\DEV\backup\portal\2023-09-01\2023-09-01-db.dump" +$target = "local-portal-db" +$pgUser = "portal-admin" +$pgDB = "portal-db" + +function LoadPostgreBackup() { + PostgreLoad +} + +function PostgreLoad() { + $local_dbDump = "/home/db-restore.dump" + & docker cp ${dataDump} ${target}:$local_dbDump + docker exec --interactive $target ` + pg_restore ` + --username=$pgUser ` + --dbname=$pgDB ` + --clean ` + $local_dbDump + & docker exec $target rm $local_dbDump +} + +LoadPostgreBackup \ No newline at end of file