diff --git a/TODO.md b/TODO.md index e69de29..1eb46a9 100644 --- a/TODO.md +++ b/TODO.md @@ -0,0 +1,13 @@ +# TODO + +## phy-z-srv-gpu01 + +Pre-work per [projektplan](server/phy-z-srv-gpu01/notes/20260710-projektplan.md) (§ references below): + +- [x] Write a script to analyse phytrons whole file share which should be feeded to the llm: file number, structure, pdf kind, etc. → `server/phy-z-srv-gpu01/scripts/share-analysis.ps1` +- [ ] Run the share analysis on the Windows server; evaluate: corpus size (§1 revisit trigger), scanned-PDF/OCR share, index size estimate +- [ ] Storage decision (§2.3): index estimate vs. ~960 GB usable NVMe — order extra disks **before** delivery if tight +- [ ] Customer checklist (§2.2): AD bind account, read-only SMB service account, share include/exclude list, DNS name + TLS, internet access at install, chat-logging/GDPR (works council), maintenance ownership, concretize "other workloads" +- [ ] Ansible prep (§2.4): `cifs_mounts` and `llm_stack` role skeletons (`nvidia_gpu` exists); fill `group_vars/phy_z_srv_gpu01.yml` overrides; test non-GPU parts in a throwaway VM +- [ ] German eval set (§2.5): collect 20–30 Q&A with the customer — becomes the acceptance criterion +- [ ] At install time: re-check and pin versions (driver / vLLM / Open WebUI / oikb / model shortlist) per re-entry checklist §5 diff --git a/ansible/roles/nvidia_gpu/defaults/main.yml b/ansible/roles/nvidia_gpu/defaults/main.yml new file mode 100644 index 0000000..3360c16 --- /dev/null +++ b/ansible/roles/nvidia_gpu/defaults/main.yml @@ -0,0 +1,12 @@ +--- +# Driver pinned explicitly — RTX PRO 6000 (Blackwell) needs >= 580. +# Re-check for a newer branch at install time (projektplan §5). +nvidia_driver_package: nvidia-driver-580-server + +# CUDA apt repository keyring (Ubuntu 24.04) +nvidia_cuda_keyring_url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb + +# The full CUDA toolkit on the host is NOT needed for Docker-based workloads +# (driver + container toolkit suffice). Enable only for compiling on the host. +nvidia_install_cuda_toolkit: false +nvidia_cuda_toolkit_package: cuda-toolkit diff --git a/ansible/roles/nvidia_gpu/handlers/main.yml b/ansible/roles/nvidia_gpu/handlers/main.yml new file mode 100644 index 0000000..3627303 --- /dev/null +++ b/ansible/roles/nvidia_gpu/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart docker + service: + name: docker + state: restarted diff --git a/ansible/roles/nvidia_gpu/tasks/main.yml b/ansible/roles/nvidia_gpu/tasks/main.yml new file mode 100644 index 0000000..69e9e60 --- /dev/null +++ b/ansible/roles/nvidia_gpu/tasks/main.yml @@ -0,0 +1,70 @@ +--- +# Automates server/phy-z-srv-gpu01/manuals/20260714-nvidia-driver-install.md +# Requires Docker (geerlingguy.docker) for the container toolkit part. + +- name: Check Secure Boot state + command: mokutil --sb-state + register: nvidia_sb_state + changed_when: false + failed_when: false + +- name: Fail if Secure Boot is enabled + fail: + msg: "Secure Boot is enabled — disable it in the BIOS before installing the NVIDIA driver." + when: "'SecureBoot enabled' in nvidia_sb_state.stdout" + +- name: Install NVIDIA driver + apt: + name: "{{ nvidia_driver_package }}" + state: present + register: nvidia_driver_install + +- name: Reboot after fresh driver install + reboot: + reboot_timeout: 600 + when: nvidia_driver_install.changed + +- name: Install NVIDIA CUDA repository keyring + apt: + deb: "{{ nvidia_cuda_keyring_url }}" + +- name: Install CUDA toolkit (optional, see defaults) + apt: + name: "{{ nvidia_cuda_toolkit_package }}" + state: present + update_cache: true + when: nvidia_install_cuda_toolkit + +- name: Add NVIDIA container toolkit repository key + shell: > + curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey + | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg + args: + creates: /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg + +- name: Add NVIDIA container toolkit repository + copy: + dest: /etc/apt/sources.list.d/nvidia-container-toolkit.list + content: "deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://nvidia.github.io/libnvidia-container/stable/deb/$(ARCH) /\n" + mode: "0644" + +- name: Install NVIDIA container toolkit + apt: + name: nvidia-container-toolkit + state: present + update_cache: true + +- name: Check whether Docker already uses the NVIDIA runtime + command: grep -q nvidia /etc/docker/daemon.json + register: nvidia_docker_runtime + changed_when: false + failed_when: false + +- name: Configure Docker to use the NVIDIA runtime + command: nvidia-ctk runtime configure --runtime=docker + when: nvidia_docker_runtime.rc != 0 + notify: restart docker + +- name: Verify the driver works + command: nvidia-smi + changed_when: false diff --git a/ansible/run.yml b/ansible/run.yml index be9ef07..6e2ce38 100644 --- a/ansible/run.yml +++ b/ansible/run.yml @@ -33,3 +33,58 @@ roles: - role: geerlingguy.security tags: base + + +- hosts: phy_z_srv_jira + become: yes + vars_files: + - "group_vars/secrets.yml" + + pre_tasks: + - name: Update apt cache. + apt: + update_cache: true + cache_valid_time: 3600 + when: ansible_os_family == 'Debian' + + roles: + - role: geerlingguy.security + tags: base + + +- hosts: phy_z_srv_git + become: yes + vars_files: + - "group_vars/secrets.yml" + + pre_tasks: + - name: Update apt cache. + apt: + update_cache: true + cache_valid_time: 3600 + when: ansible_os_family == 'Debian' + + roles: + - role: geerlingguy.security + tags: base + + +- hosts: phy_z_srv_gpu01 + become: yes + vars_files: + - "group_vars/secrets.yml" + + pre_tasks: + - name: Update apt cache. + apt: + update_cache: true + cache_valid_time: 3600 + when: ansible_os_family == 'Debian' + + roles: + - role: geerlingguy.security + tags: base + - role: geerlingguy.docker + tags: docker + - role: nvidia_gpu + tags: gpu diff --git a/server/phy-z-srv-gpu01/README.md b/server/phy-z-srv-gpu01/README.md index 5474fd3..2592a3e 100644 --- a/server/phy-z-srv-gpu01/README.md +++ b/server/phy-z-srv-gpu01/README.md @@ -13,13 +13,19 @@ GPU server for AI/ML workloads. Hardware is ordered/assessed; OS setup and confi HPE ProLiant DL380 Gen12, 2× Intel Xeon 6714P (8-core, 4.0 GHz), 128 GB RAM, NVIDIA RTX PRO 6000 96 GB, 2× 960 GB NVMe SSD, redundant PSU — full BOM in [HW.md](HW.md). -## Documents +## Planning notes -- [Hardware assessment (2026-07-06)](20260706-hardware-assessment.md) -- [Software assessment (2026-07-06)](20260706-software-assessment.md) -- [Deep dive (2026-07-07)](20260707-deep-dive.md) -- [Project plan for the setup (2026-07-10)](20260710-projektplan.md) — start here when the server arrives +- [Hardware assessment (2026-07-06)](notes/20260706-hardware-assessment.md) +- [Software assessment (2026-07-06)](notes/20260706-software-assessment.md) +- [Deep dive (2026-07-07)](notes/20260707-deep-dive.md) +- [Project plan for the setup (2026-07-10)](notes/20260710-projektplan.md) — start here when the server arrives + +Open pre-work items are tracked in the repo-root [TODO.md](../../TODO.md). ## Runbooks -none yet +- [NVIDIA driver, CUDA repo & container toolkit (2026-07-14)](manuals/20260714-nvidia-driver-install.md) — automated by the Ansible role `nvidia_gpu` + +## Scripts + +- [share-analysis.ps1](scripts/share-analysis.ps1) — read-only SMB share analysis (projektplan §2.1); run on a Windows machine with read access to the shares diff --git a/server/phy-z-srv-gpu01/manuals/20260714-nvidia-driver-install.md b/server/phy-z-srv-gpu01/manuals/20260714-nvidia-driver-install.md new file mode 100644 index 0000000..8f1c248 --- /dev/null +++ b/server/phy-z-srv-gpu01/manuals/20260714-nvidia-driver-install.md @@ -0,0 +1,81 @@ +# Manual nvidia driver, cuda repo and container toolkit + +Automated by the Ansible role `ansible/roles/nvidia_gpu` — this manual documents the steps. + +**Disable Secure Boot in BIOS first** (unsigned kernel modules won't load otherwise). + +## NVIDIA driver + +Check if GPUs are recognized by the base OS: +```bash +sudo lspci | grep -i nvidia +``` + +Which should show some output if it finds nvidia devices. + +Search for available drivers for your GPUs: +```bash +sudo ubuntu-drivers devices +``` + +Install the driver pinned. The RTX PRO 6000 (Blackwell) needs **driver >= 580**; +use the `-server` variant and prefer a pinned install over `ubuntu-drivers autoinstall` +so the choice is explicit and reproducible (re-check for a newer branch at install time): +```bash +sudo apt install -y nvidia-driver-580-server +``` + +> Note: do **not** additionally install `cuda-drivers` from the NVIDIA repo — +> that would mix the Ubuntu-archive driver with the NVIDIA-repo driver and the +> two can conflict. Pick one source; we use the Ubuntu archive. + +Reboot the system for changes to take effect: +```bash +sudo reboot +``` + +Show GPU stats with: +```bash +nvidia-smi +``` + +## CUDA repository (toolkit optional) + +Add the NVIDIA CUDA apt repository: + +```bash +wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb +sudo dpkg -i cuda-keyring_1.1-1_all.deb +sudo apt update +``` + +The full CUDA toolkit is **not needed** for Docker-based workloads (vLLM etc. — +the driver plus container toolkit suffice). Only if compiling on the host: +```bash +sudo apt install -y cuda-toolkit # meta package, pulls the current release +``` + +## Container toolkit + +Install the Nvidia Container toolkit: + +```bash +curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \ + && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \ + sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \ + sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list +sudo apt update +sudo apt install -y nvidia-container-toolkit +``` + +Configure Docker to use the NVIDIA runtime (writes `/etc/docker/daemon.json`) and restart it — +without this step `docker run --gpus all` fails: +```bash +sudo nvidia-ctk runtime configure --runtime=docker +sudo systemctl restart docker +``` + +Test a simple cuda container and nvidia-smi command inside: +```bash +docker run --rm --gpus all nvidia/cuda:13.0.0-base-ubuntu24.04 nvidia-smi +``` diff --git a/server/phy-z-srv-gpu01/20260706-hardware-assessment.md b/server/phy-z-srv-gpu01/notes/20260706-hardware-assessment.md similarity index 100% rename from server/phy-z-srv-gpu01/20260706-hardware-assessment.md rename to server/phy-z-srv-gpu01/notes/20260706-hardware-assessment.md diff --git a/server/phy-z-srv-gpu01/20260706-software-assessment.md b/server/phy-z-srv-gpu01/notes/20260706-software-assessment.md similarity index 100% rename from server/phy-z-srv-gpu01/20260706-software-assessment.md rename to server/phy-z-srv-gpu01/notes/20260706-software-assessment.md diff --git a/server/phy-z-srv-gpu01/20260707-deep-dive.md b/server/phy-z-srv-gpu01/notes/20260707-deep-dive.md similarity index 100% rename from server/phy-z-srv-gpu01/20260707-deep-dive.md rename to server/phy-z-srv-gpu01/notes/20260707-deep-dive.md diff --git a/server/phy-z-srv-gpu01/20260710-projektplan.md b/server/phy-z-srv-gpu01/notes/20260710-projektplan.md similarity index 100% rename from server/phy-z-srv-gpu01/20260710-projektplan.md rename to server/phy-z-srv-gpu01/notes/20260710-projektplan.md diff --git a/server/phy-z-srv-gpu01/nvidia-driver-install.md b/server/phy-z-srv-gpu01/nvidia-driver-install.md deleted file mode 100644 index b7e8fa1..0000000 --- a/server/phy-z-srv-gpu01/nvidia-driver-install.md +++ /dev/null @@ -1,62 +0,0 @@ -## Manual nvidia driver, cuda driver and container toolkit - -### NVIDIA driver - -Check if GPUs are recognized by the base OS: -```bash -sudo lspci | grep -i nvidia -``` - -Which should some output if it finds nvidia deivces. - -Search for required drivers for your GPUs: -```bash -sudo ubuntu-drivers devices -``` - -Automatically install all drivers: -```bash -sudo ubuntu-drivers autoinstall -``` - -Reboot the system for changes to take effect: -```bash -sudo reboot -``` - -Shot GPU stats with: -```bash -nvidia-smi -``` - -### Cuda driver - -**Disable Secure Boot in BIOS** - -Install Cuda drivers: - -```bash -wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb -sudo dpkg -i cuda-keyring_1.1-1_all.deb -sudo apt update -sudo apt -y install cuda-toolkit-12-8 -sudo apt install -y cuda-drivers -``` - -### Container toolkit - -Install the Nvidia Container toolkit: - -```bash -curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \ - && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \ - sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \ - sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list -apt update -apt install -y nvidia-container-toolkit -``` - -Test a simple cuda container and nvidia-smi command inside: -```bash -docker run --rm --gpus all nvidia/cuda:13.0.0-base-ubuntu24.04 nvidia-smi -``` diff --git a/server/phy-z-srv-gpu01/scripts/share-analysis.ps1 b/server/phy-z-srv-gpu01/scripts/share-analysis.ps1 new file mode 100644 index 0000000..c41ee3e --- /dev/null +++ b/server/phy-z-srv-gpu01/scripts/share-analysis.ps1 @@ -0,0 +1,215 @@ +<# +.SYNOPSIS + Read-only analysis of SMB shares as LLM knowledge base (projektplan §2.1). + +.DESCRIPTION + Walks one or more share paths and reports, per share: + - volume and file count per file type / category (extractable vs. not) + - size and count per top-level folder (for the include/exclude decision) + - modification-time distribution (sync frequency, data age) + - scanned-PDF ratio via sampling (PDFs without a text layer -> OCR need) + - duplicate/old-version candidates by name patterns + - long paths (> 240 chars) and non-ASCII names (umlauts etc.) + Writes CSVs plus a summary.txt into the output directory. Never writes to + the shares themselves. PowerShell 5.1 compatible; no external modules. + +.EXAMPLE + .\share-analysis.ps1 -Paths '\\fileserver\projekte','\\fileserver\doku' + +.EXAMPLE + .\share-analysis.ps1 -Paths 'D:\shares\projekte' -OutDir C:\temp\analysis -PdfSampleSize 300 +#> +param( + [Parameter(Mandatory = $true)] + [string[]]$Paths, + + [string]$OutDir = (Join-Path (Get-Location) ("share-analysis-" + (Get-Date -Format "yyyyMMdd-HHmmss"))), + + # PDFs sampled per share for the text-layer check + [int]$PdfSampleSize = 200 +) + +$ErrorActionPreference = 'Continue' +New-Item -ItemType Directory -Path $OutDir -Force | Out-Null + +# --- classification --------------------------------------------------------- + +$categories = @{ + 'office' = @('.pdf', '.doc', '.docx', '.xls', '.xlsx', '.xlsm', '.ppt', '.pptx', '.txt', '.md', '.rtf', '.odt', '.ods', '.odp', '.csv', '.vsd', '.vsdx') + 'email' = @('.msg', '.eml') + 'image' = @('.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tif', '.tiff', '.svg', '.heic', '.webp') + 'cad' = @('.dwg', '.dxf', '.step', '.stp', '.iges', '.igs', '.sldprt', '.sldasm', '.slddrw', '.ipt', '.iam', '.catpart', '.catproduct', '.3ds', '.stl') + 'archive' = @('.zip', '.rar', '.7z', '.tar', '.gz', '.bz2', '.iso') + 'media' = @('.mp4', '.avi', '.mov', '.wmv', '.mp3', '.wav') +} +$extToCategory = @{} +foreach ($cat in $categories.Keys) { + foreach ($ext in $categories[$cat]) { $extToCategory[$ext] = $cat } +} + +# name patterns that suggest duplicates / old versions +$dupePattern = '(?i)(kopie|copy|backup|_old|_alt\b|\.bak$|~\$|\(\d+\)\s*(\.[^.]+)?$)' + +$summaryLines = New-Object System.Collections.Generic.List[string] +$summaryLines.Add("Share analysis $(Get-Date -Format 'yyyy-MM-dd HH:mm') — read-only") +$summaryLines.Add("Paths: $($Paths -join ', ')") +$summaryLines.Add("") + +# --- helpers ----------------------------------------------------------------- + +function Test-PdfTextLayer { + # Heuristic: a PDF without any /Font reference in its first 4 MB most + # likely has no text layer (scanned). Also detects encrypted PDFs. + param([string]$Path) + try { + $fs = [System.IO.File]::Open($Path, 'Open', 'Read', 'ReadWrite') + try { + $len = [int][Math]::Min($fs.Length, 4MB) + $buf = New-Object byte[] $len + [void]$fs.Read($buf, 0, $len) + } finally { $fs.Close() } + $text = [System.Text.Encoding]::ASCII.GetString($buf) + if ($text -match '/Encrypt') { return 'encrypted' } + if ($text -match '/Font') { return 'text' } + return 'no-text-layer' + } catch { + return 'unreadable' + } +} + +# --- per-share pass ---------------------------------------------------------- + +foreach ($root in $Paths) { + $shareName = ($root.TrimEnd('\') -split '[\\/]')[-1] + Write-Host "=== Analyzing '$root' ..." -ForegroundColor Cyan + + if (-not (Test-Path -LiteralPath $root)) { + Write-Warning "Path not found or no access: $root" + $summaryLines.Add("[$shareName] SKIPPED — path not found or no access: $root") + continue + } + + # aggregates (streaming — file objects are not kept in memory) + $extStats = @{} # ext -> @{Count; Bytes} + $topStats = @{} # top-level folder -> @{Count; Bytes} + $yearStats = @{} # mtime year -> count + $totalCount = 0L + $totalBytes = 0L + $dupeCount = 0L + $longPaths = 0L + $nonAscii = 0L + $now = Get-Date + $recency = @{ 'last 30 days' = 0L; 'last 90 days' = 0L; 'last 365 days' = 0L; 'older' = 0L } + + # reservoir sample of PDF paths for the text-layer check + $pdfSample = New-Object System.Collections.Generic.List[string] + $pdfSeen = 0L + $rand = New-Object System.Random + $rootLen = $root.TrimEnd('\').Length + + Get-ChildItem -LiteralPath $root -Recurse -File -Force -ErrorAction SilentlyContinue -ErrorVariable +enumErrors | + ForEach-Object { + $totalCount++ + $totalBytes += $_.Length + + $ext = $_.Extension.ToLowerInvariant() + if (-not $ext) { $ext = '(none)' } + if (-not $extStats.ContainsKey($ext)) { $extStats[$ext] = @{ Count = 0L; Bytes = 0L } } + $extStats[$ext].Count++ + $extStats[$ext].Bytes += $_.Length + + # top-level folder relative to the share root + $rel = $_.FullName.Substring($rootLen).TrimStart('\') + $top = if ($rel.Contains('\')) { $rel.Split('\')[0] } else { '(root)' } + if (-not $topStats.ContainsKey($top)) { $topStats[$top] = @{ Count = 0L; Bytes = 0L } } + $topStats[$top].Count++ + $topStats[$top].Bytes += $_.Length + + $year = $_.LastWriteTime.Year + if (-not $yearStats.ContainsKey($year)) { $yearStats[$year] = 0L } + $yearStats[$year]++ + + $age = ($now - $_.LastWriteTime).TotalDays + if ($age -le 30) { $recency['last 30 days']++ } + elseif ($age -le 90) { $recency['last 90 days']++ } + elseif ($age -le 365) { $recency['last 365 days']++ } + else { $recency['older']++ } + + if ($_.Name -match $dupePattern) { $dupeCount++ } + if ($_.FullName.Length -gt 240) { $longPaths++ } + if ($_.Name -match '[^\x00-\x7F]') { $nonAscii++ } + + if ($ext -eq '.pdf') { + $pdfSeen++ + if ($pdfSample.Count -lt $PdfSampleSize) { + $pdfSample.Add($_.FullName) + } else { + $i = $rand.Next(0, [int][Math]::Min($pdfSeen, [int]::MaxValue)) + if ($i -lt $PdfSampleSize) { $pdfSample[$i] = $_.FullName } + } + } + + if ($totalCount % 20000 -eq 0) { + Write-Host (" {0:N0} files, {1:N1} GB ..." -f $totalCount, ($totalBytes / 1GB)) + } + } + + # PDF text-layer sampling + Write-Host " Sampling $($pdfSample.Count) of $pdfSeen PDFs for text layer ..." + $pdfResults = @{ 'text' = 0; 'no-text-layer' = 0; 'encrypted' = 0; 'unreadable' = 0 } + foreach ($p in $pdfSample) { $pdfResults[(Test-PdfTextLayer $p)]++ } + + # --- write per-share CSVs --- + $prefix = Join-Path $OutDir $shareName + + $extStats.GetEnumerator() | ForEach-Object { + $cat = if ($extToCategory.ContainsKey($_.Key)) { $extToCategory[$_.Key] } else { 'other' } + [PSCustomObject]@{ Extension = $_.Key; Category = $cat; Count = $_.Value.Count; GB = [Math]::Round($_.Value.Bytes / 1GB, 2) } + } | Sort-Object GB -Descending | Export-Csv "$prefix-file-types.csv" -NoTypeInformation -Encoding UTF8 + + $topStats.GetEnumerator() | ForEach-Object { + [PSCustomObject]@{ Folder = $_.Key; Count = $_.Value.Count; GB = [Math]::Round($_.Value.Bytes / 1GB, 2) } + } | Sort-Object GB -Descending | Export-Csv "$prefix-toplevel-folders.csv" -NoTypeInformation -Encoding UTF8 + + $yearStats.GetEnumerator() | ForEach-Object { + [PSCustomObject]@{ Year = $_.Key; Count = $_.Value } + } | Sort-Object Year | Export-Csv "$prefix-mtime-years.csv" -NoTypeInformation -Encoding UTF8 + + # --- category rollup for the summary --- + $catBytes = @{}; $catCount = @{} + foreach ($e in $extStats.GetEnumerator()) { + $cat = if ($extToCategory.ContainsKey($e.Key)) { $extToCategory[$e.Key] } else { 'other' } + if (-not $catBytes.ContainsKey($cat)) { $catBytes[$cat] = 0L; $catCount[$cat] = 0L } + $catBytes[$cat] += $e.Value.Bytes + $catCount[$cat] += $e.Value.Count + } + + $summaryLines.Add("[$shareName] $root") + $summaryLines.Add((" Total: {0:N0} files, {1:N1} GB" -f $totalCount, ($totalBytes / 1GB))) + foreach ($c in ($catBytes.Keys | Sort-Object { $catBytes[$_] } -Descending)) { + $summaryLines.Add((" {0,-8} {1,10:N0} files {2,10:N1} GB" -f $c, $catCount[$c], ($catBytes[$c] / 1GB))) + } + if ($pdfSample.Count -gt 0) { + $scannedPct = [Math]::Round(100 * $pdfResults['no-text-layer'] / $pdfSample.Count, 1) + $summaryLines.Add((" PDFs: {0:N0} total; sample of {1}: {2} with text, {3} WITHOUT text layer (~{4}% -> OCR), {5} encrypted, {6} unreadable" -f ` + $pdfSeen, $pdfSample.Count, $pdfResults['text'], $pdfResults['no-text-layer'], $scannedPct, $pdfResults['encrypted'], $pdfResults['unreadable'])) + } + $summaryLines.Add((" Duplicate/old-version name patterns: {0:N0} files" -f $dupeCount)) + $summaryLines.Add((" Paths > 240 chars: {0:N0}; non-ASCII names: {1:N0}" -f $longPaths, $nonAscii)) + $summaryLines.Add(" Modified: " + (($recency.GetEnumerator() | Sort-Object { @('last 30 days','last 90 days','last 365 days','older').IndexOf($_.Key) } | + ForEach-Object { "$($_.Key): $("{0:N0}" -f $_.Value)" }) -join ' | ')) + $summaryLines.Add("") +} + +if ($enumErrors) { + $summaryLines.Add(("NOTE: {0:N0} paths could not be read (access denied / path too long). Counts are lower bounds." -f $enumErrors.Count)) + $enumErrors | ForEach-Object { $_.TargetObject } | Select-Object -First 50 | + Set-Content (Join-Path $OutDir "enumeration-errors-sample.txt") -Encoding UTF8 +} + +$summaryPath = Join-Path $OutDir "summary.txt" +$summaryLines | Set-Content $summaryPath -Encoding UTF8 + +Write-Host "" +Write-Host "Done. Results in: $OutDir" -ForegroundColor Green +Get-Content $summaryPath | Write-Host