Add base plays for all hosts, nvidia_gpu role, GPU pre-work items
- run.yml: base plays (geerlingguy.security) for jira and git; gpu01 play with security + docker + nvidia_gpu - roles/nvidia_gpu: driver pinned >=580 (Blackwell), CUDA repo, container toolkit incl. the nvidia-ctk runtime configure step - manuals/20260714-nvidia-driver-install.md: dated per convention, corrected (pinned -server driver instead of autoinstall+cuda-drivers mix, toolkit optional, added missing nvidia-ctk/docker restart step) - gpu01 folder: planning docs under notes/, runbooks under manuals/, scripts/; convention documented in CLAUDE.md - scripts/share-analysis.ps1: read-only SMB share analysis for the Windows server (projektplan §2.1) - TODO.md: Phase-0 pre-work items from the projektplan Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,13 @@
|
|||||||
|
# TODO
|
||||||
|
|
||||||
|
## phy-z-srv-gpu01
|
||||||
|
|
||||||
|
Pre-work per [projektplan](server/phy-z-srv-gpu01/notes/20260710-projektplan.md) (§ references below):
|
||||||
|
|
||||||
|
- [x] Write a script to analyse phytrons whole file share which should be feeded to the llm: file number, structure, pdf kind, etc. → `server/phy-z-srv-gpu01/scripts/share-analysis.ps1`
|
||||||
|
- [ ] Run the share analysis on the Windows server; evaluate: corpus size (§1 revisit trigger), scanned-PDF/OCR share, index size estimate
|
||||||
|
- [ ] Storage decision (§2.3): index estimate vs. ~960 GB usable NVMe — order extra disks **before** delivery if tight
|
||||||
|
- [ ] Customer checklist (§2.2): AD bind account, read-only SMB service account, share include/exclude list, DNS name + TLS, internet access at install, chat-logging/GDPR (works council), maintenance ownership, concretize "other workloads"
|
||||||
|
- [ ] Ansible prep (§2.4): `cifs_mounts` and `llm_stack` role skeletons (`nvidia_gpu` exists); fill `group_vars/phy_z_srv_gpu01.yml` overrides; test non-GPU parts in a throwaway VM
|
||||||
|
- [ ] German eval set (§2.5): collect 20–30 Q&A with the customer — becomes the acceptance criterion
|
||||||
|
- [ ] At install time: re-check and pin versions (driver / vLLM / Open WebUI / oikb / model shortlist) per re-entry checklist §5
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
---
|
||||||
|
# Driver pinned explicitly — RTX PRO 6000 (Blackwell) needs >= 580.
|
||||||
|
# Re-check for a newer branch at install time (projektplan §5).
|
||||||
|
nvidia_driver_package: nvidia-driver-580-server
|
||||||
|
|
||||||
|
# CUDA apt repository keyring (Ubuntu 24.04)
|
||||||
|
nvidia_cuda_keyring_url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
|
||||||
|
|
||||||
|
# The full CUDA toolkit on the host is NOT needed for Docker-based workloads
|
||||||
|
# (driver + container toolkit suffice). Enable only for compiling on the host.
|
||||||
|
nvidia_install_cuda_toolkit: false
|
||||||
|
nvidia_cuda_toolkit_package: cuda-toolkit
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
---
|
||||||
|
- name: restart docker
|
||||||
|
service:
|
||||||
|
name: docker
|
||||||
|
state: restarted
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
---
|
||||||
|
# Automates server/phy-z-srv-gpu01/manuals/20260714-nvidia-driver-install.md
|
||||||
|
# Requires Docker (geerlingguy.docker) for the container toolkit part.
|
||||||
|
|
||||||
|
- name: Check Secure Boot state
|
||||||
|
command: mokutil --sb-state
|
||||||
|
register: nvidia_sb_state
|
||||||
|
changed_when: false
|
||||||
|
failed_when: false
|
||||||
|
|
||||||
|
- name: Fail if Secure Boot is enabled
|
||||||
|
fail:
|
||||||
|
msg: "Secure Boot is enabled — disable it in the BIOS before installing the NVIDIA driver."
|
||||||
|
when: "'SecureBoot enabled' in nvidia_sb_state.stdout"
|
||||||
|
|
||||||
|
- name: Install NVIDIA driver
|
||||||
|
apt:
|
||||||
|
name: "{{ nvidia_driver_package }}"
|
||||||
|
state: present
|
||||||
|
register: nvidia_driver_install
|
||||||
|
|
||||||
|
- name: Reboot after fresh driver install
|
||||||
|
reboot:
|
||||||
|
reboot_timeout: 600
|
||||||
|
when: nvidia_driver_install.changed
|
||||||
|
|
||||||
|
- name: Install NVIDIA CUDA repository keyring
|
||||||
|
apt:
|
||||||
|
deb: "{{ nvidia_cuda_keyring_url }}"
|
||||||
|
|
||||||
|
- name: Install CUDA toolkit (optional, see defaults)
|
||||||
|
apt:
|
||||||
|
name: "{{ nvidia_cuda_toolkit_package }}"
|
||||||
|
state: present
|
||||||
|
update_cache: true
|
||||||
|
when: nvidia_install_cuda_toolkit
|
||||||
|
|
||||||
|
- name: Add NVIDIA container toolkit repository key
|
||||||
|
shell: >
|
||||||
|
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey
|
||||||
|
| gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
|
||||||
|
args:
|
||||||
|
creates: /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
|
||||||
|
|
||||||
|
- name: Add NVIDIA container toolkit repository
|
||||||
|
copy:
|
||||||
|
dest: /etc/apt/sources.list.d/nvidia-container-toolkit.list
|
||||||
|
content: "deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://nvidia.github.io/libnvidia-container/stable/deb/$(ARCH) /\n"
|
||||||
|
mode: "0644"
|
||||||
|
|
||||||
|
- name: Install NVIDIA container toolkit
|
||||||
|
apt:
|
||||||
|
name: nvidia-container-toolkit
|
||||||
|
state: present
|
||||||
|
update_cache: true
|
||||||
|
|
||||||
|
- name: Check whether Docker already uses the NVIDIA runtime
|
||||||
|
command: grep -q nvidia /etc/docker/daemon.json
|
||||||
|
register: nvidia_docker_runtime
|
||||||
|
changed_when: false
|
||||||
|
failed_when: false
|
||||||
|
|
||||||
|
- name: Configure Docker to use the NVIDIA runtime
|
||||||
|
command: nvidia-ctk runtime configure --runtime=docker
|
||||||
|
when: nvidia_docker_runtime.rc != 0
|
||||||
|
notify: restart docker
|
||||||
|
|
||||||
|
- name: Verify the driver works
|
||||||
|
command: nvidia-smi
|
||||||
|
changed_when: false
|
||||||
@@ -33,3 +33,58 @@
|
|||||||
roles:
|
roles:
|
||||||
- role: geerlingguy.security
|
- role: geerlingguy.security
|
||||||
tags: base
|
tags: base
|
||||||
|
|
||||||
|
|
||||||
|
- hosts: phy_z_srv_jira
|
||||||
|
become: yes
|
||||||
|
vars_files:
|
||||||
|
- "group_vars/secrets.yml"
|
||||||
|
|
||||||
|
pre_tasks:
|
||||||
|
- name: Update apt cache.
|
||||||
|
apt:
|
||||||
|
update_cache: true
|
||||||
|
cache_valid_time: 3600
|
||||||
|
when: ansible_os_family == 'Debian'
|
||||||
|
|
||||||
|
roles:
|
||||||
|
- role: geerlingguy.security
|
||||||
|
tags: base
|
||||||
|
|
||||||
|
|
||||||
|
- hosts: phy_z_srv_git
|
||||||
|
become: yes
|
||||||
|
vars_files:
|
||||||
|
- "group_vars/secrets.yml"
|
||||||
|
|
||||||
|
pre_tasks:
|
||||||
|
- name: Update apt cache.
|
||||||
|
apt:
|
||||||
|
update_cache: true
|
||||||
|
cache_valid_time: 3600
|
||||||
|
when: ansible_os_family == 'Debian'
|
||||||
|
|
||||||
|
roles:
|
||||||
|
- role: geerlingguy.security
|
||||||
|
tags: base
|
||||||
|
|
||||||
|
|
||||||
|
- hosts: phy_z_srv_gpu01
|
||||||
|
become: yes
|
||||||
|
vars_files:
|
||||||
|
- "group_vars/secrets.yml"
|
||||||
|
|
||||||
|
pre_tasks:
|
||||||
|
- name: Update apt cache.
|
||||||
|
apt:
|
||||||
|
update_cache: true
|
||||||
|
cache_valid_time: 3600
|
||||||
|
when: ansible_os_family == 'Debian'
|
||||||
|
|
||||||
|
roles:
|
||||||
|
- role: geerlingguy.security
|
||||||
|
tags: base
|
||||||
|
- role: geerlingguy.docker
|
||||||
|
tags: docker
|
||||||
|
- role: nvidia_gpu
|
||||||
|
tags: gpu
|
||||||
|
|||||||
@@ -13,13 +13,19 @@ GPU server for AI/ML workloads. Hardware is ordered/assessed; OS setup and confi
|
|||||||
HPE ProLiant DL380 Gen12, 2× Intel Xeon 6714P (8-core, 4.0 GHz), 128 GB RAM,
|
HPE ProLiant DL380 Gen12, 2× Intel Xeon 6714P (8-core, 4.0 GHz), 128 GB RAM,
|
||||||
NVIDIA RTX PRO 6000 96 GB, 2× 960 GB NVMe SSD, redundant PSU — full BOM in [HW.md](HW.md).
|
NVIDIA RTX PRO 6000 96 GB, 2× 960 GB NVMe SSD, redundant PSU — full BOM in [HW.md](HW.md).
|
||||||
|
|
||||||
## Documents
|
## Planning notes
|
||||||
|
|
||||||
- [Hardware assessment (2026-07-06)](20260706-hardware-assessment.md)
|
- [Hardware assessment (2026-07-06)](notes/20260706-hardware-assessment.md)
|
||||||
- [Software assessment (2026-07-06)](20260706-software-assessment.md)
|
- [Software assessment (2026-07-06)](notes/20260706-software-assessment.md)
|
||||||
- [Deep dive (2026-07-07)](20260707-deep-dive.md)
|
- [Deep dive (2026-07-07)](notes/20260707-deep-dive.md)
|
||||||
- [Project plan for the setup (2026-07-10)](20260710-projektplan.md) — start here when the server arrives
|
- [Project plan for the setup (2026-07-10)](notes/20260710-projektplan.md) — start here when the server arrives
|
||||||
|
|
||||||
|
Open pre-work items are tracked in the repo-root [TODO.md](../../TODO.md).
|
||||||
|
|
||||||
## Runbooks
|
## Runbooks
|
||||||
|
|
||||||
none yet
|
- [NVIDIA driver, CUDA repo & container toolkit (2026-07-14)](manuals/20260714-nvidia-driver-install.md) — automated by the Ansible role `nvidia_gpu`
|
||||||
|
|
||||||
|
## Scripts
|
||||||
|
|
||||||
|
- [share-analysis.ps1](scripts/share-analysis.ps1) — read-only SMB share analysis (projektplan §2.1); run on a Windows machine with read access to the shares
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
# Manual nvidia driver, cuda repo and container toolkit
|
||||||
|
|
||||||
|
Automated by the Ansible role `ansible/roles/nvidia_gpu` — this manual documents the steps.
|
||||||
|
|
||||||
|
**Disable Secure Boot in BIOS first** (unsigned kernel modules won't load otherwise).
|
||||||
|
|
||||||
|
## NVIDIA driver
|
||||||
|
|
||||||
|
Check if GPUs are recognized by the base OS:
|
||||||
|
```bash
|
||||||
|
sudo lspci | grep -i nvidia
|
||||||
|
```
|
||||||
|
|
||||||
|
Which should show some output if it finds nvidia devices.
|
||||||
|
|
||||||
|
Search for available drivers for your GPUs:
|
||||||
|
```bash
|
||||||
|
sudo ubuntu-drivers devices
|
||||||
|
```
|
||||||
|
|
||||||
|
Install the driver pinned. The RTX PRO 6000 (Blackwell) needs **driver >= 580**;
|
||||||
|
use the `-server` variant and prefer a pinned install over `ubuntu-drivers autoinstall`
|
||||||
|
so the choice is explicit and reproducible (re-check for a newer branch at install time):
|
||||||
|
```bash
|
||||||
|
sudo apt install -y nvidia-driver-580-server
|
||||||
|
```
|
||||||
|
|
||||||
|
> Note: do **not** additionally install `cuda-drivers` from the NVIDIA repo —
|
||||||
|
> that would mix the Ubuntu-archive driver with the NVIDIA-repo driver and the
|
||||||
|
> two can conflict. Pick one source; we use the Ubuntu archive.
|
||||||
|
|
||||||
|
Reboot the system for changes to take effect:
|
||||||
|
```bash
|
||||||
|
sudo reboot
|
||||||
|
```
|
||||||
|
|
||||||
|
Show GPU stats with:
|
||||||
|
```bash
|
||||||
|
nvidia-smi
|
||||||
|
```
|
||||||
|
|
||||||
|
## CUDA repository (toolkit optional)
|
||||||
|
|
||||||
|
Add the NVIDIA CUDA apt repository:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
|
||||||
|
sudo dpkg -i cuda-keyring_1.1-1_all.deb
|
||||||
|
sudo apt update
|
||||||
|
```
|
||||||
|
|
||||||
|
The full CUDA toolkit is **not needed** for Docker-based workloads (vLLM etc. —
|
||||||
|
the driver plus container toolkit suffice). Only if compiling on the host:
|
||||||
|
```bash
|
||||||
|
sudo apt install -y cuda-toolkit # meta package, pulls the current release
|
||||||
|
```
|
||||||
|
|
||||||
|
## Container toolkit
|
||||||
|
|
||||||
|
Install the Nvidia Container toolkit:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
|
||||||
|
&& curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
|
||||||
|
sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
|
||||||
|
sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
|
||||||
|
sudo apt update
|
||||||
|
sudo apt install -y nvidia-container-toolkit
|
||||||
|
```
|
||||||
|
|
||||||
|
Configure Docker to use the NVIDIA runtime (writes `/etc/docker/daemon.json`) and restart it —
|
||||||
|
without this step `docker run --gpus all` fails:
|
||||||
|
```bash
|
||||||
|
sudo nvidia-ctk runtime configure --runtime=docker
|
||||||
|
sudo systemctl restart docker
|
||||||
|
```
|
||||||
|
|
||||||
|
Test a simple cuda container and nvidia-smi command inside:
|
||||||
|
```bash
|
||||||
|
docker run --rm --gpus all nvidia/cuda:13.0.0-base-ubuntu24.04 nvidia-smi
|
||||||
|
```
|
||||||
@@ -1,62 +0,0 @@
|
|||||||
## Manual nvidia driver, cuda driver and container toolkit
|
|
||||||
|
|
||||||
### NVIDIA driver
|
|
||||||
|
|
||||||
Check if GPUs are recognized by the base OS:
|
|
||||||
```bash
|
|
||||||
sudo lspci | grep -i nvidia
|
|
||||||
```
|
|
||||||
|
|
||||||
Which should some output if it finds nvidia deivces.
|
|
||||||
|
|
||||||
Search for required drivers for your GPUs:
|
|
||||||
```bash
|
|
||||||
sudo ubuntu-drivers devices
|
|
||||||
```
|
|
||||||
|
|
||||||
Automatically install all drivers:
|
|
||||||
```bash
|
|
||||||
sudo ubuntu-drivers autoinstall
|
|
||||||
```
|
|
||||||
|
|
||||||
Reboot the system for changes to take effect:
|
|
||||||
```bash
|
|
||||||
sudo reboot
|
|
||||||
```
|
|
||||||
|
|
||||||
Shot GPU stats with:
|
|
||||||
```bash
|
|
||||||
nvidia-smi
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cuda driver
|
|
||||||
|
|
||||||
**Disable Secure Boot in BIOS**
|
|
||||||
|
|
||||||
Install Cuda drivers:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
|
|
||||||
sudo dpkg -i cuda-keyring_1.1-1_all.deb
|
|
||||||
sudo apt update
|
|
||||||
sudo apt -y install cuda-toolkit-12-8
|
|
||||||
sudo apt install -y cuda-drivers
|
|
||||||
```
|
|
||||||
|
|
||||||
### Container toolkit
|
|
||||||
|
|
||||||
Install the Nvidia Container toolkit:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
|
|
||||||
&& curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
|
|
||||||
sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
|
|
||||||
sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
|
|
||||||
apt update
|
|
||||||
apt install -y nvidia-container-toolkit
|
|
||||||
```
|
|
||||||
|
|
||||||
Test a simple cuda container and nvidia-smi command inside:
|
|
||||||
```bash
|
|
||||||
docker run --rm --gpus all nvidia/cuda:13.0.0-base-ubuntu24.04 nvidia-smi
|
|
||||||
```
|
|
||||||
@@ -0,0 +1,215 @@
|
|||||||
|
<#
|
||||||
|
.SYNOPSIS
|
||||||
|
Read-only analysis of SMB shares as LLM knowledge base (projektplan §2.1).
|
||||||
|
|
||||||
|
.DESCRIPTION
|
||||||
|
Walks one or more share paths and reports, per share:
|
||||||
|
- volume and file count per file type / category (extractable vs. not)
|
||||||
|
- size and count per top-level folder (for the include/exclude decision)
|
||||||
|
- modification-time distribution (sync frequency, data age)
|
||||||
|
- scanned-PDF ratio via sampling (PDFs without a text layer -> OCR need)
|
||||||
|
- duplicate/old-version candidates by name patterns
|
||||||
|
- long paths (> 240 chars) and non-ASCII names (umlauts etc.)
|
||||||
|
Writes CSVs plus a summary.txt into the output directory. Never writes to
|
||||||
|
the shares themselves. PowerShell 5.1 compatible; no external modules.
|
||||||
|
|
||||||
|
.EXAMPLE
|
||||||
|
.\share-analysis.ps1 -Paths '\\fileserver\projekte','\\fileserver\doku'
|
||||||
|
|
||||||
|
.EXAMPLE
|
||||||
|
.\share-analysis.ps1 -Paths 'D:\shares\projekte' -OutDir C:\temp\analysis -PdfSampleSize 300
|
||||||
|
#>
|
||||||
|
param(
|
||||||
|
[Parameter(Mandatory = $true)]
|
||||||
|
[string[]]$Paths,
|
||||||
|
|
||||||
|
[string]$OutDir = (Join-Path (Get-Location) ("share-analysis-" + (Get-Date -Format "yyyyMMdd-HHmmss"))),
|
||||||
|
|
||||||
|
# PDFs sampled per share for the text-layer check
|
||||||
|
[int]$PdfSampleSize = 200
|
||||||
|
)
|
||||||
|
|
||||||
|
$ErrorActionPreference = 'Continue'
|
||||||
|
New-Item -ItemType Directory -Path $OutDir -Force | Out-Null
|
||||||
|
|
||||||
|
# --- classification ---------------------------------------------------------
|
||||||
|
|
||||||
|
$categories = @{
|
||||||
|
'office' = @('.pdf', '.doc', '.docx', '.xls', '.xlsx', '.xlsm', '.ppt', '.pptx', '.txt', '.md', '.rtf', '.odt', '.ods', '.odp', '.csv', '.vsd', '.vsdx')
|
||||||
|
'email' = @('.msg', '.eml')
|
||||||
|
'image' = @('.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tif', '.tiff', '.svg', '.heic', '.webp')
|
||||||
|
'cad' = @('.dwg', '.dxf', '.step', '.stp', '.iges', '.igs', '.sldprt', '.sldasm', '.slddrw', '.ipt', '.iam', '.catpart', '.catproduct', '.3ds', '.stl')
|
||||||
|
'archive' = @('.zip', '.rar', '.7z', '.tar', '.gz', '.bz2', '.iso')
|
||||||
|
'media' = @('.mp4', '.avi', '.mov', '.wmv', '.mp3', '.wav')
|
||||||
|
}
|
||||||
|
$extToCategory = @{}
|
||||||
|
foreach ($cat in $categories.Keys) {
|
||||||
|
foreach ($ext in $categories[$cat]) { $extToCategory[$ext] = $cat }
|
||||||
|
}
|
||||||
|
|
||||||
|
# name patterns that suggest duplicates / old versions
|
||||||
|
$dupePattern = '(?i)(kopie|copy|backup|_old|_alt\b|\.bak$|~\$|\(\d+\)\s*(\.[^.]+)?$)'
|
||||||
|
|
||||||
|
$summaryLines = New-Object System.Collections.Generic.List[string]
|
||||||
|
$summaryLines.Add("Share analysis $(Get-Date -Format 'yyyy-MM-dd HH:mm') — read-only")
|
||||||
|
$summaryLines.Add("Paths: $($Paths -join ', ')")
|
||||||
|
$summaryLines.Add("")
|
||||||
|
|
||||||
|
# --- helpers -----------------------------------------------------------------
|
||||||
|
|
||||||
|
function Test-PdfTextLayer {
|
||||||
|
# Heuristic: a PDF without any /Font reference in its first 4 MB most
|
||||||
|
# likely has no text layer (scanned). Also detects encrypted PDFs.
|
||||||
|
param([string]$Path)
|
||||||
|
try {
|
||||||
|
$fs = [System.IO.File]::Open($Path, 'Open', 'Read', 'ReadWrite')
|
||||||
|
try {
|
||||||
|
$len = [int][Math]::Min($fs.Length, 4MB)
|
||||||
|
$buf = New-Object byte[] $len
|
||||||
|
[void]$fs.Read($buf, 0, $len)
|
||||||
|
} finally { $fs.Close() }
|
||||||
|
$text = [System.Text.Encoding]::ASCII.GetString($buf)
|
||||||
|
if ($text -match '/Encrypt') { return 'encrypted' }
|
||||||
|
if ($text -match '/Font') { return 'text' }
|
||||||
|
return 'no-text-layer'
|
||||||
|
} catch {
|
||||||
|
return 'unreadable'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- per-share pass ----------------------------------------------------------
|
||||||
|
|
||||||
|
foreach ($root in $Paths) {
|
||||||
|
$shareName = ($root.TrimEnd('\') -split '[\\/]')[-1]
|
||||||
|
Write-Host "=== Analyzing '$root' ..." -ForegroundColor Cyan
|
||||||
|
|
||||||
|
if (-not (Test-Path -LiteralPath $root)) {
|
||||||
|
Write-Warning "Path not found or no access: $root"
|
||||||
|
$summaryLines.Add("[$shareName] SKIPPED — path not found or no access: $root")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
# aggregates (streaming — file objects are not kept in memory)
|
||||||
|
$extStats = @{} # ext -> @{Count; Bytes}
|
||||||
|
$topStats = @{} # top-level folder -> @{Count; Bytes}
|
||||||
|
$yearStats = @{} # mtime year -> count
|
||||||
|
$totalCount = 0L
|
||||||
|
$totalBytes = 0L
|
||||||
|
$dupeCount = 0L
|
||||||
|
$longPaths = 0L
|
||||||
|
$nonAscii = 0L
|
||||||
|
$now = Get-Date
|
||||||
|
$recency = @{ 'last 30 days' = 0L; 'last 90 days' = 0L; 'last 365 days' = 0L; 'older' = 0L }
|
||||||
|
|
||||||
|
# reservoir sample of PDF paths for the text-layer check
|
||||||
|
$pdfSample = New-Object System.Collections.Generic.List[string]
|
||||||
|
$pdfSeen = 0L
|
||||||
|
$rand = New-Object System.Random
|
||||||
|
$rootLen = $root.TrimEnd('\').Length
|
||||||
|
|
||||||
|
Get-ChildItem -LiteralPath $root -Recurse -File -Force -ErrorAction SilentlyContinue -ErrorVariable +enumErrors |
|
||||||
|
ForEach-Object {
|
||||||
|
$totalCount++
|
||||||
|
$totalBytes += $_.Length
|
||||||
|
|
||||||
|
$ext = $_.Extension.ToLowerInvariant()
|
||||||
|
if (-not $ext) { $ext = '(none)' }
|
||||||
|
if (-not $extStats.ContainsKey($ext)) { $extStats[$ext] = @{ Count = 0L; Bytes = 0L } }
|
||||||
|
$extStats[$ext].Count++
|
||||||
|
$extStats[$ext].Bytes += $_.Length
|
||||||
|
|
||||||
|
# top-level folder relative to the share root
|
||||||
|
$rel = $_.FullName.Substring($rootLen).TrimStart('\')
|
||||||
|
$top = if ($rel.Contains('\')) { $rel.Split('\')[0] } else { '(root)' }
|
||||||
|
if (-not $topStats.ContainsKey($top)) { $topStats[$top] = @{ Count = 0L; Bytes = 0L } }
|
||||||
|
$topStats[$top].Count++
|
||||||
|
$topStats[$top].Bytes += $_.Length
|
||||||
|
|
||||||
|
$year = $_.LastWriteTime.Year
|
||||||
|
if (-not $yearStats.ContainsKey($year)) { $yearStats[$year] = 0L }
|
||||||
|
$yearStats[$year]++
|
||||||
|
|
||||||
|
$age = ($now - $_.LastWriteTime).TotalDays
|
||||||
|
if ($age -le 30) { $recency['last 30 days']++ }
|
||||||
|
elseif ($age -le 90) { $recency['last 90 days']++ }
|
||||||
|
elseif ($age -le 365) { $recency['last 365 days']++ }
|
||||||
|
else { $recency['older']++ }
|
||||||
|
|
||||||
|
if ($_.Name -match $dupePattern) { $dupeCount++ }
|
||||||
|
if ($_.FullName.Length -gt 240) { $longPaths++ }
|
||||||
|
if ($_.Name -match '[^\x00-\x7F]') { $nonAscii++ }
|
||||||
|
|
||||||
|
if ($ext -eq '.pdf') {
|
||||||
|
$pdfSeen++
|
||||||
|
if ($pdfSample.Count -lt $PdfSampleSize) {
|
||||||
|
$pdfSample.Add($_.FullName)
|
||||||
|
} else {
|
||||||
|
$i = $rand.Next(0, [int][Math]::Min($pdfSeen, [int]::MaxValue))
|
||||||
|
if ($i -lt $PdfSampleSize) { $pdfSample[$i] = $_.FullName }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($totalCount % 20000 -eq 0) {
|
||||||
|
Write-Host (" {0:N0} files, {1:N1} GB ..." -f $totalCount, ($totalBytes / 1GB))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# PDF text-layer sampling
|
||||||
|
Write-Host " Sampling $($pdfSample.Count) of $pdfSeen PDFs for text layer ..."
|
||||||
|
$pdfResults = @{ 'text' = 0; 'no-text-layer' = 0; 'encrypted' = 0; 'unreadable' = 0 }
|
||||||
|
foreach ($p in $pdfSample) { $pdfResults[(Test-PdfTextLayer $p)]++ }
|
||||||
|
|
||||||
|
# --- write per-share CSVs ---
|
||||||
|
$prefix = Join-Path $OutDir $shareName
|
||||||
|
|
||||||
|
$extStats.GetEnumerator() | ForEach-Object {
|
||||||
|
$cat = if ($extToCategory.ContainsKey($_.Key)) { $extToCategory[$_.Key] } else { 'other' }
|
||||||
|
[PSCustomObject]@{ Extension = $_.Key; Category = $cat; Count = $_.Value.Count; GB = [Math]::Round($_.Value.Bytes / 1GB, 2) }
|
||||||
|
} | Sort-Object GB -Descending | Export-Csv "$prefix-file-types.csv" -NoTypeInformation -Encoding UTF8
|
||||||
|
|
||||||
|
$topStats.GetEnumerator() | ForEach-Object {
|
||||||
|
[PSCustomObject]@{ Folder = $_.Key; Count = $_.Value.Count; GB = [Math]::Round($_.Value.Bytes / 1GB, 2) }
|
||||||
|
} | Sort-Object GB -Descending | Export-Csv "$prefix-toplevel-folders.csv" -NoTypeInformation -Encoding UTF8
|
||||||
|
|
||||||
|
$yearStats.GetEnumerator() | ForEach-Object {
|
||||||
|
[PSCustomObject]@{ Year = $_.Key; Count = $_.Value }
|
||||||
|
} | Sort-Object Year | Export-Csv "$prefix-mtime-years.csv" -NoTypeInformation -Encoding UTF8
|
||||||
|
|
||||||
|
# --- category rollup for the summary ---
|
||||||
|
$catBytes = @{}; $catCount = @{}
|
||||||
|
foreach ($e in $extStats.GetEnumerator()) {
|
||||||
|
$cat = if ($extToCategory.ContainsKey($e.Key)) { $extToCategory[$e.Key] } else { 'other' }
|
||||||
|
if (-not $catBytes.ContainsKey($cat)) { $catBytes[$cat] = 0L; $catCount[$cat] = 0L }
|
||||||
|
$catBytes[$cat] += $e.Value.Bytes
|
||||||
|
$catCount[$cat] += $e.Value.Count
|
||||||
|
}
|
||||||
|
|
||||||
|
$summaryLines.Add("[$shareName] $root")
|
||||||
|
$summaryLines.Add((" Total: {0:N0} files, {1:N1} GB" -f $totalCount, ($totalBytes / 1GB)))
|
||||||
|
foreach ($c in ($catBytes.Keys | Sort-Object { $catBytes[$_] } -Descending)) {
|
||||||
|
$summaryLines.Add((" {0,-8} {1,10:N0} files {2,10:N1} GB" -f $c, $catCount[$c], ($catBytes[$c] / 1GB)))
|
||||||
|
}
|
||||||
|
if ($pdfSample.Count -gt 0) {
|
||||||
|
$scannedPct = [Math]::Round(100 * $pdfResults['no-text-layer'] / $pdfSample.Count, 1)
|
||||||
|
$summaryLines.Add((" PDFs: {0:N0} total; sample of {1}: {2} with text, {3} WITHOUT text layer (~{4}% -> OCR), {5} encrypted, {6} unreadable" -f `
|
||||||
|
$pdfSeen, $pdfSample.Count, $pdfResults['text'], $pdfResults['no-text-layer'], $scannedPct, $pdfResults['encrypted'], $pdfResults['unreadable']))
|
||||||
|
}
|
||||||
|
$summaryLines.Add((" Duplicate/old-version name patterns: {0:N0} files" -f $dupeCount))
|
||||||
|
$summaryLines.Add((" Paths > 240 chars: {0:N0}; non-ASCII names: {1:N0}" -f $longPaths, $nonAscii))
|
||||||
|
$summaryLines.Add(" Modified: " + (($recency.GetEnumerator() | Sort-Object { @('last 30 days','last 90 days','last 365 days','older').IndexOf($_.Key) } |
|
||||||
|
ForEach-Object { "$($_.Key): $("{0:N0}" -f $_.Value)" }) -join ' | '))
|
||||||
|
$summaryLines.Add("")
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($enumErrors) {
|
||||||
|
$summaryLines.Add(("NOTE: {0:N0} paths could not be read (access denied / path too long). Counts are lower bounds." -f $enumErrors.Count))
|
||||||
|
$enumErrors | ForEach-Object { $_.TargetObject } | Select-Object -First 50 |
|
||||||
|
Set-Content (Join-Path $OutDir "enumeration-errors-sample.txt") -Encoding UTF8
|
||||||
|
}
|
||||||
|
|
||||||
|
$summaryPath = Join-Path $OutDir "summary.txt"
|
||||||
|
$summaryLines | Set-Content $summaryPath -Encoding UTF8
|
||||||
|
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host "Done. Results in: $OutDir" -ForegroundColor Green
|
||||||
|
Get-Content $summaryPath | Write-Host
|
||||||
Reference in New Issue
Block a user