infra-phytron/server/phy-z-srv-gpu01/scripts/share-analysis.ps1

<#
.SYNOPSIS
    Read-only analysis of SMB shares as LLM knowledge base (projektplan §2.1).

.DESCRIPTION
    Walks one or more share paths and reports, per share:
      - volume and file count per file type / category (extractable vs. not)
      - size and count per top-level folder (for the include/exclude decision)
      - modification-time distribution (sync frequency, data age)
      - scanned-PDF ratio via sampling (PDFs without a text layer -> OCR need)
      - duplicate/old-version candidates by name patterns
      - long paths (> 240 chars) and non-ASCII names (umlauts etc.)
    Writes CSVs plus a summary.txt into the output directory. Never writes to
    the shares themselves. PowerShell 5.1 compatible; no external modules.

.EXAMPLE
    .\share-analysis.ps1 -Paths '\\fileserver\projekte','\\fileserver\doku'

.EXAMPLE
    .\share-analysis.ps1 -Paths 'D:\shares\projekte' -OutDir C:\temp\analysis -PdfSampleSize 300
#>
param(
    [Parameter(Mandatory = $true)]
    [string[]]$Paths,

    [string]$OutDir = (Join-Path (Get-Location) ("share-analysis-" + (Get-Date -Format "yyyyMMdd-HHmmss"))),

    # PDFs sampled per share for the text-layer check
    [int]$PdfSampleSize = 200
)

$ErrorActionPreference = 'Continue'
New-Item -ItemType Directory -Path $OutDir -Force | Out-Null

# --- classification ---------------------------------------------------------

$categories = @{
    'office'  = @('.pdf', '.doc', '.docx', '.xls', '.xlsx', '.xlsm', '.ppt', '.pptx', '.txt', '.md', '.rtf', '.odt', '.ods', '.odp', '.csv', '.vsd', '.vsdx')
    'email'   = @('.msg', '.eml')
    'image'   = @('.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tif', '.tiff', '.svg', '.heic', '.webp')
    'cad'     = @('.dwg', '.dxf', '.step', '.stp', '.iges', '.igs', '.sldprt', '.sldasm', '.slddrw', '.ipt', '.iam', '.catpart', '.catproduct', '.3ds', '.stl')
    'archive' = @('.zip', '.rar', '.7z', '.tar', '.gz', '.bz2', '.iso')
    'media'   = @('.mp4', '.avi', '.mov', '.wmv', '.mp3', '.wav')
}
$extToCategory = @{}
foreach ($cat in $categories.Keys) {
    foreach ($ext in $categories[$cat]) { $extToCategory[$ext] = $cat }
}

# name patterns that suggest duplicates / old versions
$dupePattern = '(?i)(kopie|copy|backup|_old|_alt\b|\.bak$|~\$|\(\d+\)\s*(\.[^.]+)?$)'

$summaryLines = New-Object System.Collections.Generic.List[string]
$summaryLines.Add("Share analysis $(Get-Date -Format 'yyyy-MM-dd HH:mm') — read-only")
$summaryLines.Add("Paths: $($Paths -join ', ')")
$summaryLines.Add("")

# --- helpers -----------------------------------------------------------------

function Test-PdfTextLayer {
    # Heuristic: a PDF without any /Font reference in its first 4 MB most
    # likely has no text layer (scanned). Also detects encrypted PDFs.
    param([string]$Path)
    try {
        $fs = [System.IO.File]::Open($Path, 'Open', 'Read', 'ReadWrite')
        try {
            $len = [int][Math]::Min($fs.Length, 4MB)
            $buf = New-Object byte[] $len
            [void]$fs.Read($buf, 0, $len)
        } finally { $fs.Close() }
        $text = [System.Text.Encoding]::ASCII.GetString($buf)
        if ($text -match '/Encrypt') { return 'encrypted' }
        if ($text -match '/Font')    { return 'text' }
        return 'no-text-layer'
    } catch {
        return 'unreadable'
    }
}

# --- per-share pass ----------------------------------------------------------

foreach ($root in $Paths) {
    $shareName = ($root.TrimEnd('\') -split '[\\/]')[-1]
    Write-Host "=== Analyzing '$root' ..." -ForegroundColor Cyan

    if (-not (Test-Path -LiteralPath $root)) {
        Write-Warning "Path not found or no access: $root"
        $summaryLines.Add("[$shareName] SKIPPED — path not found or no access: $root")
        continue
    }

    # aggregates (streaming — file objects are not kept in memory)
    $extStats     = @{}   # ext -> @{Count; Bytes}
    $topStats     = @{}   # top-level folder -> @{Count; Bytes}
    $yearStats    = @{}   # mtime year -> count
    $totalCount   = 0L
    $totalBytes   = 0L
    $dupeCount    = 0L
    $longPaths    = 0L
    $nonAscii     = 0L
    $now          = Get-Date
    $recency      = @{ 'last 30 days' = 0L; 'last 90 days' = 0L; 'last 365 days' = 0L; 'older' = 0L }

    # reservoir sample of PDF paths for the text-layer check
    $pdfSample = New-Object System.Collections.Generic.List[string]
    $pdfSeen   = 0L
    $rand      = New-Object System.Random
    $rootLen   = $root.TrimEnd('\').Length

    Get-ChildItem -LiteralPath $root -Recurse -File -Force -ErrorAction SilentlyContinue -ErrorVariable +enumErrors |
        ForEach-Object {
            $totalCount++
            $totalBytes += $_.Length

            $ext = $_.Extension.ToLowerInvariant()
            if (-not $ext) { $ext = '(none)' }
            if (-not $extStats.ContainsKey($ext)) { $extStats[$ext] = @{ Count = 0L; Bytes = 0L } }
            $extStats[$ext].Count++
            $extStats[$ext].Bytes += $_.Length

            # top-level folder relative to the share root
            $rel = $_.FullName.Substring($rootLen).TrimStart('\')
            $top = if ($rel.Contains('\')) { $rel.Split('\')[0] } else { '(root)' }
            if (-not $topStats.ContainsKey($top)) { $topStats[$top] = @{ Count = 0L; Bytes = 0L } }
            $topStats[$top].Count++
            $topStats[$top].Bytes += $_.Length

            $year = $_.LastWriteTime.Year
            if (-not $yearStats.ContainsKey($year)) { $yearStats[$year] = 0L }
            $yearStats[$year]++

            $age = ($now - $_.LastWriteTime).TotalDays
            if     ($age -le 30)  { $recency['last 30 days']++ }
            elseif ($age -le 90)  { $recency['last 90 days']++ }
            elseif ($age -le 365) { $recency['last 365 days']++ }
            else                  { $recency['older']++ }

            if ($_.Name -match $dupePattern)      { $dupeCount++ }
            if ($_.FullName.Length -gt 240)       { $longPaths++ }
            if ($_.Name -match '[^\x00-\x7F]') { $nonAscii++ }

            if ($ext -eq '.pdf') {
                $pdfSeen++
                if ($pdfSample.Count -lt $PdfSampleSize) {
                    $pdfSample.Add($_.FullName)
                } else {
                    $i = $rand.Next(0, [int][Math]::Min($pdfSeen, [int]::MaxValue))
                    if ($i -lt $PdfSampleSize) { $pdfSample[$i] = $_.FullName }
                }
            }

            if ($totalCount % 20000 -eq 0) {
                Write-Host ("  {0:N0} files, {1:N1} GB ..." -f $totalCount, ($totalBytes / 1GB))
            }
        }

    # PDF text-layer sampling
    Write-Host "  Sampling $($pdfSample.Count) of $pdfSeen PDFs for text layer ..."
    $pdfResults = @{ 'text' = 0; 'no-text-layer' = 0; 'encrypted' = 0; 'unreadable' = 0 }
    foreach ($p in $pdfSample) { $pdfResults[(Test-PdfTextLayer $p)]++ }

    # --- write per-share CSVs ---
    $prefix = Join-Path $OutDir $shareName

    $extStats.GetEnumerator() | ForEach-Object {
        $cat = if ($extToCategory.ContainsKey($_.Key)) { $extToCategory[$_.Key] } else { 'other' }
        [PSCustomObject]@{ Extension = $_.Key; Category = $cat; Count = $_.Value.Count; GB = [Math]::Round($_.Value.Bytes / 1GB, 2) }
    } | Sort-Object GB -Descending | Export-Csv "$prefix-file-types.csv" -NoTypeInformation -Encoding UTF8

    $topStats.GetEnumerator() | ForEach-Object {
        [PSCustomObject]@{ Folder = $_.Key; Count = $_.Value.Count; GB = [Math]::Round($_.Value.Bytes / 1GB, 2) }
    } | Sort-Object GB -Descending | Export-Csv "$prefix-toplevel-folders.csv" -NoTypeInformation -Encoding UTF8

    $yearStats.GetEnumerator() | ForEach-Object {
        [PSCustomObject]@{ Year = $_.Key; Count = $_.Value }
    } | Sort-Object Year | Export-Csv "$prefix-mtime-years.csv" -NoTypeInformation -Encoding UTF8

    # --- category rollup for the summary ---
    $catBytes = @{}; $catCount = @{}
    foreach ($e in $extStats.GetEnumerator()) {
        $cat = if ($extToCategory.ContainsKey($e.Key)) { $extToCategory[$e.Key] } else { 'other' }
        if (-not $catBytes.ContainsKey($cat)) { $catBytes[$cat] = 0L; $catCount[$cat] = 0L }
        $catBytes[$cat] += $e.Value.Bytes
        $catCount[$cat] += $e.Value.Count
    }

    $summaryLines.Add("[$shareName] $root")
    $summaryLines.Add(("  Total: {0:N0} files, {1:N1} GB" -f $totalCount, ($totalBytes / 1GB)))
    foreach ($c in ($catBytes.Keys | Sort-Object { $catBytes[$_] } -Descending)) {
        $summaryLines.Add(("    {0,-8} {1,10:N0} files {2,10:N1} GB" -f $c, $catCount[$c], ($catBytes[$c] / 1GB)))
    }
    if ($pdfSample.Count -gt 0) {
        $scannedPct = [Math]::Round(100 * $pdfResults['no-text-layer'] / $pdfSample.Count, 1)
        $summaryLines.Add(("  PDFs: {0:N0} total; sample of {1}: {2} with text, {3} WITHOUT text layer (~{4}% -> OCR), {5} encrypted, {6} unreadable" -f `
            $pdfSeen, $pdfSample.Count, $pdfResults['text'], $pdfResults['no-text-layer'], $scannedPct, $pdfResults['encrypted'], $pdfResults['unreadable']))
    }
    $summaryLines.Add(("  Duplicate/old-version name patterns: {0:N0} files" -f $dupeCount))
    $summaryLines.Add(("  Paths > 240 chars: {0:N0}; non-ASCII names: {1:N0}" -f $longPaths, $nonAscii))
    $summaryLines.Add("  Modified: " + (($recency.GetEnumerator() | Sort-Object { @('last 30 days','last 90 days','last 365 days','older').IndexOf($_.Key) } |
        ForEach-Object { "$($_.Key): $("{0:N0}" -f $_.Value)" }) -join ' | '))
    $summaryLines.Add("")
}

if ($enumErrors) {
    $summaryLines.Add(("NOTE: {0:N0} paths could not be read (access denied / path too long). Counts are lower bounds." -f $enumErrors.Count))
    $enumErrors | ForEach-Object { $_.TargetObject } | Select-Object -First 50 |
        Set-Content (Join-Path $OutDir "enumeration-errors-sample.txt") -Encoding UTF8
}

$summaryPath = Join-Path $OutDir "summary.txt"
$summaryLines | Set-Content $summaryPath -Encoding UTF8

Write-Host ""
Write-Host "Done. Results in: $OutDir" -ForegroundColor Green
Get-Content $summaryPath | Write-Host