Ramsess/fix release pipeline (#3593)

* add explicit working dir to intall tools step

* add registry flag

* derive nuget feed uri in script

* remove use of powershell-yaml

* replace rush update purge

* set npm userconfig for all npm subprocessess

* use npmrc for subsequent npm auth

* ensure deps downloaded

* add logging to determine failure

* resolve autorest modelerfour

* ensure autorest doesnt reach for npm registry

* ensure autorest is electing cached files

* fix args

* retrieve nuget without blocked call

* pick up auth from npmrc file

* move from global to local build

* Address PR review: harden network isolation in release pipeline

- Add --no-install to all npx calls (rush, autorest) to prevent silent
  registry downloads when tools are already installed locally
- Add --no-package-lock to npm install commands to avoid polluting the
  working tree with package-lock.json that could be accidentally committed
- Gate autorest/npm diagnostics behind ENABLE_AUTOREST_DIAGNOSTICS env var
  to avoid npm calls in network-isolated release pipelines
- Harden NuGet feed URL derivation with input normalization and validation
  instead of fragile regex replacement

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* consolidate scripts and function

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

Author: 3 people

.azure-pipelines/common-templates/install-tools.yml

@@ -48,23 +48,46 @@ steps:
     inputs:
       workingFile: $(Build.SourcesDirectory)/autorest.powershell/common/config/rush/.npmrc
 
+  - task: PowerShell@2
+    displayName: Apply npm auth config to user profile
+    inputs:
+      targetType: inline
+      pwsh: true
+      script: |
+        # Copy authenticated .npmrc to the user home dir so ALL npm subprocesses
+        # (including autorest's internal npm calls) use the private feed + auth tokens.
+        $src = "$(Build.SourcesDirectory)/.npmrc"
+        $dst = Join-Path $env:USERPROFILE ".npmrc"
+        Copy-Item -Path $src -Destination $dst -Force
+        Write-Host "Copied npm config to $dst"
+
   - task: Npm@1
     displayName: Install AutoRest
     inputs:
       command: custom
-      customCommand: install -g autorest@3.7.2
-    
+      workingDir: $(Build.SourcesDirectory)
+      customCommand: install --no-package-lock autorest@3.7.2
+
   - task: Npm@1
     displayName: Install AutorestCore
     inputs:
       command: custom
-      customCommand: install -g @autorest/core@3.10.4
+      workingDir: $(Build.SourcesDirectory)
+      customCommand: install --no-package-lock @autorest/core@3.10.4
+
+  - task: PowerShell@2
+    displayName: Pre-populate autorest extension cache
+    inputs:
+      targetType: filePath
+      pwsh: true
+      filePath: $(Build.SourcesDirectory)/tools/utilities/PrePopulateAutorestCache.ps1
 
   - task: Npm@1
     displayName: Install Rush
     inputs:
       command: custom
-      customCommand: install -g @microsoft/rush
+      workingDir: $(Build.SourcesDirectory)
+      customCommand: install --no-package-lock @microsoft/rush
 
   - task: PowerShell@2
     displayName: Rush Build
@@ -73,6 +96,9 @@ steps:
       pwsh: true
       workingDirectory: "autorest.powershell"
       script: |
-        rush install
-        rush link
-        rush rebuild
+        npx --no-install rush install
+        if ($LASTEXITCODE -ne 0) { throw "rush install failed with exit code $LASTEXITCODE" }
+        npx --no-install rush link
+        if ($LASTEXITCODE -ne 0) { throw "rush link failed with exit code $LASTEXITCODE" }
+        npx --no-install rush rebuild
+        if ($LASTEXITCODE -ne 0) { throw "rush rebuild failed with exit code $LASTEXITCODE" }

tools/Configure-PrivateNpmFeed.ps1

@@ -35,3 +35,24 @@ Write-Host "Created $rootNpmrc"
 $rushNpmrc = Join-Path $SourcesDirectory "autorest.powershell/common/config/rush/.npmrc"
 Set-Content -Path $rushNpmrc -Value $npmrcContent -NoNewline
 Write-Host "Updated $rushNpmrc"
+
+# Create NuGet.config to redirect dotnet restore to the private feed
+# Normalize the registry URL and validate the expected suffix before transforming
+$normalizedRegistry = $Registry.TrimEnd('/')
+$npmSuffix = "/npm/registry"
+if (-not $normalizedRegistry.EndsWith($npmSuffix)) {
+    throw "Cannot derive NuGet feed URL: Registry '$Registry' does not end with '$npmSuffix'. Pass an explicit Azure Artifacts npm registry URL using the -Registry parameter."
+}
+$nugetFeed = $normalizedRegistry.Substring(0, $normalizedRegistry.Length - $npmSuffix.Length) + "/nuget/v3/index.json"
+$nugetConfig = @"
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="PowerShell_V2_Build" value="$nugetFeed" />
+  </packageSources>
+</configuration>
+"@
+$nugetConfigPath = Join-Path $SourcesDirectory "NuGet.config"
+Set-Content -Path $nugetConfigPath -Value $nugetConfig -NoNewline
+Write-Host "Created $nugetConfigPath"

tools/GenerateModules.ps1

@@ -32,10 +32,6 @@ if (-not $Isolated) {
 # Module import.
 Import-Module PowerShellGet
 
-# Install Powershell-yaml
-if (!(Get-Module -Name powershell-yaml -ListAvailable)) {
-    Install-Module powershell-yaml -Repository PSGallery -Scope CurrentUser -Force
-}
 
 $ScriptRoot = $PSScriptRoot
 $ModulesSrc = Join-Path $ScriptRoot "..\src\"
@@ -52,8 +48,50 @@ if (-not (Test-Path $ModuleMappingPath)) {
 
 # Build AutoREST.PowerShell submodule.
 Set-Location (Join-Path $ScriptRoot "../autorest.powershell")
-rush update --purge
-rush build
+npx --no-install rush install
+if ($LASTEXITCODE -ne 0) {
+    throw "Command 'npx --no-install rush install' failed with exit code $LASTEXITCODE."
+}
+
+npx --no-install rush build
+if ($LASTEXITCODE -ne 0) {
+    throw "Command 'npx --no-install rush build' failed with exit code $LASTEXITCODE."
+}
+
+# Diagnostic: show autorest cache state and npm registry config before generation.
+# Gated behind ENABLE_AUTOREST_DIAGNOSTICS to avoid npm calls in network-isolated release pipelines.
+if ($env:ENABLE_AUTOREST_DIAGNOSTICS) {
+    Write-Host "--- Autorest/npm diagnostics ---"
+    $autorestHome = if ($env:AUTOREST_HOME) { $env:AUTOREST_HOME } else { Join-Path $env:USERPROFILE ".autorest" }
+    Write-Host "AUTOREST_HOME: $autorestHome"
+    $coreCacheDir = Join-Path $autorestHome "@autorest\core"
+    if (Test-Path $coreCacheDir) {
+        Write-Host "Autorest core cache versions: $(Get-ChildItem $coreCacheDir -Directory | Select-Object -ExpandProperty Name)"
+        $nodeModulesDir = Join-Path $coreCacheDir "3.10.4\node_modules\@autorest\core"
+        Write-Host "Cache 3.10.4 node_modules present: $(Test-Path $nodeModulesDir)"
+    } else {
+        Write-Host "WARNING: Autorest core cache directory not found at $coreCacheDir"
+    }
+    $modelerfourCacheDir = Join-Path $autorestHome "@autorest\modelerfour"
+    if (Test-Path $modelerfourCacheDir) {
+        Write-Host "Autorest modelerfour cache versions: $(Get-ChildItem $modelerfourCacheDir -Directory | Select-Object -ExpandProperty Name)"
+        $modelerfourNodeModules = Join-Path $modelerfourCacheDir "4.24.3\node_modules\@autorest\modelerfour"
+        Write-Host "Cache modelerfour 4.24.3 node_modules present: $(Test-Path $modelerfourNodeModules)"
+    } else {
+        Write-Host "WARNING: Autorest modelerfour cache directory not found at $modelerfourCacheDir"
+    }
+    Write-Host "npm registry: $(npm config get registry 2>&1)"
+    Write-Host "npm_config_registry env: $env:npm_config_registry"
+    Write-Host "NPM_CONFIG_USERCONFIG: $env:NPM_CONFIG_USERCONFIG"
+    $userNpmrc = Join-Path $env:USERPROFILE ".npmrc"
+    Write-Host "~/.npmrc exists: $(Test-Path $userNpmrc)"
+    if (Test-Path $userNpmrc) {
+        Write-Host "~/.npmrc registry line: $(Select-String -Path $userNpmrc -Pattern '^registry=' | Select-Object -First 1 -ExpandProperty Line)"
+    }
+    Write-Host "--- End diagnostics ---"
+} else {
+    Write-Host "Autorest diagnostics skipped (set ENABLE_AUTOREST_DIAGNOSTICS=1 to enable)"
+}
 
 $RequiredGraphModules = @()
 $AuthModuleManifest = Join-Path $ModulesSrc "Authentication" "Authentication" "artifacts" "Microsoft.Graph.Authentication.psd1"

tools/GenerateServiceModule.ps1

@@ -69,10 +69,33 @@ $ApiVersion | ForEach-Object {
             else {
                 $FullModuleVersion = $ModuleMetadata.versions[$CurrentApiVersion].version
             }
-            npx autorest --max-memory-size=$MaxMemorySize --module-version:$FullModuleVersion --module-name:$ModuleFullName --service-name:$Module --input-file:$OpenApiFile $AutoRestModuleConfig --max-cpu=2 --network-calls=2
-            if ($LastExitCode -ne 0) {
-                Write-Host -ForegroundColor Red "AutoREST failed to generate '$ModuleFullName' module."
-                return $LastExitCode
+            # Pass @autorest/modelerfour as a local --use:<path> argument so autorest loads it
+            # directly from the pre-populated cache without calling fetchPackageMetadata.
+            # fetchPackageMetadata is called unconditionally before the cache check inside
+            # ExtensionManager.findPackage, and it hits the npm registry which is DNS-blocked
+            # by 1ES supply-chain security on release pipelines. A local --use path bypasses
+            # findPackage entirely: the extension goes straight into localExtensions, and when
+            # use-extension in autorest-configuration.md is processed, resolveExtension finds
+            # it there without any registry call.
+            $autorestHome = if ($env:AUTOREST_HOME) { $env:AUTOREST_HOME } else { Join-Path $env:USERPROFILE ".autorest" }
+            $modelerFourPath = Join-Path $autorestHome "@autorest" "modelerfour" "4.24.3" "node_modules" "@autorest" "modelerfour"
+            # @(if ...) always produces [object[]], preventing PowerShell from unwrapping the
+            # single-element array to a [string] scalar. A scalar string splatted with @
+            # enumerates IEnumerable<char>, passing each character as a separate argument.
+            $modelerFourUseFlag = @(if (Test-Path $modelerFourPath) { "--use:$modelerFourPath" })
+            if ($modelerFourUseFlag.Count -eq 0) {
+                Write-Host -ForegroundColor Yellow "WARNING: @autorest/modelerfour local cache not found at $modelerFourPath — autorest will attempt npm registry lookup (may fail in network-isolated environment)"
+            }
+
+            $autorestLog = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), "autorest-$($ModuleFullName -replace '[^a-zA-Z0-9-]', '-').log")
+            npx --no-install autorest @modelerFourUseFlag --verbose --max-memory-size=$MaxMemorySize --module-version:$FullModuleVersion --module-name:$ModuleFullName --service-name:$Module --input-file:$OpenApiFile $AutoRestModuleConfig --max-cpu=2 --network-calls=2 2>&1 | Out-File -FilePath $autorestLog -Encoding utf8
+            $autorestExitCode = $LASTEXITCODE
+            if ($autorestExitCode -ne 0) {
+                Write-Host -ForegroundColor Red "AutoREST failed (exit $autorestExitCode) generating '$ModuleFullName'."
+                Write-Host -ForegroundColor Yellow "=== AutoREST log: $autorestLog ==="
+                if (Test-Path $autorestLog) { Get-Content $autorestLog | ForEach-Object { Write-Host $_ } }
+                Write-Host -ForegroundColor Yellow "=== End AutoREST log ==="
+                return $autorestExitCode
             }
             Write-Debug "AutoRest generated '$ModuleFullName' successfully."
 

tools/ManageGeneratedModule.ps1

@@ -47,7 +47,7 @@ foreach ($Package in $NugetPackagesToRemove) {
 # Add nuget packages from generate modules.
 foreach ($Package in $NugetPackagesToAdd) {
     Write-Debug "Executing: dotnet add $ModuleCsProj package $Package"
-    dotnet add $ModuleCsProj package $Package -s https://api.nuget.org/v3/index.json | Out-Null
+    dotnet add $ModuleCsProj package $Package | Out-Null
     if ($LastExitCode) {
         Write-Error "Failed to execute: dotnet add $ModuleCsProj package $Package"
     }

tools/ReadModuleReadMe.ps1

@@ -6,12 +6,14 @@ param(
     [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] $FieldToRead
 ) 
 $ErrorActionPreference = "Stop"
+. "$PSScriptRoot\utilities\utils.ps1"
+
 $FieldValue = $null
 # Read readme.md.
 $ReadMeContent = Get-Content $ReadMePath -Delimiter "### Versioning"
 if ($ReadMeContent.Length -eq 2) {
     # Convert versioning section to yaml.
-    $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-Yaml
+    $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-SimpleYaml
     $FieldValue = $VersioningSection[$FieldToRead]
 }
 return $FieldValue

tools/Utilities/PrePopulateAutorestCache.ps1

@@ -0,0 +1,42 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+<#
+.SYNOPSIS
+    Pre-populates the autorest extension cache so that autorest makes zero npm
+    network calls during module generation.
+
+.DESCRIPTION
+    Autorest resolves extensions from ~/.autorest/<pkg>/<version>/node_modules/.
+    This script pre-installs every extension referenced by
+    autorest-configuration.md into that cache layout.
+    Registry and auth are read from ~/.npmrc (copied from the authenticated
+    .npmrc earlier in the pipeline).
+#>
+
+[CmdletBinding()]
+param()
+
+$extensions = @(
+    "@autorest/core@3.10.4",
+    "@autorest/modelerfour@4.24.3"
+)
+
+foreach ($ext in $extensions) {
+    $parts   = $ext -split '@(?=[^@]+$)'   # split on last @
+    $pkg     = $parts[0]
+    $ver     = $parts[1]
+    $cacheDir = Join-Path $env:USERPROFILE ".autorest\$($pkg.Replace('/','\'))\$ver"
+    $nodeModules = Join-Path $cacheDir "node_modules\$($pkg.Replace('/','\'))"
+
+    if (Test-Path $nodeModules) {
+        Write-Host "Cache already present: $ext"
+        continue
+    }
+
+    New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null
+    Write-Host "Pre-installing $ext into $cacheDir"
+    npm install $ext --prefix $cacheDir
+    if ($LASTEXITCODE -ne 0) { throw "Failed to pre-install $ext (exit $LASTEXITCODE)" }
+    Write-Host "Done: $ext"
+}

tools/Utilities/utils.ps1

@@ -24,4 +24,18 @@ Function Get-LocalCertificate {
         $global:DefaultCertificate = $pfxCertificate
     }
     return $global:DefaultCertificate
+}
+
+<#
+    Converts a simple single-level YAML string into a hashtable.
+#>
+Function ConvertFrom-SimpleYaml {
+    param([string]$Yaml)
+    $result = @{}
+    $Yaml -split "`n" | ForEach-Object {
+        if ($_.Trim() -match '^([^:]+):\s*(.*)$') {
+            $result[$Matches[1].Trim()] = $Matches[2].Trim()
+        }
+    }
+    return $result
 }

tools/WriteToModuleReadMe.ps1

@@ -7,14 +7,18 @@ param(
     [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] $NewFieldValue
 ) 
 $ErrorActionPreference = "Stop"
+. "$PSScriptRoot\utilities\utils.ps1"
+
 # Read readme.md.
 $ReadMeContent = Get-Content $ReadMePath -Delimiter "### Versioning"
 if ($ReadMeContent.Length -eq 2) {
     # Convert versioning section to yaml.
     $UpdatedVersionSection = "### Versioning" + $ReadMeContent[1]
-    $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-Yaml
+    $VersioningSection = $ReadMeContent[1].Replace("``", "").Replace("yaml", "") | ConvertFrom-SimpleYaml
     $FieldValue = $VersioningSection[$FieldName]
-    $RegexPattern = "$FieldName`:\s*$FieldValue"
+    $EscapedFieldName = [regex]::Escape($FieldName)
+    $EscapedFieldValue = [regex]::Escape($FieldValue)
+    $RegexPattern = "$EscapedFieldName`:\s*$EscapedFieldValue"
     $UpdatedVersionSection = $UpdatedVersionSection -replace $RegexPattern, "$FieldName`: $NewFieldValue"
 
     $ReadMeContent[0] = $ReadMeContent[0].Trim()