// mixin implementing the reify method
const PackageJson = require('@npmcli/package-json')
const hgi = require('hosted-git-info')
const npa = require('npm-package-arg')
const packageContents = require('@npmcli/installed-package-contents')
const pacote = require('pacote')
const { pickRegistry } = require('npm-registry-fetch')
const promiseAllRejectLate = require('promise-all-reject-late')
const runScript = require('@npmcli/run-script')
const { callLimit: promiseCallLimit } = require('promise-call-limit')
const { depth: dfwalk } = require('treeverse')
const { dirname, resolve, relative, join, sep } = require('node:path')
const { log, time } = require('proc-log')
const { existsSync, realpathSync } = require('node:fs')
const { lstat, mkdir, readdir, readlink, rm, symlink } = require('node:fs/promises')
const { moveFile } = require('@npmcli/fs')
const { subset, intersects } = require('semver')
const { walkUp } = require('walk-up-path')

const AuditReport = require('../audit-report.js')
const Diff = require('../diff.js')
const calcDepFlags = require('../calc-dep-flags.js')
const debug = require('../debug.js')
const onExit = require('../signal-handling.js')
const optionalSet = require('../optional-set.js')
const relpath = require('../relpath.js')
const retirePath = require('../retire-path.js')
const treeCheck = require('../tree-check.js')
const Shrinkwrap = require('../shrinkwrap.js')
const { defaultLockfileVersion } = Shrinkwrap
const { saveTypeMap, hasSubKey } = require('../add-rm-pkg-deps.js')
const { IsolatedNode, IsolatedLink } = require('../isolated-classes.js')

// Part of steps (steps need refactoring before we can do anything about these)
const _retireShallowNodes = Symbol.for('retireShallowNodes')
const _loadBundlesAndUpdateTrees = Symbol.for('loadBundlesAndUpdateTrees')
const _submitQuickAudit = Symbol('submitQuickAudit')
const _unpackNewModules = Symbol.for('unpackNewModules')
const _build = Symbol.for('build')

// shared by rebuild mixin
const _trashList = Symbol.for('trashList')
const _handleOptionalFailure = Symbol.for('handleOptionalFailure')
const _loadTrees = Symbol.for('loadTrees')
// defined by rebuild mixin
const _checkBins = Symbol.for('checkBins')

// shared symbols for swapping out when testing
// TODO tests should not be this deep into internals
const _diffTrees = Symbol.for('diffTrees')
const _createSparseTree = Symbol.for('createSparseTree')
const _loadShrinkwrapsAndUpdateTrees = Symbol.for('loadShrinkwrapsAndUpdateTrees')
const _reifyNode = Symbol.for('reifyNode')
const _updateAll = Symbol.for('updateAll')
const _updateNames = Symbol.for('updateNames')
const _moveContents = Symbol.for('moveContents')
const _moveBackRetiredUnchanged = Symbol.for('moveBackRetiredUnchanged')
const _removeTrash = Symbol.for('removeTrash')
const _renamePath = Symbol.for('renamePath')
const _rollbackRetireShallowNodes = Symbol.for('rollbackRetireShallowNodes')
const _rollbackCreateSparseTree = Symbol.for('rollbackCreateSparseTree')
const _rollbackMoveBackRetiredUnchanged = Symbol.for('rollbackMoveBackRetiredUnchanged')
const _saveIdealTree = Symbol.for('saveIdealTree')

// defined by build-ideal-tree mixin
const _resolvedAdd = Symbol.for('resolvedAdd')
// used by build-ideal-tree mixin
const _addNodeToTrashList = Symbol.for('addNodeToTrashList')

module.exports = cls => class Reifier extends cls {
  #bundleMissing = new Set() // child nodes we'd EXPECT to be included in a bundle, but aren't
  #bundleUnpacked = new Set() // the nodes we unpack to read their bundles
  #nmValidated = new Set()
  #omit
  #retiredPaths = {}
  #retiredUnchanged = {}
  #shrinkwrapInflated = new Set()
  #sparseTreeDirs = new Set()
  #sparseTreeRoots = new Set()
  #linkedActualForDiff = null
  // Under the linked strategy the audit runs against this non-isolated ideal tree.
  // The isolated tree's inventory has no queryable indexes and its edges route through symlinks, so auditing it reports no vulnerabilities.
  #linkedIdealForAudit = null

  constructor (options) {
    super(options)

    this[_trashList] = new Set()
  }

  // public method
  async reify (options = {}) {
    // Global installs are normalized to the shallow strategy in the constructor; honor that here so a per-call installStrategy:'linked' can't re-engage the unsupported linked path.
    const linked = !this.options.global &&
      (options.installStrategy || this.options.installStrategy) === 'linked'

    if (this.options.packageLockOnly && this.options.global) {
      const er = new Error('cannot generate lockfile for global packages')
      er.code = 'ESHRINKWRAPGLOBAL'
      throw er
    }

    this.#omit = new Set(options.omit)

    // start tracker block
    this.addTracker('reify')
    const timeEnd = time.start('reify')
    // don't create missing dirs on dry runs
    if (!this.options.packageLockOnly && !this.options.dryRun) {
      // we do NOT want to set ownership on this folder, especially
      // recursively, because it can have other side effects to do that
      // in a project directory.  We just want to make it if it's missing.
      await mkdir(resolve(this.path), { recursive: true })

      // do not allow the top-level node_modules to be a symlink
      await this.#validateNodeModules(resolve(this.path, 'node_modules'))
    }
    await this[_loadTrees](options)

    const oldTree = this.idealTree
    // Kept to serialize the hidden lockfile from the on-disk .store/symlink layout.
    let isolatedTree = null
    if (linked) {
      // swap out the tree with the isolated tree
      // this is currently technical debt which will be resolved in a refactor
      // of Node/Link trees
      this.idealTree = await this.createIsolatedTree()
      isolatedTree = this.idealTree
      if (this.actualTree) {
        this.#linkedActualForDiff = this.#buildLinkedActualForDiff(
          this.idealTree, this.actualTree
        )
      }
      // Keep the non-isolated tree so the quick audit can run against it.
      this.#linkedIdealForAudit = oldTree
    }
    try {
      await this[_diffTrees]()
      await this.#reifyPackages()
      if (linked) {
        // The sweep mutates node_modules on disk, so skip it for dry runs and lockfile-only installs (those modes also short-circuit #reifyPackages).
        // The sweep itself scopes to in-filter workspaces when a filter is active, so it's safe to run for filtered installs too.
        if (!this.options.dryRun && !this.options.packageLockOnly) {
          await this.#cleanOrphanedStoreEntries()
        }
      } else if (!this.options.dryRun && !this.options.packageLockOnly) {
        // The .store directory is exclusively a linked-strategy artifact, and load-actual ignores dot-directories, so the diff never sees it.
        // Remove it here when switching away from linked so it does not linger under hoisted/nested.
        // Only do this for a full-project install: a workspace-filtered or --workspaces=false install may leave out-of-scope workspaces with links still pointing into the store.
        const filtered = this.options.workspaces.length > 0 || !this.options.workspacesEnabled
        if (!filtered) {
          await this.#removeStaleStoreDir()
        }
      }
    } finally {
      // Restore the non-isolated tree so the lockfile is preserved and a reused Arborist never sees the isolated tree, even if reify throws.
      if (linked) {
        this.idealTree = oldTree
      }
      // The quick audit has captured its tree synchronously by now, so drop the stashed references even on throw.
      this.#linkedIdealForAudit = null
      this.#linkedActualForDiff = null
    }
    await this[_saveIdealTree](options)
    // clean inert
    for (const node of this.idealTree.inventory.values()) {
      if (node.inert) {
        node.parent = null
      }
    }
    // clean up any trash that is still in the tree
    for (const path of this[_trashList]) {
      const loc = relpath(this.idealTree.realpath, path)
      const node = this.idealTree.inventory.get(loc)
      if (node && node.root === this.idealTree) {
        node.parent = null
      }
    }

    // if we filtered to only certain nodes, then anything ELSE needs
    // to be untouched in the resulting actual tree, even if it differs
    // in the idealTree.  Copy over anything that was in the actual and
    // was not changed, delete anything in the ideal and not actual.
    // Then we move the entire idealTree over to this.actualTree, and
    // save the hidden lockfile.
    if (this.diff && this.diff.filterSet.size && !linked) {
      const reroot = new Set()

      const { filterSet } = this.diff
      const seen = new Set()
      for (const [loc, ideal] of this.idealTree.inventory.entries()) {
        seen.add(loc)

        // if it's an ideal node from the filter set, then skip it
        // because we already made whatever changes were necessary
        if (filterSet.has(ideal)) {
          continue
        }

        // otherwise, if it's not in the actualTree, then it's not a thing
        // that we actually added.  And if it IS in the actualTree, then
        // it's something that we left untouched, so we need to record
        // that.
        const actual = this.actualTree.inventory.get(loc)
        if (!actual) {
          ideal.root = null
        } else {
          if ([...actual.linksIn].some(link => filterSet.has(link))) {
            seen.add(actual.location)
            continue
          }
          const { realpath, isLink } = actual
          if (isLink && ideal.isLink && ideal.realpath === realpath) {
            continue
          } else {
            reroot.add(actual)
          }
        }
      }

      // now find any actual nodes that may not be present in the ideal
      // tree, but were left behind by virtue of not being in the filter
      for (const [loc, actual] of this.actualTree.inventory.entries()) {
        if (seen.has(loc)) {
          continue
        }
        seen.add(loc)

        // we know that this is something that ISN'T in the idealTree,
        // or else we will have addressed it in the previous loop.
        // If it's in the filterSet, that means we intentionally removed
        // it, so nothing to do here.
        if (filterSet.has(actual)) {
          continue
        }

        reroot.add(actual)
      }

      // go through the rerooted actual nodes, and move them over.
      for (const actual of reroot) {
        actual.root = this.idealTree
      }

      // prune out any tops that lack a linkIn, they are no longer relevant.
      for (const top of this.idealTree.tops) {
        if (top.linksIn.size === 0) {
          top.root = null
        }
      }

      // need to calculate dep flags, since nodes may have been marked
      // as extraneous or otherwise incorrect during transit.
      calcDepFlags(this.idealTree)
    }

    // save the ideal's meta as a hidden lockfile after we actualize it.
    // Under linked the logical tree is the hoisted layout, so the hidden lockfile is serialized from the isolated tree instead.
    if (!linked) {
      this.idealTree.meta.filename =
        this.idealTree.realpath + '/node_modules/.package-lock.json'
      this.idealTree.meta.hiddenLockfile = true
      this.idealTree.meta.lockfileVersion = defaultLockfileVersion
    }

    this.actualTree = this.idealTree
    this.idealTree = null

    if (!this.options.global && !this.options.dryRun) {
      if (linked) {
        await this.#saveLinkedHiddenLockfile(isolatedTree)
      } else {
        await this.actualTree.meta.save()
      }
      const ignoreScripts = !!this.options.ignoreScripts
      // if we aren't doing a dry run or ignoring scripts and we actually made changes to the dep
      // tree, then run the dependencies scripts
      if (!this.options.dryRun && !ignoreScripts && this.diff && this.diff.children.length) {
        const { path, package: pkg } = this.actualTree.target
        const stdio = this.options.foregroundScripts ? 'inherit' : 'pipe'
        const { scripts = {} } = pkg
        for (const event of ['predependencies', 'dependencies', 'postdependencies']) {
          if (Object.prototype.hasOwnProperty.call(scripts, event)) {
            log.info('run', pkg._id, event, scripts[event])
            await time.start(`reify:run:${event}`, () => runScript({
              event,
              path,
              pkg,
              stdio,
              scriptShell: this.options.scriptShell,
            }))
          }
        }
      }
    }
    // This is a very bad pattern and I can't wait to stop doing it
    this.auditReport = await this.auditReport

    this.finishTracker('reify')
    timeEnd()
    return treeCheck(this.actualTree)
  }

  async #reifyPackages () {
    // we don't submit the audit report or write to disk on dry runs
    if (this.options.dryRun) {
      return
    }

    if (this.options.packageLockOnly) {
      // we already have the complete tree, so just audit it now,
      // and that's all we have to do here.
      return this[_submitQuickAudit]()
    }

    // ok, we're about to start touching the fs.  need to roll back
    // if we get an early termination.
    let reifyTerminated = null
    const removeHandler = onExit(({ signal }) => {
      // only call once.  if signal hits twice, we just terminate
      removeHandler()
      reifyTerminated = Object.assign(new Error('process terminated'), {
        signal,
      })
      return false
    })

    // [rollbackfn, [...actions]]
    // after each step, if the process was terminated, execute the rollback
    // note that each rollback *also* calls the previous one when it's
    // finished, and then the first one throws the error, so we only need
    // a new rollback step when we have a new thing that must be done to
    // revert the install.
    const steps = [
      [_rollbackRetireShallowNodes, [
        _retireShallowNodes,
      ]],
      [_rollbackCreateSparseTree, [
        _createSparseTree,
        _loadShrinkwrapsAndUpdateTrees,
        _loadBundlesAndUpdateTrees,
        _submitQuickAudit,
        _unpackNewModules,
      ]],
      [_rollbackMoveBackRetiredUnchanged, [
        _moveBackRetiredUnchanged,
        _build,
      ]],
    ]
    for (const [rollback, actions] of steps) {
      for (const action of actions) {
        try {
          await this[action]()
          if (reifyTerminated) {
            throw reifyTerminated
          }
        } catch (er) {
          // TODO rollbacks shouldn't be relied on to throw err
          await this[rollback](er)
          /* istanbul ignore next - rollback throws, should never hit this */
          throw er
        }
      }
    }

    // no rollback for this one, just exit with the error, since the
    // install completed and can't be safely recovered at this point.
    await this[_removeTrash]()
    if (reifyTerminated) {
      throw reifyTerminated
    }

    // done modifying the file system, no need to keep listening for sigs
    removeHandler()
  }

  // when doing a local install, we load everything and figure it all out.
  // when doing a global install, we *only* care about the explicit requests.
  [_loadTrees] (options) {
    const timeEnd = time.start('reify:loadTrees')
    const bitOpt = {
      ...options,
      complete: this.options.packageLockOnly || this.options.dryRun,
    }

    // if we're only writing a package lock, then it doesn't matter what's here
    if (this.options.packageLockOnly) {
      return this.buildIdealTree(bitOpt).then(timeEnd)
    }

    const actualOpt = this.options.global ? {
      ignoreMissing: true,
      global: true,
      filter: (node, kid) => {
        // if it's not the project root, and we have no explicit requests,
        // then we're already into a nested dep, so we keep it
        if (this.explicitRequests.size === 0 || !node.isProjectRoot) {
          return true
        }

        // if we added it as an edgeOut, then we want it
        if (this.idealTree.edgesOut.has(kid)) {
          return true
        }

        // if it's an explicit request, then we want it
        const hasExplicit = [...this.explicitRequests]
          .some(edge => edge.name === kid)
        if (hasExplicit) {
          return true
        }

        // ignore the rest of the global install folder
        return false
      },
    } : { ignoreMissing: true }

    if (!this.options.global) {
      return Promise.all([
        this.loadActual(actualOpt),
        this.buildIdealTree(bitOpt),
      ]).then(timeEnd)
    }

    // the global install space tends to have a lot of stuff in it.  don't
    // load all of it, just what we care about.  we won't be saving a
    // hidden lockfile in there anyway.  Note that we have to load ideal
    // BEFORE loading actual, so that the actualOpt can use the
    // explicitRequests which is set during buildIdealTree
    return this.buildIdealTree(bitOpt)
      .then(() => this.loadActual(actualOpt))
      .then(timeEnd)
  }

  [_diffTrees] () {
    if (this.options.packageLockOnly) {
      return
    }

    const timeEnd = time.start('reify:diffTrees')
    // XXX if we have an existing diff already, there should be a way
    // to just invalidate the parts that changed, but avoid walking the
    // whole tree again.

    const includeWorkspaces = this.options.workspacesEnabled
    const includeRootDeps = !includeWorkspaces
      || this.options.includeWorkspaceRoot && this.options.workspaces.length > 0

    const filterNodes = []
    if (this.options.global && this.explicitRequests.size) {
      const idealTree = this.idealTree.target
      const actualTree = this.actualTree.target
      // we ONLY are allowed to make changes in the global top-level
      // children where there's an explicit request.
      for (const { name } of this.explicitRequests) {
        const ideal = idealTree.children.get(name)
        if (ideal) {
          filterNodes.push(ideal)
        }
        const actual = actualTree.children.get(name)
        if (actual) {
          filterNodes.push(actual)
        }
      }
    } else {
      if (includeWorkspaces) {
        // add all ws nodes to filterNodes
        for (const ws of this.options.workspaces) {
          const ideal = this.idealTree.children.get(ws)
          if (ideal) {
            filterNodes.push(ideal)
          }
          // Skip actual-side filterNodes when using the linked diff wrapper.
          // Those nodes have root===actualTree, not root===linkedActualForDiff, and Diff.calculate requires filterNode.root to match actual.
          // The ideal filterNode alone is sufficient to scope the workspace diff.
          if (!this.#linkedActualForDiff) {
            const actual = this.actualTree.children.get(ws)
            if (actual) {
              filterNodes.push(actual)
            }
          }
        }
      }
      if (includeRootDeps) {
        // add all non-workspace nodes to filterNodes
        // Skip the actual tree under the linked diff wrapper: its edge targets have root===actualTree, not the wrapper, which trips Diff.calculate's filterNode guard.
        // The ideal-side targets alone scope the diff.
        const trees = this.#linkedActualForDiff ? [this.idealTree] : [this.idealTree, this.actualTree]
        for (const tree of trees) {
          for (const { type, to } of tree.edgesOut.values()) {
            if (type !== 'workspace' && to) {
              filterNodes.push(to)
            }
          }
        }
      }
    }

    // find all the nodes that need to change between the actual
    // and ideal trees.
    this.diff = Diff.calculate({
      omit: this.#omit,
      shrinkwrapInflated: this.#shrinkwrapInflated,
      filterNodes,
      actual: this.#linkedActualForDiff || this.actualTree,
      ideal: this.idealTree,
    })

    // we don't have to add 'removed' folders to the trashlist, because
    // they'll be moved aside to a retirement folder, and then the retired
    // folder will be deleted at the end.  This is important when we have
    // a folder like FOO being "removed" in favor of a folder like "foo",
    // because if we remove node_modules/FOO on case-insensitive systems,
    // it will remove the dep that we *want* at node_modules/foo.

    timeEnd()
  }

  // add the node and all its bins to the list of things to be
  // removed later on in the process.  optionally, also mark them
  // as a retired paths, so that we move them out of the way and
  // replace them when rolling back on failure.
  [_addNodeToTrashList] (node, retire = false) {
    const paths = [node.path, ...node.binPaths]
    const moves = this.#retiredPaths
    log.silly('reify', 'mark', retire ? 'retired' : 'deleted', paths)
    for (const path of paths) {
      if (retire) {
        const retired = retirePath(path)
        moves[path] = retired
        this[_trashList].add(retired)
      } else {
        this[_trashList].add(path)
      }
    }
  }

  // move aside the shallowest nodes in the tree that have to be
  // changed or removed, so that we can rollback if necessary.
  [_retireShallowNodes] () {
    const timeEnd = time.start('reify:retireShallow')
    const moves = this.#retiredPaths = {}
    for (const diff of this.diff.children) {
      if (diff.action === 'CHANGE' || diff.action === 'REMOVE') {
        // we'll have to clean these up at the end, so add them to the list
        this[_addNodeToTrashList](diff.actual, true)
      }
    }
    log.silly('reify', 'moves', moves)
    const movePromises = Object.entries(moves)
      .map(([from, to]) => this[_renamePath](from, to))
    return promiseAllRejectLate(movePromises).then(timeEnd)
  }

  [_renamePath] (from, to, didMkdirp = false) {
    return moveFile(from, to)
      .catch(er => {
        // Occasionally an expected bin file might not exist in the package,
        // or a shim/symlink might have been moved aside.  If we've already
        // handled the most common cause of ENOENT (dir doesn't exist yet),
        // then just ignore any ENOENT.
        if (er.code === 'ENOENT') {
          return didMkdirp ? null : mkdir(dirname(to), { recursive: true }).then(() =>
            this[_renamePath](from, to, true))
        } else if (er.code === 'EEXIST' || er.code === 'ENOTEMPTY') {
          return rm(to, { recursive: true, force: true }).then(() => moveFile(from, to))
        } else {
          throw er
        }
      })
  }

  [_rollbackRetireShallowNodes] (er) {
    const timeEnd = time.start('reify:rollback:retireShallow')
    const moves = this.#retiredPaths
    const movePromises = Object.entries(moves)
      .map(([from, to]) => this[_renamePath](to, from))
    return promiseAllRejectLate(movePromises)
      // ignore subsequent rollback errors
      .catch(() => {})
      .then(timeEnd)
      .then(() => {
        throw er
      })
  }

  [_createSparseTree] () {
    const timeEnd = time.start('reify:createSparse')
    // if we call this fn again, we look for the previous list
    // so that we can avoid making the same directory multiple times
    const leaves = this.diff.leaves
      .filter(diff => {
        return (diff.action === 'ADD' || diff.action === 'CHANGE') &&
          !this.#sparseTreeDirs.has(diff.ideal.path) &&
          !diff.ideal.isLink
      })
      .map(diff => diff.ideal)

    // we check this in parallel, so guard against multiple attempts to
    // retire the same path at the same time.
    const dirsChecked = new Set()
    return promiseAllRejectLate(leaves.map(async node => {
      for (const d of walkUp(node.path)) {
        if (d === node.top.path) {
          break
        }
        if (dirsChecked.has(d)) {
          continue
        }
        dirsChecked.add(d)
        const st = await lstat(d).catch(() => null)
        // this can happen if we have a link to a package with a name
        // that the filesystem treats as if it is the same thing.
        // would be nice to have conditional istanbul ignores here...
        /* istanbul ignore next - defense in depth */
        if (st && !st.isDirectory()) {
          const retired = retirePath(d)
          this.#retiredPaths[d] = retired
          this[_trashList].add(retired)
          await this[_renamePath](d, retired)
        }
      }
      this.#sparseTreeDirs.add(node.path)
      const made = await mkdir(node.path, { recursive: true })
      // if the directory already exists, made will be undefined. if that's the case
      // we don't want to remove it because we aren't the ones who created it so we
      // omit it from the #sparseTreeRoots
      /* istanbul ignore next -- pre-existing: mkdir returns undefined when dir exists, covered in reify tests but lost in aggregate coverage merge */
      if (made) {
        this.#sparseTreeRoots.add(made)
      }
    })).then(timeEnd)
  }

  [_rollbackCreateSparseTree] (er) {
    const timeEnd = time.start('reify:rollback:createSparse')
    // cut the roots of the sparse tree that were created, not the leaves
    const roots = this.#sparseTreeRoots
    // also delete the moves that we retired, so that we can move them back
    const failures = []
    const targets = [...roots, ...Object.keys(this.#retiredPaths)]
    const unlinks = targets
      .map(path => rm(path, { recursive: true, force: true }).catch(er => failures.push([path, er])))
    return promiseAllRejectLate(unlinks).then(() => {
      // eslint-disable-next-line promise/always-return
      if (failures.length) {
        log.warn('cleanup', 'Failed to remove some directories', failures)
      }
    })
      .then(timeEnd)
      .then(() => this[_rollbackRetireShallowNodes](er))
  }

  // shrinkwrap nodes define their dependency branches with a file, so
  // we need to unpack them, read that shrinkwrap file, and then update
  // the tree by calling loadVirtual with the node as the root.
  [_loadShrinkwrapsAndUpdateTrees] () {
    const seen = this.#shrinkwrapInflated
    const shrinkwraps = this.diff.leaves
      .filter(d => (d.action === 'CHANGE' || d.action === 'ADD' || !d.action) &&
        d.ideal.hasShrinkwrap && !seen.has(d.ideal) &&
        !this[_trashList].has(d.ideal.path))

    if (!shrinkwraps.length) {
      return
    }

    const timeEnd = time.start('reify:loadShrinkwraps')

    const Arborist = this.constructor
    return promiseAllRejectLate(shrinkwraps.map(diff => {
      const node = diff.ideal
      seen.add(node)
      return diff.action ? this[_reifyNode](node) : node
    }))
      .then(nodes => promiseAllRejectLate(nodes.map(node => new Arborist({
        ...this.options,
        path: node.path,
      }).loadVirtual({ root: node, subtreeOnly: true }))))
      // reload the diff and sparse tree because the ideal tree changed
      .then(() => this[_diffTrees]())
      .then(() => this[_createSparseTree]())
      .then(() => this[_loadShrinkwrapsAndUpdateTrees]())
      .then(timeEnd)
  }

  // create a symlink for Links, extract for Nodes
  // return the node object, since we usually want that
  // handle optional dep failures here
  // If reifying fails, and the node is optional, add it and its optionalSet
  // to the trash list
  // Always return the node.
  [_reifyNode] (node) {
    const timeEnd = time.start(`reifyNode:${node.location}`)
    this.addTracker('reify', node.name, node.location)

    const p = Promise.resolve().then(async () => {
      await this[_checkBins](node)
      await this.#extractOrLink(node)
      const { _id, deprecated } = node.package
      // The .catch is in _handleOptionalFailure.  Not ideal, this should be cleaned up.
      // eslint-disable-next-line promise/always-return
      if (deprecated) {
        log.warn('deprecated', `${_id}: ${deprecated}`)
      }
    })

    return this[_handleOptionalFailure](node, p)
      .then(() => {
        this.finishTracker('reify', node.name, node.location)
        timeEnd()
        return node
      })
  }

  // do not allow node_modules to be a symlink
  async #validateNodeModules (nm) {
    if (this.options.force || this.#nmValidated.has(nm)) {
      return
    }
    const st = await lstat(nm).catch(() => null)
    if (!st || st.isDirectory()) {
      this.#nmValidated.add(nm)
      return
    }
    log.warn('reify', 'Removing non-directory', nm)
    await rm(nm, { recursive: true, force: true })
  }

  async #extractOrLink (node) {
    const nm = resolve(node.parent.path, 'node_modules')
    await this.#validateNodeModules(nm)

    if (!node.isLink) {
      // in normal cases, node.resolved should *always* be set by now.
      // however, it is possible when a lockfile is damaged, or very old,
      // or in some other race condition bugs in npm v6, that a previously
      // bundled dependency will have just a version, but no resolved value,
      // and no 'bundled: true' setting.
      // Do the best with what we have, or else remove it from the tree
      // entirely, since we can't possibly reify it.
      let res = null
      if (node.resolved) {
        const registryResolved = this.#registryResolved(node.resolved)
        if (registryResolved) {
          res = `${node.name}@${registryResolved}`
        }
      } else if (node.package.name && node.version) {
        res = `${node.package.name}@${node.version}`
      }

      // no idea what this thing is.  remove it from the tree.
      if (!res) {
        const warning = 'invalid or damaged lockfile detected\n' +
          'please re-try this operation once it completes\n' +
          'so that the damage can be corrected, or perform\n' +
          'a fresh install with no lockfile if the problem persists.'
        log.warn('reify', warning)
        log.verbose('reify', 'unrecognized node in tree', node.path)
        node.parent = null
        node.fsParent = null
        this[_addNodeToTrashList](node)
        return
      }
      await debug(async () => {
        const st = await lstat(node.path).catch(() => null)
        if (st && !st.isDirectory()) {
          debug.log('unpacking into a non-directory', node)
          throw Object.assign(new Error('ENOTDIR: not a directory'), {
            code: 'ENOTDIR',
            path: node.path,
          })
        }
      })
      await pacote.extract(res, node.path, {
        ...this.options,
        resolved: node.resolved,
        integrity: node.integrity,
        // A node counts as "root" for allow-* enforcement if it satisfies at least one valid dependency edge declared by the project root or a workspace.
        // node.parent is unsafe here: after hoisting, transitive packages can have the project root as their tree parent.
        // In the linked strategy the store node has no edgesIn, so isolated-reifier precomputes isRootDependency from the source node's edges.
        _isRoot: node.isRootDependency || [...node.edgesIn].some(e =>
          e.valid && (e.from?.isProjectRoot || e.from?.isWorkspace)
        ),
        // pacote's npa re-parses our `name@URL` spec as type=remote, so allowRemote would mis-fire on registry tarballs.
        // Override only when we can prove the URL is registry-mediated; see #isRegistryResolvedTarball.
        ...(this.#isRegistryResolvedTarball(node) ? { allowRemote: 'all' } : {}),
      })
      // store nodes don't use Node class so node.package doesn't get updated
      if (node.isInStore) {
        const { content: pkg } = await PackageJson.normalize(node.path)
        node.package.scripts = pkg.scripts
      }
      return
    }

    // node.isLink

    // Tree-only Link: present in the tree for diff/filter participation, never materialized on disk.
    if (node.isUndeclaredWorkspaceLink) {
      return
    }

    await rm(node.path, { recursive: true, force: true })

    // symlink
    const dir = dirname(node.path)
    const target = node.realpath

    let rel
    if (node.resolved?.startsWith('file:')) {
      rel = this.#calculateRelativePath(node, dir, target, nm)
    } else {
      rel = relative(dir, target)
    }

    await mkdir(dir, { recursive: true })
    return symlink(rel, node.path, 'junction')
  }

  // if the node is optional, then the failure of the promise is nonfatal
  // just add it and its optional set to the trash list.
  [_handleOptionalFailure] (node, p) {
    return (node.optional ? p.catch(() => {
      const set = optionalSet(node)
      for (const node of set) {
        log.verbose('reify', 'failed optional dependency', node.path)
        node.inert = true
        this[_addNodeToTrashList](node)
      }
    }) : p).then(() => node)
  }

  #calculateRelativePath (node, dir, target) {
    // Check if the node is affected by a root override
    let hasRootOverride = [...node.edgesIn].some(edge => edge.from.isRoot && edge.overrides)
    // If not set via edges, see if the root package.json explicitly lists an override
    if (!hasRootOverride && node.root) {
      const rootPackage = node.root.target
      hasRootOverride = !!(rootPackage &&
        rootPackage.package.overrides &&
        rootPackage.package.overrides[node.name])
    }
    if (!hasRootOverride) {
      return relative(dir, target)
    }
    // If an override is detected, attempt to retrieve the override spec from the root package.json
    const overrideSpec = node.root?.target?.package?.overrides?.[node.name]
    if (typeof overrideSpec === 'string' && overrideSpec.startsWith('file:')) {
      const overridePath = overrideSpec.replace(/^file:/, '')
      const rootDir = node.root.target.path
      return relative(dir, resolve(rootDir, overridePath))
    }

    // Fallback: derive the file path from node.resolved in a platform-agnostic way
    const filePath = node.resolved.replace(/^file:/, '')
    return join(filePath)
  }

  // Serialize the hidden lockfile from the isolated tree, which mirrors the on-disk .store/symlink layout.
  // Its children are every materialized node_modules entry: store package dirs and all symlinks.
  async #saveLinkedHiddenLockfile (isolatedTree) {
    const path = isolatedTree.realpath
    const meta = new Shrinkwrap({
      path,
      hiddenLockfile: true,
      lockfileVersion: defaultLockfileVersion,
      resolveOptions: this.options,
    })
    meta.reset()
    meta.filename = resolve(path, 'node_modules/.package-lock.json')
    const storeRe = /^(.*\/\.store\/.+?)\/node_modules\//
    const containers = new Set()
    const nodes = new Set()
    for (const node of isolatedTree.children.values()) {
      // Tree-only undeclared workspace self-links aren't on disk.
      if (node.isUndeclaredWorkspaceLink) {
        continue
      }
      nodes.add(node)
      // Record the enclosing .store/<key> dir so loadVirtual can resolve a store package's sibling deps.
      // node.location uses the platform separator; lockfile keys are posix.
      const m = node.location.replace(/\\/g, '/').match(storeRe)
      if (m) {
        containers.add(m[1])
      }
    }
    // Workspace dirs hold their own dep symlinks; record them so the cache can validate those subtrees.
    for (const ws of isolatedTree.fsChildren) {
      nodes.add(ws)
    }
    for (const node of nodes) {
      meta.add(node)
    }
    for (const loc of containers) {
      meta.data.packages[loc] = {}
    }
    await meta.save()
  }

  // Build a flat actual tree wrapper for linked installs so the diff can correctly match store entries that already exist on disk.
  // The proxy tree from createIsolatedTree() is flat (all children on root), but loadActual() produces a nested tree where store entries are deep link targets.
  // This wrapper surfaces them at the root level for comparison.
  #buildLinkedActualForDiff (idealTree, actualTree) {
    // Combined Map keyed by path (how allChildren() in diff.js keys)
    const combined = new Map()

    // Create synthetic actual entries for ALL ideal children that exist on disk.
    // The isolated ideal tree is flat (all entries as root children), but loadActual() produces a nested tree where workspace deps are under fsChildren and store entries are deep link targets.
    // Synthetic entries ensure the diff compares matching resolved/integrity values (e.g. workspace links have resolved=undefined in the ideal tree but resolved="file:../packages/..." in the actual tree).
    for (const child of idealTree.children.values()) {
      if (combined.has(child.path) || !existsSync(child.path)) {
        continue
      }
      // Skip store links whose ideal realpath doesn't exist on disk yet — the store hash changed and the symlink needs recreating via ADD.
      if (child.isLink && child.resolved?.startsWith('file:.store/') && !existsSync(child.realpath)) {
        continue
      }
      // Skip a link whose on-disk target is a valid-but-wrong store key (e.g. an interrupted update), so the diff repoints it via ADD.
      if (child.isLink && this.#linkTargetMismatch(child)) {
        continue
      }
      let entry
      if (child.isLink) {
        entry = new IsolatedLink(child)
      } else {
        entry = new IsolatedNode(child)
      }
      if (child.isLink && combined.has(child.realpath)) {
        entry.target = combined.get(child.realpath)
      }
      combined.set(child.path, entry)
    }

    // Proxy .get(name) to original actual tree for filterNodes compatibility
    // (scoped workspace installs use .get(name), allChildren uses .values())
    const origGet = actualTree.children.get.bind(actualTree.children)
    const combinedGet = combined.get.bind(combined)
    /* istanbul ignore next -- only reached during scoped workspace installs */
    combined.get = (key) => combinedGet(key) || origGet(key)

    let wrapper
    /* istanbul ignore next - untested! */
    if (actualTree.isLink) {
      wrapper = new IsolatedLink(actualTree)
    } else {
      wrapper = new IsolatedNode(actualTree)
    }
    wrapper.root = wrapper
    wrapper.binPaths = actualTree.binPaths
    wrapper.children = combined
    wrapper.edgesOut = actualTree.edgesOut
    // Use empty fsChildren so that allChildren() only picks up entries from the combined map.
    // The actual fsChildren have real children with different resolved values (e.g. file:../../../node_modules/.store/... vs file:.store/...) that would overwrite our synthetic entries in allChildren().
    wrapper.fsChildren = new Set()
    wrapper.integrity = actualTree.integrity
    wrapper.inventory = actualTree.inventory

    return wrapper
  }

  // True when the link's on-disk target resolves to a different path than its ideal target.
  // The caller only invokes this once both paths exist, so realpathSync won't throw.
  #linkTargetMismatch (child) {
    return realpathSync(child.path) !== realpathSync(child.realpath)
  }

  // When extracting a registry-resolved package, the spec we hand to pacote is name@URL.
  // pacote re-parses that with npa and gets spec.type === 'remote', so without an override the allow-remote gate would fire on every registry tarball (both =none and =root mis-fire).
  // Returns true only when we are confident this is a registry-mediated install.
  #isRegistryResolvedTarball (node) {
    if (!node.resolved || !node.isRegistryDependency) {
      return false
    }
    try {
      // Match the effective fetch URL, not the raw lockfile value.
      // #registryResolved applies replace-registry-host, rewriting a public-registry pin to the configured proxy/mirror so it matches.
      const resolvedURL = new URL(this.#registryResolved(node.resolved))
      // pickRegistry only consults spec.scope, so a bare-name (tag) parse is sufficient and avoids a node.version dependency.
      const registry = new URL(pickRegistry(npa(node.name), this.options))
      const registryPath = registry.pathname.replace(/\/?$/, '/')
      return resolvedURL.origin === registry.origin &&
        (registryPath === '/' || resolvedURL.pathname.startsWith(registryPath))
    } catch {
      return false
    }
  }

  #registryResolved (resolved) {
    // the default registry url is a magic value meaning "the currently
    // configured registry".
    // `resolved` must never be falsey.
    //
    // XXX: use a magic string that isn't also a valid value, like
    // ${REGISTRY} or something.  This has to be threaded through the
    // Shrinkwrap and Node classes carefully, so for now, just treat
    // the default reg as the magical animal that it has been.
    try {
      const resolvedURL = hgi.parseUrl(resolved)
      const registryURL = new URL(this.registry)
      const registryPath = registryURL.pathname.replace(/\/$/, '')

      let matchURL = null
      try {
        matchURL = new URL(this.options.replaceRegistryHost)
      } catch {
        // keep matchURL null
      }

      const matchHost = matchURL?.hostname ?? this.options.replaceRegistryHost
      const matchPath = matchURL?.pathname.replace(/\/$/, '') ?? null
      const hasPathPrefix = (pathname, prefix) =>
        pathname === prefix || pathname.startsWith(`${prefix}/`)

      const hostMatches = this.options.replaceRegistryHost === 'always' || matchHost === resolvedURL.hostname
      const pathMatches = !matchPath || hasPathPrefix(resolvedURL.pathname, matchPath)

      if (!hostMatches || !pathMatches) {
        return resolved
      }

      resolvedURL.protocol = registryURL.protocol
      resolvedURL.hostname = registryURL.hostname
      resolvedURL.port = registryURL.port

      if (matchPath) {
        // full-URL prefix: swap old path prefix for the registry path
        resolvedURL.pathname = registryPath + resolvedURL.pathname.slice(matchPath.length)
      } else if (registryPath && !hasPathPrefix(resolvedURL.pathname, registryPath)) {
        // host-only: prepend registry path if not already present
        resolvedURL.pathname = registryPath + resolvedURL.pathname
      }

      return resolvedURL.toString()
    } catch {
      // if we could not parse the url at all then returning nothing
      // here means it will get removed from the tree in the next step
      return undefined
    }
  }

  // bundles are *sort of* like shrinkwraps, in that the branch is defined
  // by the contents of the package.  however, in their case, rather than
  // shipping a virtual tree that must be reified, they ship an entire
  // reified actual tree that must be unpacked and not modified.
  [_loadBundlesAndUpdateTrees] (depth = 0, bundlesByDepth) {
    let maxBundleDepth
    if (!bundlesByDepth) {
      bundlesByDepth = new Map()
      maxBundleDepth = -1
      dfwalk({
        tree: this.diff,
        visit: diff => {
          const node = diff.ideal
          if (!node) {
            return
          }
          if (node.isProjectRoot) {
            return
          }

          const { bundleDependencies } = node.package
          if (bundleDependencies && bundleDependencies.length) {
            maxBundleDepth = Math.max(maxBundleDepth, node.depth)
            if (!bundlesByDepth.has(node.depth)) {
              bundlesByDepth.set(node.depth, [node])
            } else {
              bundlesByDepth.get(node.depth).push(node)
            }
          }
        },
        getChildren: diff => diff.children,
      })

      bundlesByDepth.set('maxBundleDepth', maxBundleDepth)
    } else {
      maxBundleDepth = bundlesByDepth.get('maxBundleDepth')
    }

    if (depth === 0) {
      time.start('reify:loadBundles')
    }

    if (depth > maxBundleDepth) {
      // if we did something, then prune the tree and update the diffs
      if (maxBundleDepth !== -1) {
        this.#pruneBundledMetadeps(bundlesByDepth)
        this[_diffTrees]()
      }
      time.end('reify:loadBundles')
      return
    }

    // skip any that have since been removed from the tree, eg by a
    // shallower bundle overwriting them with a bundled meta-dep.
    const set = (bundlesByDepth.get(depth) || [])
      .filter(node => node.root === this.idealTree &&
        node.target !== node.root &&
        !this[_trashList].has(node.path))

    if (!set.length) {
      return this[_loadBundlesAndUpdateTrees](depth + 1, bundlesByDepth)
    }

    // extract all the nodes with bundles
    return promiseCallLimit(set.map(node => {
      return () => {
        this.#bundleUnpacked.add(node)
        return this[_reifyNode](node)
      }
    }), { rejectLate: true })
    // then load their unpacked children and move into the ideal tree
      .then(nodes =>
        promiseAllRejectLate(nodes.map(async node => {
          const arb = new this.constructor({
            ...this.options,
            path: node.path,
          })
          const notTransplanted = new Set(node.children.keys())
          await arb.loadActual({
            root: node,
            // don't transplant any sparse folders we created
            // loadActual will set node.package to {} for empty directories
            // if by chance there are some empty folders in the node_modules
            // tree for some other reason, then ok, ignore those too.
            transplantFilter: node => {
              if (node.package._id) {
                // it's actually in the bundle if it gets transplanted
                notTransplanted.delete(node.name)
                return true
              } else {
                return false
              }
            },
          })
          for (const name of notTransplanted) {
            this.#bundleMissing.add(node.children.get(name))
          }
        })))
    // move onto the next level of bundled items
      .then(() => this[_loadBundlesAndUpdateTrees](depth + 1, bundlesByDepth))
  }

  // https://github.com/npm/cli/issues/1597#issuecomment-667639545
  #pruneBundledMetadeps (bundlesByDepth) {
    const bundleShadowed = new Set()

    // Example dep graph:
    // root -> (a, c)
    // a -> BUNDLE(b)
    // b -> c
    // c -> b
    //
    // package tree:
    // root
    // +-- a
    // |   +-- b(1)
    // |   +-- c(1)
    // +-- b(2)
    // +-- c(2)
    // 1. mark everything that's shadowed by anything in the bundle.  This
    //    marks b(2) and c(2).
    // 2. anything with edgesIn from outside the set, mark not-extraneous,
    //    remove from set.  This unmarks c(2).
    // 3. continue until no change
    // 4. remove everything in the set from the tree.  b(2) is pruned

    // create the list of nodes shadowed by children of bundlers
    for (const bundles of bundlesByDepth.values()) {
      // skip the 'maxBundleDepth' item
      if (!Array.isArray(bundles)) {
        continue
      }
      for (const node of bundles) {
        for (const name of node.children.keys()) {
          const shadow = node.parent.resolve(name)
          if (!shadow) {
            continue
          }
          bundleShadowed.add(shadow)
          shadow.extraneous = true
        }
      }
    }

    // lib -> (a@1.x) BUNDLE(a@1.2.3 (b@1.2.3))
    // a@1.2.3 -> (b@1.2.3)
    // a@1.3.0 -> (b@2)
    // b@1.2.3 -> ()
    // b@2 -> (c@2)
    //
    // root
    // +-- lib
    // |   +-- a@1.2.3
    // |   +-- b@1.2.3
    // +-- b@2 <-- shadowed, now extraneous
    // +-- c@2 <-- also shadowed, because only dependent is shadowed
    for (const shadow of bundleShadowed) {
      for (const shadDep of shadow.edgesOut.values()) {
        /* istanbul ignore else - pretty unusual situation, just being
         * defensive here. Would mean that a bundled dep has a dependency
         * that is unmet. which, weird, but if you bundle it, we take
         * whatever you put there and assume the publisher knows best. */
        if (shadDep.to) {
          bundleShadowed.add(shadDep.to)
          shadDep.to.extraneous = true
        }
      }
    }

    let changed
    do {
      changed = false
      for (const shadow of bundleShadowed) {
        for (const edge of shadow.edgesIn) {
          if (!bundleShadowed.has(edge.from)) {
            shadow.extraneous = false
            bundleShadowed.delete(shadow)
            changed = true
            break
          }
        }
      }
    } while (changed)

    for (const shadow of bundleShadowed) {
      this[_addNodeToTrashList](shadow)
      shadow.root = null
    }
  }

  async [_submitQuickAudit] () {
    if (this.options.audit === false) {
      this.auditReport = null
      return
    }

    // we submit the quick audit at this point in the process, as soon as
    // we have all the deps resolved, so that it can overlap with the other
    // actions as much as possible.  Stash the promise, which we resolve
    // before finishing the reify() and returning the tree.  Thus, we do
    // NOT return the promise, as the intent is for this to run in parallel
    // with the reification, and be resolved at a later time.
    const timeEnd = time.start('reify:audit')
    const options = { ...this.options }
    // Under the linked strategy idealTree is the isolated tree, which the audit cannot traverse; audit the non-isolated tree instead.
    const tree = this.#linkedIdealForAudit || this.idealTree

    // if we're operating on a workspace, only audit the workspace deps
    if (this.options.workspaces.length) {
      options.filterSet = this.workspaceDependencySet(
        tree,
        this.options.workspaces,
        this.options.includeWorkspaceRoot
      )
    }

    this.auditReport = AuditReport.load(tree, options).then(res => {
      timeEnd()
      return res
    })
  }

  // ok!  actually unpack stuff into their target locations!
  // The sparse tree has already been created, so we walk the diff
  // kicking off each unpack job.  If any fail, we rm the sparse
  // tree entirely and try to put everything back where it was.
  [_unpackNewModules] () {
    const timeEnd = time.start('reify:unpack')
    const unpacks = []
    dfwalk({
      tree: this.diff,
      visit: diff => {
        // no unpacking if we don't want to change this thing
        if (diff.action !== 'CHANGE' && diff.action !== 'ADD') {
          return
        }

        const node = diff.ideal
        const bd = this.#bundleUnpacked.has(node)
        const sw = this.#shrinkwrapInflated.has(node)
        const bundleMissing = this.#bundleMissing.has(node)

        // check whether we still need to unpack this one.
        // test the inDepBundle last, since that's potentially a tree walk.
        const doUnpack = node && // can't unpack if removed!
          // root node already exists
          !node.isRoot &&
          // already unpacked to read bundle
          !bd &&
          // already unpacked to read sw
          !sw &&
          // already unpacked by another dep's bundle
          (bundleMissing || !node.inDepBundle)

        if (doUnpack) {
          unpacks.push(this[_reifyNode](node))
        }
      },
      getChildren: diff => diff.children,
    })
    return promiseAllRejectLate(unpacks).then(timeEnd)
  }

  // This is the part where we move back the unchanging nodes that were
  // the children of a node that did change.  If this fails, the rollback
  // is a three-step process.  First, we try to move the retired unchanged
  // nodes BACK to their retirement folders, then delete the sparse tree,
  // then move everything out of retirement.
  [_moveBackRetiredUnchanged] () {
    // get a list of all unchanging children of any shallow retired nodes
    // if they are not the ancestor of any node in the diff set, then the
    // directory won't already exist, so just rename it over.
    // This is sort of an inverse diff tree, of all the nodes where
    // the actualTree and idealTree _don't_ differ, starting from the
    // shallowest nodes that we moved aside in the first place.
    const timeEnd = time.start('reify:unretire')
    const moves = this.#retiredPaths
    this.#retiredUnchanged = {}
    return promiseAllRejectLate(this.diff.children.map(diff => {
      // skip if nothing was retired
      if (diff.action !== 'CHANGE' && diff.action !== 'REMOVE') {
        return
      }

      const { path: realFolder } = diff.actual
      const retireFolder = moves[realFolder]
      /* istanbul ignore next - should be impossible */
      debug(() => {
        if (!retireFolder) {
          const er = new Error('trying to un-retire but not retired')
          throw Object.assign(er, {
            realFolder,
            retireFolder,
            actual: diff.actual,
            ideal: diff.ideal,
            action: diff.action,
          })
        }
      })

      this.#retiredUnchanged[retireFolder] = []
      return promiseAllRejectLate(diff.unchanged.map(node => {
        // no need to roll back links, since we'll just delete them anyway
        if (node.isLink) {
          return mkdir(dirname(node.path), { recursive: true, force: true })
            .then(() => this[_reifyNode](node))
        }

        // will have been moved/unpacked along with bundler
        if (node.inDepBundle && !this.#bundleMissing.has(node)) {
          return
        }

        this.#retiredUnchanged[retireFolder].push(node)

        const rel = relative(realFolder, node.path)
        const fromPath = resolve(retireFolder, rel)
        // if it has bundleDependencies, then make node_modules.  otherwise
        // skip it.
        const bd = node.package.bundleDependencies
        const dir = bd && bd.length ? node.path + '/node_modules' : node.path
        return mkdir(dir, { recursive: true }).then(() => this[_moveContents](node, fromPath))
      }))
    })).then(timeEnd)
  }

  // move the contents from the fromPath to the node.path
  [_moveContents] (node, fromPath) {
    return packageContents({
      path: fromPath,
      depth: 1,
      packageJsonCache: new Map([[fromPath + '/package.json', node.package]]),
    }).then(res => promiseAllRejectLate(res.map(path => {
      const rel = relative(fromPath, path)
      const to = resolve(node.path, rel)
      return this[_renamePath](path, to)
    })))
  }

  [_rollbackMoveBackRetiredUnchanged] (er) {
    const moves = this.#retiredPaths
    // flip the mapping around to go back
    const realFolders = new Map(Object.entries(moves).map(([k, v]) => [v, k]))
    const promises = Object.entries(this.#retiredUnchanged)
      .map(([retireFolder, nodes]) => promiseAllRejectLate(nodes.map(node => {
        const realFolder = realFolders.get(retireFolder)
        const rel = relative(realFolder, node.path)
        const fromPath = resolve(retireFolder, rel)
        return this[_moveContents]({ ...node, path: fromPath }, node.path)
      })))
    return promiseAllRejectLate(promises)
      .then(() => this[_rollbackCreateSparseTree](er))
  }

  [_build] () {
    const timeEnd = time.start('reify:build')

    // for all the things being installed, run their appropriate scripts
    // run in tip->root order, so as to be more likely to build a node's
    // deps before attempting to build it itself
    const nodes = []
    dfwalk({
      tree: this.diff,
      leave: diff => {
        if (!diff.ideal.isProjectRoot) {
          nodes.push(diff.ideal)
        }
      },
      // process adds before changes, ignore removals
      getChildren: diff => diff && diff.children,
      filter: diff => diff.action === 'ADD' || diff.action === 'CHANGE',
    })

    // pick up link nodes from the unchanged list as we want to run their
    // scripts in every install despite of having a diff status change
    for (const node of this.diff.unchanged) {
      const tree = node.root.target

      // skip links that only live within node_modules as they are most
      // likely managed by packages we installed, we only want to rebuild
      // unchanged links we directly manage
      const linkedFromRoot = (node.parent === tree && !node.inert) || node.target.fsTop === tree
      if (node.isLink && linkedFromRoot) {
        nodes.push(node)
      }
    }

    return this.rebuild({ nodes, handleOptionalFailure: true }).then(timeEnd)
  }

  // the tree is pretty much built now, so it's cleanup time.
  // remove the retired folders, and any deleted nodes
  // If this fails, there isn't much we can do but tell the user about it.
  // Thankfully, it's pretty unlikely that it'll fail, since rm is a node builtin.
  async [_removeTrash] () {
    const timeEnd = time.start('reify:trash')
    const promises = []
    const failures = []
    const _rm = path => rm(path, { recursive: true, force: true }).catch(er => failures.push([path, er]))

    for (const path of this[_trashList]) {
      promises.push(_rm(path))
    }

    await promiseAllRejectLate(promises)
    if (failures.length) {
      log.warn('cleanup', 'Failed to remove some directories', failures)
    }

    timeEnd()
  }

  // Remove the root .store left behind by a previous linked install when reifying under a non-linked strategy.
  async #removeStaleStoreDir () {
    const storeDir = resolve(this.path, 'node_modules', '.store')
    if (!existsSync(storeDir)) {
      return
    }
    log.silly('reify', 'removing stale .store from a previous linked install')
    await rm(storeDir, { recursive: true, force: true })
      .catch(/* istanbul ignore next -- rm with force rarely fails */
        er => log.warn('cleanup', 'Failed to remove stale .store directory', er))
  }

  // After a linked install, scan node_modules/.store/ and remove any directories that are not referenced by the current ideal tree.
  // Store entries become orphaned when dependencies are updated or removed, because the diff never sees the old store keys.
  // Then sweep the top-level node_modules/ for orphaned symlinks (e.g. an uninstalled dep whose store entry was just removed) so we don't leave dangling links.
  async #cleanOrphanedStoreEntries () {
    const nmDir = resolve(this.path, 'node_modules')
    const storeDir = resolve(nmDir, '.store')

    // Enumerate on-disk store entries as full keys, descending one level into each @scope directory because scoped keys nest as .store/@scope/pkg@version-hash.
    let entries
    try {
      const topLevel = await readdir(storeDir, { withFileTypes: true })
      entries = []
      for (const ent of topLevel) {
        if (ent.name.startsWith('@')) {
          let scoped
          try {
            scoped = await readdir(resolve(storeDir, ent.name))
          } catch {
            /* istanbul ignore next -- readdir of an entry we just listed should not fail */
            continue
          }
          for (const name of scoped) {
            entries.push(`${ent.name}/${name}`)
          }
        } else {
          entries.push(ent.name)
        }
      }
    } catch {
      entries = null
    }

    // Collect valid store keys and valid top-level links per node_modules directory.
    // Store entries have location node_modules/.store/{key}/node_modules/{pkg}.
    // Top-level links have location {prefix}/node_modules/{pkg} or {prefix}/node_modules/@scope/{pkg}, where {prefix} is empty for the root project and the workspace's localLocation for workspace deps.
    // Locations are normalized to forward slashes here because IsolatedNode/IsolatedLink locations are built with path.join, which uses backslashes on Windows.
    const validKeys = new Set()
    const nmDirs = new Map()
    // Valid bin shim names per node_modules dir, collected from each top-level entry's package.bin so the .bin sweep keeps only shims a still-installed package provides.
    const binsByDir = new Map()
    const NM_PREFIX = 'node_modules/'
    const STORE_MARKER = '/.store/'
    for (const child of this.idealTree.children.values()) {
      const loc = child.location.replace(/\\/g, '/')
      if (child.isInStore) {
        // Store location is node_modules/.store/{key}/node_modules/{pkg}.
        // For a scoped package the key is @scope/pkg@version-hash, which spans two path segments, so reconstruct both instead of taking only the scope.
        const parts = loc.split('/')
        const key = parts[2].startsWith('@') ? `${parts[2]}/${parts[3]}` : parts[2]
        validKeys.add(key)
        continue
      }
      // Tree-only Links never exist on disk; skipping them lets the sweep remove any stale self-link left by an older npm version.
      if (child.isLink && child.isUndeclaredWorkspaceLink) {
        continue
      }
      // Real top-level Nodes (e.g. the root's bundled deps) fall through here too, so they are recorded as valid and never swept as stale.
      const nmIdx = loc.lastIndexOf(NM_PREFIX)
      if (nmIdx === -1 || loc.includes(STORE_MARKER)) {
        continue
      }
      const prefix = loc.slice(0, nmIdx)
      const dir = resolve(this.path, prefix, 'node_modules')
      const rest = loc.slice(nmIdx + NM_PREFIX.length)
      let entry
      if (rest.startsWith('@')) {
        const [scope, name] = rest.split('/')
        entry = `${scope}${sep}${name}`
      } else {
        entry = rest.split('/')[0]
      }
      let set = nmDirs.get(dir)
      if (!set) {
        set = new Set()
        nmDirs.set(dir, set)
      }
      set.add(entry)

      // package.bin is normalized to an object keyed by bin name; shim names are unscoped even for scoped packages.
      const bin = child.package?.bin
      if (bin && typeof bin === 'object') {
        let binSet = binsByDir.get(dir)
        if (!binSet) {
          binSet = new Set()
          binsByDir.set(dir, binSet)
        }
        for (const bn of Object.keys(bin)) {
          binSet.add(bn)
        }
      }
    }

    // Determine which node_modules directories to sweep.
    // For an unfiltered install, sweep the project root and every workspace's node_modules even if no top-level links remain (e.g. last dep was just uninstalled).
    // For a filtered install (npm install -w <ws>), restrict the sweep to the in-scope workspaces so out-of-scope workspaces are left untouched, mirroring what the diff would do.
    // When --include-workspace-root is set, the filter scope pulls in root deps too, so the root node_modules is included in the sweep.
    const filteredNames = this.options.workspaces
    const isFiltered = Array.isArray(filteredNames) && filteredNames.length > 0
    if (isFiltered) {
      const allowedDirs = new Set()
      for (const ws of this.idealTree.fsChildren) {
        if (filteredNames.includes(ws.packageName) || filteredNames.includes(ws.name)) {
          allowedDirs.add(resolve(ws.path, 'node_modules'))
        }
      }
      if (this.options.includeWorkspaceRoot) {
        allowedDirs.add(nmDir)
      }
      for (const dir of [...nmDirs.keys()]) {
        if (!allowedDirs.has(dir)) {
          nmDirs.delete(dir)
        }
      }
      for (const dir of allowedDirs) {
        if (!nmDirs.has(dir)) {
          nmDirs.set(dir, new Set())
        }
      }
    } else {
      if (!nmDirs.has(nmDir)) {
        nmDirs.set(nmDir, new Set())
      }
      for (const ws of this.idealTree.fsChildren) {
        const wsNmDir = resolve(ws.path, 'node_modules')
        if (!nmDirs.has(wsNmDir)) {
          nmDirs.set(wsNmDir, new Set())
        }
      }
    }

    if (entries) {
      const orphaned = entries.filter(e => !validKeys.has(e))
      if (orphaned.length) {
        log.silly('reify', 'cleaning orphaned store entries', orphaned)
        await promiseAllRejectLate(
          orphaned.map(e =>
            rm(resolve(storeDir, e), { recursive: true, force: true })
              .catch(/* istanbul ignore next -- rm with force rarely fails */
                er => log.warn('cleanup', `Failed to remove orphaned store entry ${e}`, er))
          )
        )
        // Removing the last scoped orphan under a scope leaves an empty @scope directory behind, so prune any scope directory that is now empty.
        const scopes = new Set(
          orphaned.filter(e => e.startsWith('@')).map(e => e.split('/')[0])
        )
        await promiseAllRejectLate(
          [...scopes].map(async scope => {
            const scopeDir = resolve(storeDir, scope)
            try {
              const remaining = await readdir(scopeDir)
              if (!remaining.length) {
                await rm(scopeDir, { recursive: true, force: true })
              }
            } catch {
              /* istanbul ignore next -- readdir of a scope dir we just listed should not fail */
            }
          })
        )
      }
    }

    for (const [dir, valid] of nmDirs) {
      await this.#cleanOrphanedTopLevelLinks(dir, valid)
      await this.#cleanStaleBinLinks(dir, binsByDir.get(dir))
    }
  }

  // Remove stale bin shims left in node_modules/.bin after an uninstall under linked, where the diff never emits an action to drop them.
  // A shim is stale when no still-linked package provides its name, or when it is a dangling symlink; matching by name handles both POSIX symlinks and Windows .cmd/.ps1 shims.
  async #cleanStaleBinLinks (nmDir, validBins = new Set()) {
    const binDir = resolve(nmDir, '.bin')
    let names
    try {
      names = await readdir(binDir)
    } catch {
      return
    }

    const stale = names.filter(name => {
      const base = name.replace(/\.(cmd|ps1)$/, '')
      return !validBins.has(base) || !existsSync(resolve(binDir, name))
    })

    if (!stale.length) {
      return
    }

    log.silly('reify', 'cleaning stale bin links', stale)
    await promiseAllRejectLate(
      stale.map(name =>
        rm(resolve(binDir, name), { force: true })
          .catch(/* istanbul ignore next -- rm with force rarely fails */
            er => log.warn('cleanup', `Failed to remove stale bin link ${name}`, er))
      )
    )
  }

  // Remove node_modules/ entries that aren't represented in the ideal tree.
  // Run for the project root and each workspace's node_modules.
  // The linked diff path can't see these because #buildLinkedActualForDiff derives the actual tree from the ideal, so removed deps are never compared.
  // Two kinds of stale entry are removed:
  //   - symlinks whose target resolves inside the project root — store links (node_modules/.store/...) and workspace self-links (e.g. node_modules/<ws> -> ../packages/<ws>) that npm itself created.
  //   - real package directories — hoisted-layout deps left behind when switching from the hoisted strategy to linked, where every valid top-level entry is a symlink.
  // Symlinks pointing outside the project (e.g. `npm link foo` without --save targeting the global prefix, or hand-made `ln -s` to an external path) and non-package real directories are preserved.
  async #cleanOrphanedTopLevelLinks (nmDir, validTopLevel) {
    const projectPrefix = resolve(this.path) + sep
    let dirents
    try {
      dirents = await readdir(nmDir, { withFileTypes: true })
    } catch {
      return
    }

    const isOurOrphan = async (linkPath) => {
      let target
      try {
        target = await readlink(linkPath)
      } catch {
        /* istanbul ignore next -- readlink of an entry we just listed as a symlink should not fail */
        return false
      }
      return resolve(dirname(linkPath), target).startsWith(projectPrefix)
    }

    // A real directory is stale only when it is an actual package (has a package.json), so unrelated user directories are never touched.
    const isStaleRealPkg = (dirent, entPath) =>
      dirent.isDirectory() && existsSync(resolve(entPath, 'package.json'))

    const isOrphan = async (dirent, entPath) =>
      (dirent.isSymbolicLink() && await isOurOrphan(entPath)) || isStaleRealPkg(dirent, entPath)

    const orphaned = []
    const scopes = new Set()
    for (const ent of dirents) {
      // skip npm-managed entries (.bin, .store, .package-lock.json, etc)
      if (ent.name.startsWith('.')) {
        continue
      }
      if (ent.name.startsWith('@')) {
        let scoped
        try {
          scoped = await readdir(resolve(nmDir, ent.name), { withFileTypes: true })
        } catch {
          /* istanbul ignore next -- readdir of an entry we just listed should not fail */
          continue
        }
        for (const pkgEnt of scoped) {
          const key = `${ent.name}${sep}${pkgEnt.name}`
          if (!validTopLevel.has(key) && await isOrphan(pkgEnt, resolve(nmDir, key))) {
            orphaned.push(key)
            scopes.add(ent.name)
          }
        }
      } else if (!validTopLevel.has(ent.name) && await isOrphan(ent, resolve(nmDir, ent.name))) {
        orphaned.push(ent.name)
      }
    }

    if (!orphaned.length) {
      return
    }

    log.silly('reify', 'cleaning orphaned top-level entries', orphaned)
    await promiseAllRejectLate(
      orphaned.map(name =>
        rm(resolve(nmDir, name), { recursive: true, force: true })
          .catch(/* istanbul ignore next -- rm with force rarely fails */
            er => log.warn('cleanup', `Failed to remove orphaned entry ${name}`, er))
      )
    )

    // Removing the last package under a scope leaves an empty @scope directory behind, so prune any scope directory that is now empty.
    await promiseAllRejectLate(
      [...scopes].map(async scope => {
        const scopeDir = resolve(nmDir, scope)
        try {
          const remaining = await readdir(scopeDir)
          if (!remaining.length) {
            await rm(scopeDir, { recursive: true, force: true })
          }
        } catch {
          /* istanbul ignore next -- readdir of a scope dir we just listed should not fail */
        }
      })
    )
  }

  // last but not least, we save the ideal tree metadata to the package-lock
  // or shrinkwrap file, and any additions or removals to package.json
  async [_saveIdealTree] (options) {
    // the ideal tree is actualized now, hooray!
    // it still contains all the references to optional nodes that were removed
    // for install failures.  Those still end up in the shrinkwrap, so we
    // save it first, then prune out the optional trash, and then return it.

    const save = !(options.save === false)

    // we check for updates in order to make sure we run save ideal tree
    // even though save=false since we want `npm update` to be able to
    // write to package-lock files by default
    const hasUpdates = this[_updateAll] || this[_updateNames].length

    // we're going to completely skip save ideal tree in case of a global or
    // dry-run install and also if the save option is set to false, EXCEPT for
    // update since the expected behavior for npm7+ is for update to
    // NOT save to package.json, we make that exception since we still want
    // saveIdealTree to be able to write the lockfile by default.
    const saveIdealTree = !(
      (!save && !hasUpdates)
      || this.options.global
      || this.options.dryRun
    )

    if (!saveIdealTree) {
      return false
    }

    const timeEnd = time.start('reify:save')

    const updatedTrees = new Set()
    const updateNodes = nodes => {
      for (const { name, tree: addTree } of nodes) {
        // addTree either the root, or a workspace
        const edge = addTree.edgesOut.get(name)
        const pkg = addTree.package
        const req = npa.resolve(name, edge.spec, addTree.realpath)
        const { rawSpec, subSpec } = req

        const spec = subSpec ? subSpec.rawSpec : rawSpec
        const child = edge.to

        // if we tried to install an optional dep, but it was a version
        // that we couldn't resolve, this MAY be missing.  if we haven't
        // blown up by now, it's because it was not a problem, though, so
        // just move on.
        if (!child || !addTree.isTop) {
          continue
        }

        let newSpec
        // True if the dependency is getting installed from a local file path
        // In this case it is not possible to do the normal version comparisons
        // as the new version will be a file path
        const isLocalDep = req.type === 'directory' || req.type === 'file'
        if (req.registry) {
          const version = child.version
          const prefixRange = version ? this.options.savePrefix + version : '*'
          // if we installed a range, then we save the range specified
          // if it is not a subset of the ^x.y.z.  eg, installing a range
          // of `1.x <1.2.3` will not be saved as `^1.2.0`, because that
          // would allow versions outside the requested range.  Tags and
          // specific versions save with the save-prefix.
          const isRange = (subSpec || req).type === 'range'

          let range = spec
          if (
            !isRange ||
            spec === '*' ||
            subset(prefixRange, spec, { loose: true })
          ) {
            range = prefixRange
          }

          const pname = child.packageName
          const alias = name !== pname
          newSpec = alias ? `npm:${pname}@${range}` : range
        } else if (req.hosted) {
          // save the git+https url if it has auth; otherwise, shortcut
          const h = req.hosted
          const opt = { noCommittish: false }
          if (h.https && h.auth) {
            newSpec = `git+${h.https(opt)}`
          } else {
            newSpec = h.shortcut(opt)
          }
        } else if (isLocalDep) {
          // when finding workspace nodes, make sure that
          // we save them using their version instead of
          // using their relative path
          if (edge.type === 'workspace') {
            const { version } = edge.to.target
            const prefixRange = version ? this.options.savePrefix + version : '*'
            newSpec = prefixRange
          } else {
            // save the relative path in package.json
            // Normally saveSpec is updated with the proper relative
            // path already, but it's possible to specify a full absolute
            // path initially, in which case we can end up with the wrong
            // thing, so just get the ultimate fetchSpec and relativize it.
            const p = req.fetchSpec.replace(/^file:/, '')
            const rel = relpath(addTree.realpath, p)
            newSpec = `file:${rel}`
          }
        } else {
          newSpec = req.saveSpec
        }

        if (options.saveType) {
          const depType = saveTypeMap.get(options.saveType)
          pkg[depType][name] = newSpec
          // PackageJson.normalize will have moved it here if it was in both, if it is empty it will be deleted later
          if (options.saveType === 'prod' && pkg.optionalDependencies) {
            delete pkg.optionalDependencies[name]
          }
        } else {
          if (hasSubKey(pkg, 'dependencies', name)) {
            pkg.dependencies[name] = newSpec
          }

          if (hasSubKey(pkg, 'devDependencies', name)) {
            pkg.devDependencies[name] = newSpec
            // don't update peer or optional if we don't have to
            if (hasSubKey(pkg, 'peerDependencies', name) && (isLocalDep || !intersects(newSpec, pkg.peerDependencies[name]))) {
              pkg.peerDependencies[name] = newSpec
            }

            if (hasSubKey(pkg, 'optionalDependencies', name) && (isLocalDep || !intersects(newSpec, pkg.optionalDependencies[name]))) {
              pkg.optionalDependencies[name] = newSpec
            }
          } else {
            if (hasSubKey(pkg, 'peerDependencies', name)) {
              pkg.peerDependencies[name] = newSpec
            }

            if (hasSubKey(pkg, 'optionalDependencies', name)) {
              pkg.optionalDependencies[name] = newSpec
            }
          }
        }

        updatedTrees.add(addTree)
      }
    }

    // Returns true if any of the edges from this node has a semver
    // range definition that is an exact match to the version installed
    // e.g: should return true if for a given and installed version 1.0.0,
    // range is either =1.0.0 or 1.0.0
    const exactVersion = node => {
      for (const edge of node.edgesIn) {
        try {
          if (subset(edge.spec, node.version)) {
            return false
          }
        } catch {
          // ignore errors
        }
      }
      return true
    }

    // helper that retrieves an array of nodes that were
    // potentially updated during the reify process, in order
    // to limit the number of nodes to check and update, only
    // select nodes from the inventory that are direct deps
    // of a given package.json (project root or a workspace)
    // and in ase of using a list of `names`, restrict nodes
    // to only names that are found in this list
    const retrieveUpdatedNodes = names => {
      const filterDirectDependencies = node =>
        !node.isRoot && node.resolveParent && node.resolveParent.isRoot
        && (!names || names.includes(node.name))
        && exactVersion(node) // skip update for exact ranges

      const directDeps = this.idealTree.inventory
        .filter(filterDirectDependencies)

      // traverses the list of direct dependencies and collect all nodes
      // to be updated, since any of them might have changed during reify
      const nodes = []
      for (const node of directDeps) {
        for (const edgeIn of node.edgesIn) {
          nodes.push({
            name: node.name,
            tree: edgeIn.from.target,
          })
        }
      }
      return nodes
    }

    if (save) {
      // when using update all alongside with save, we'll make
      // sure to refresh every dependency of the root idealTree
      if (this[_updateAll]) {
        const nodes = retrieveUpdatedNodes()
        updateNodes(nodes)
      } else {
        // resolvedAdd is the list of user add requests, but with names added
        // to things like git repos and tarball file/urls.  However, if the
        // user requested 'foo@', and we have a foo@file:../foo, then we should
        // end up saving the spec we actually used, not whatever they gave us.
        if (this[_resolvedAdd].length) {
          updateNodes(this[_resolvedAdd])
        }

        // if updating given dependencies by name, restrict the list of
        // nodes to check to only those currently in _updateNames
        if (this[_updateNames].length) {
          const nodes = retrieveUpdatedNodes(this[_updateNames])
          updateNodes(nodes)
        }

        // grab any from explicitRequests that had deps removed
        for (const { from: tree } of this.explicitRequests) {
          updatedTrees.add(tree)
        }
      }
    }

    if (save) {
      for (const tree of updatedTrees) {
        // refresh the edges so they have the correct specs
        tree.package = tree.package
        const pkgJson = await PackageJson.load(tree.path, { create: true })
        const {
          dependencies = {},
          devDependencies = {},
          optionalDependencies = {},
          peerDependencies = {},
          // bundleDependencies is not required by PackageJson like the other
          // fields here PackageJson also doesn't omit an empty array for this
          // field so defaulting this to an empty array would add that field to
          // every package.json file.
          bundleDependencies,
        } = tree.package

        pkgJson.update({
          dependencies,
          devDependencies,
          optionalDependencies,
          peerDependencies,
          bundleDependencies,
        })
        await pkgJson.save()
      }
    }

    // before now edge specs could be changing, affecting the `requires` field
    // in the package lock, so we hold off saving to the very last action
    if (this.options.usePackageLock) {
      // preserve indentation, if possible
      let format = this.idealTree.package[Symbol.for('indent')]
      if (format === undefined) {
        format = '  '
      }

      // TODO this ignores options.save
      await this.idealTree.meta.save({
        format: (this.options.formatPackageLock && format) ? format
        : this.options.formatPackageLock,
      })
    }

    timeEnd()
    return true
  }
}