verify.js

'use strict'

const util = require('util')

const pMap = require('p-map')
const contentPath = require('./content/path')
const fixOwner = require('./util/fix-owner')
const fs = require('fs')
const fsm = require('fs-minipass')
const glob = util.promisify(require('glob'))
const index = require('./entry-index')
const path = require('path')
const rimraf = util.promisify(require('rimraf'))
const ssri = require('ssri')

const hasOwnProperty = (obj, key) =>
  Object.prototype.hasOwnProperty.call(obj, key)

const stat = util.promisify(fs.stat)
const truncate = util.promisify(fs.truncate)
const writeFile = util.promisify(fs.writeFile)
const readFile = util.promisify(fs.readFile)

const verifyOpts = (opts) => ({
  concurrency: 20,
  log: { silly () {} },
  ...opts,
})

module.exports = verify

function verify (cache, opts) {
  opts = verifyOpts(opts)
  opts.log.silly('verify', 'verifying cache at', cache)

  const steps = [
    markStartTime,
    fixPerms,
    garbageCollect,
    rebuildIndex,
    cleanTmp,
    writeVerifile,
    markEndTime,
  ]

  return steps
    .reduce((promise, step, i) => {
      const label = step.name
      const start = new Date()
      return promise.then((stats) => {
        return step(cache, opts).then((s) => {
          s &&
            Object.keys(s).forEach((k) => {
              stats[k] = s[k]
            })
          const end = new Date()
          if (!stats.runTime)
            stats.runTime = {}

          stats.runTime[label] = end - start
          return Promise.resolve(stats)
        })
      })
    }, Promise.resolve({}))
    .then((stats) => {
      stats.runTime.total = stats.endTime - stats.startTime
      opts.log.silly(
        'verify',
        'verification finished for',
        cache,
        'in',
        `${stats.runTime.total}ms`
      )
      return stats
    })
}

function markStartTime (cache, opts) {
  return Promise.resolve({ startTime: new Date() })
}

function markEndTime (cache, opts) {
  return Promise.resolve({ endTime: new Date() })
}

function fixPerms (cache, opts) {
  opts.log.silly('verify', 'fixing cache permissions')
  return fixOwner
    .mkdirfix(cache, cache)
    .then(() => {
      // TODO - fix file permissions too
      return fixOwner.chownr(cache, cache)
    })
    .then(() => null)
}

// Implements a naive mark-and-sweep tracing garbage collector.
//
// The algorithm is basically as follows:
// 1. Read (and filter) all index entries ("pointers")
// 2. Mark each integrity value as "live"
// 3. Read entire filesystem tree in `content-vX/` dir
// 4. If content is live, verify its checksum and delete it if it fails
// 5. If content is not marked as live, rimraf it.
//
function garbageCollect (cache, opts) {
  opts.log.silly('verify', 'garbage collecting content')
  const indexStream = index.lsStream(cache)
  const liveContent = new Set()
  indexStream.on('data', (entry) => {
    if (opts.filter && !opts.filter(entry))
      return

    liveContent.add(entry.integrity.toString())
  })
  return new Promise((resolve, reject) => {
    indexStream.on('end', resolve).on('error', reject)
  }).then(() => {
    const contentDir = contentPath.contentDir(cache)
    return glob(path.join(contentDir, '**'), {
      follow: false,
      nodir: true,
      nosort: true,
    }).then((files) => {
      return Promise.resolve({
        verifiedContent: 0,
        reclaimedCount: 0,
        reclaimedSize: 0,
        badContentCount: 0,
        keptSize: 0,
      }).then((stats) =>
        pMap(
          files,
          (f) => {
            const split = f.split(/[/\\]/)
            const digest = split.slice(split.length - 3).join('')
            const algo = split[split.length - 4]
            const integrity = ssri.fromHex(digest, algo)
            if (liveContent.has(integrity.toString())) {
              return verifyContent(f, integrity).then((info) => {
                if (!info.valid) {
                  stats.reclaimedCount++
                  stats.badContentCount++
                  stats.reclaimedSize += info.size
                } else {
                  stats.verifiedContent++
                  stats.keptSize += info.size
                }
                return stats
              })
            } else {
              // No entries refer to this content. We can delete.
              stats.reclaimedCount++
              return stat(f).then((s) => {
                return rimraf(f).then(() => {
                  stats.reclaimedSize += s.size
                  return stats
                })
              })
            }
          },
          { concurrency: opts.concurrency }
        ).then(() => stats)
      )
    })
  })
}

function verifyContent (filepath, sri) {
  return stat(filepath)
    .then((s) => {
      const contentInfo = {
        size: s.size,
        valid: true,
      }
      return ssri
        .checkStream(new fsm.ReadStream(filepath), sri)
        .catch((err) => {
          if (err.code !== 'EINTEGRITY')
            throw err

          return rimraf(filepath).then(() => {
            contentInfo.valid = false
          })
        })
        .then(() => contentInfo)
    })
    .catch((err) => {
      if (err.code === 'ENOENT')
        return { size: 0, valid: false }

      throw err
    })
}

function rebuildIndex (cache, opts) {
  opts.log.silly('verify', 'rebuilding index')
  return index.ls(cache).then((entries) => {
    const stats = {
      missingContent: 0,
      rejectedEntries: 0,
      totalEntries: 0,
    }
    const buckets = {}
    for (const k in entries) {
      /* istanbul ignore else */
      if (hasOwnProperty(entries, k)) {
        const hashed = index.hashKey(k)
        const entry = entries[k]
        const excluded = opts.filter && !opts.filter(entry)
        excluded && stats.rejectedEntries++
        if (buckets[hashed] && !excluded)
          buckets[hashed].push(entry)
        else if (buckets[hashed] && excluded) {
          // skip
        } else if (excluded) {
          buckets[hashed] = []
          buckets[hashed]._path = index.bucketPath(cache, k)
        } else {
          buckets[hashed] = [entry]
          buckets[hashed]._path = index.bucketPath(cache, k)
        }
      }
    }
    return pMap(
      Object.keys(buckets),
      (key) => {
        return rebuildBucket(cache, buckets[key], stats, opts)
      },
      { concurrency: opts.concurrency }
    ).then(() => stats)
  })
}

function rebuildBucket (cache, bucket, stats, opts) {
  return truncate(bucket._path).then(() => {
    // This needs to be serialized because cacache explicitly
    // lets very racy bucket conflicts clobber each other.
    return bucket.reduce((promise, entry) => {
      return promise.then(() => {
        const content = contentPath(cache, entry.integrity)
        return stat(content)
          .then(() => {
            return index
              .insert(cache, entry.key, entry.integrity, {
                metadata: entry.metadata,
                size: entry.size,
              })
              .then(() => {
                stats.totalEntries++
              })
          })
          .catch((err) => {
            if (err.code === 'ENOENT') {
              stats.rejectedEntries++
              stats.missingContent++
              return
            }
            throw err
          })
      })
    }, Promise.resolve())
  })
}

function cleanTmp (cache, opts) {
  opts.log.silly('verify', 'cleaning tmp directory')
  return rimraf(path.join(cache, 'tmp'))
}

function writeVerifile (cache, opts) {
  const verifile = path.join(cache, '_lastverified')
  opts.log.silly('verify', 'writing verifile to ' + verifile)
  try {
    return writeFile(verifile, '' + +new Date())
  } finally {
    fixOwner.chownr.sync(cache, verifile)
  }
}

module.exports.lastRun = lastRun

function lastRun (cache) {
  return readFile(path.join(cache, '_lastverified'), 'utf8').then(
    (data) => new Date(+data)
  )
}