Large Scale XML Sitemaps in PHP: Streaming, Sharding & Index Files

function writeSitemapFile($filePath, $urlGenerator) {
  $writer = new XMLWriter();
  $writer->openUri($filePath);
  $writer->startDocument('1.0', 'UTF-8');
  $writer->startElement('urlset');
  $writer->writeAttribute('xmlns', 'http://www.sitemaps.org/schemas/sitemap/0.9');

  // $urlGenerator is a PHP generator, one row is held in memory at a time
  foreach ($urlGenerator as $row) {
    $writer->startElement('url');
    $writer->writeElement('loc', $row['url']);
    $writer->writeElement('lastmod', $row['updated_at']);
    $writer->endElement(); // 
  }

  $writer->endElement(); // 
  $writer->endDocument();
  $writer->flush();
}

$pdo->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);

function fetchIndexableUrls($pdo) {
  $stmt = $pdo->query(
    'SELECT url, updated_at FROM pages WHERE noindex = 0 ORDER BY id'
  );

  // yield hands over one row and pauses, this is the memory-saving part
  while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
    yield $row;
  }
}

writeSitemapFile('sitemaps/sitemap-1.xml', fetchIndexableUrls($pdo));

function generateShardedSitemaps($urlGenerator, $outputDir, $limitPerFile = 50000) {
  $fileIndex = 1;
  $count = 0;
  $writer = null;
  $generatedFiles = [];

  foreach ($urlGenerator as $row) {
    if ($count % $limitPerFile === 0) {
      if ($writer) {
        $writer->endElement();
        $writer->endDocument();
        $writer->flush();
      }
      $fileName = "sitemap-{$fileIndex}.xml";
      $generatedFiles[] = $fileName;
      $writer = new XMLWriter();
      $writer->openUri($outputDir . '/' . $fileName);
      $writer->startDocument('1.0', 'UTF-8');
      $writer->startElement('urlset');
      $writer->writeAttribute('xmlns', 'http://www.sitemaps.org/schemas/sitemap/0.9');
      $fileIndex++;
    }

    $writer->startElement('url');
    $writer->writeElement('loc', $row['url']);
    $writer->writeElement('lastmod', $row['updated_at']);
    $writer->endElement();

    $count++;
  }

  if ($writer) {
    $writer->endElement();
    $writer->endDocument();
    $writer->flush();
  }

  return $generatedFiles;
}

function writeSitemapIndex($generatedFiles, $baseUrl, $outputPath) {
  $writer = new XMLWriter();
  $writer->openUri($outputPath . '/sitemap-index.xml');
  $writer->startDocument('1.0', 'UTF-8');
  $writer->startElement('sitemapindex');
  $writer->writeAttribute('xmlns', 'http://www.sitemaps.org/schemas/sitemap/0.9');

  foreach ($generatedFiles as $fileName) {
    $writer->startElement('sitemap');
    $writer->writeElement('loc', $baseUrl . '/' . $fileName);
    $writer->writeElement('lastmod', date('c'));
    $writer->endElement();
  }

  $writer->endElement();
  $writer->endDocument();
  $writer->flush();
}

function gzipSitemapFile($filePath) {
  $gzPath = $filePath . '.gz';
  $source = fopen($filePath, 'rb');
  $dest = gzopen($gzPath, 'wb9');

  while (!feof($source)) {
    gzwrite($dest, fread($source, 1048576)); // 1MB chunks
  }

  fclose($source);
  gzclose($dest);
}