Yuba/Yuba.php

<?php

// Yuba
//                                      //
//////////////////////////////////////////
$version = "0.7.5";

ini_set('memory_limit', '4096M');
date_default_timezone_set("America/Los_Angeles");
$time_start = microtime(true);

// Includes & Prefs
//////////////////////////////////////////

require("functions.php");
require("filetypes.php");

$wopt_noprofile = 1;

$p = unserialize(file_get_contents("prefs.php"));

// Path & application variables
//////////////////////////////////////////

$stamp = date("Y-m-d_H-i-s", time());

if (!isset($argv[1])) { echo "Input error"; die; }
$zpath = realpath(@$argv[1]);
if (@$argv[2]) { $bdest = realpath($argv[2]); } else { $bdest = realpath($p['bdest']); }
if (!is_dir($zpath) | !is_dir($bdest)) { echo "Filepath error"; die; }

// Check for bundle
if ($zpath == "/") { $blabel = "root"; } else { $blabel = preg_replace("/[^A-Za-z0-9\.]/", "_", basename($zpath)); }
if (is_writable($zpath)) { echo "Warning: source is writeable\n"; }

$bpath = chop($bdest,"/")."/".substr(crc32($zpath),0,3)."_".$blabel.".bundle";
if (!is_dir($bpath)) { mkdir($bpath); }
if (!is_dir($bpath."/thumbs")) { mkdir($bpath."/thumbs"); }

// Treat these directories as files
$p['bundles'] = array(	"app",
						"bundle", 
						"sparsebundle", 
						"photoslibrary", 
						"aplibrary", 
						"apvault", 
						"abbu", 
						"calendar",
						"framework",
						"plugin",
						"kext", 
						"rtfd"
						);
							
foreach ($p['bundles'] as $bundle) {
	$p['nodescend'][] = "*.".$bundle;
	}

// Ignore matching files and directories
$p['ignore'] = array(		".DS_Store",
							".DocumentRevisions-V100", 
							".Spotlight-V100",
							".TemporaryItems",
							".apdisk",
							".com.apple.timemachine.donotpresent",
							".fseventsd",
							".metadata-never-index",
							".neofinder.abemeda.volinfo.xml"
							);

// Metadata tools
$bin_gfi = __DIR__."/bin/GetFileInfo";
$bin_mediainfo = __DIR__."/bin/mediainfo";
$bin_exiftool = __DIR__."/bin/exiftool";
$bin_ffmpeg = __DIR__."/bin/ffmpeg";
$bin_qlthumb = __DIR__."/bin/ql-thumbnail";

// Logfile

$error_log_file = $bpath."/".$stamp."_error.log";
error_reporting(E_ALL);
ini_set("display_errors", TRUE);
ini_set("log_errors", TRUE);
ini_set("error_log", $error_log_file);

// Banner
//////////////////////////////////////////

$banner = "Yuba: ".$zpath." -> ".$bpath;
echo $banner."\n".str_repeat("-", strlen($banner))."\n";

// Disk info
//////////////////////////////////////////

echo "Gathering system info...\n";

$host = gethostname();
$disks = shell_exec("diskutil list 2>&1");
$df = shell_exec("df 2>&1");
$df_volume = shell_exec("df ".$zpath." | tail -n 1 | rev | cut -d' ' -f1 | rev");
$df_device = shell_exec("df ".$zpath." | tail -n 1 | cut -d' ' -f1");
$mdutil = shell_exec("mdutil -s ".$df_volume);
if (strpos($mdutil,"disabled")) {
	echo "Warning: spotlight indexing is disabled\n";
	$p['spotlight'] = false;
	}

if (substr($zpath, 0, 9) != "/Volumes/") {
	$zbase = "/";
	} else {
	$zparts = explode("/", $zpath);
	$zbase = "/Volumes/".$zparts[2];
	}

$diskutil = shell_exec("diskutil info ".$zbase." 2>&1");
$getstats = array(	"Volume Name", 
					"Protocol", 
					"Volume UUID", 
					"Device Location", 
					"Volume Total Space",
					"Volume Available Space",
					"Level Type"
					);
foreach ($getstats as $stat) {
	preg_match("/(".$stat.":)(.*)(\n)/",$diskutil,$matches);
	if (isset($matches[2])) {
		if (substr($stat, -5, 5) == "Space") {
			$pieces = explode(" ", trim($matches[2]));
			$summary = $pieces[0]." ".$pieces[1];
			$stats[$stat] = $summary;
			} else {
			$stats[$stat] = trim($matches[2]);
			}
		}
	}

$dstats = serialize($stats);
if ($zpath == "/") {
	$type = "Startup disk";
	} elseif (strtolower($zpath) == strtolower("/Volumes/".$stats["Volume Name"])) {
	if ($stats["Protocol"] == "Disk Image") {
		$type = "Disk image";
		} else {
		$type = "External disk";
		}
	} else {
	$type = "Folder";
	}

if ($wopt_noprofile) {
	$profile = "disabled";
	} else {
	$profile = shell_exec("system_profiler SPHardwareDataType SPStorageDataType SPThunderboltDataType SPUSBDataType 2>&1");
	}
$qlmanage = shell_exec("qlmanage -m 2>&1");
$sysvers = shell_exec("sw_vers 2>&1");

// Database
//////////////////////////////////////////

echo "Building database...\n";

$dbo = new PDO("sqlite:".$bpath."/".$stamp.".sqlite3");
$dbo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);

/*
$dbo->query("PRAGMA page_size = 4096");
$dbo->query("PRAGMA cache_size = 10000");
$dbo->query("PRAGMA locking_mode = EXCLUSIVE");
$dbo->query("PRAGMA synchronous = NORMAL");
$dbo->query("PRAGMA journal_mode = WAL");
*/

$dbo->exec("CREATE TABLE _skim (
				version TEXT, 
				opts TEXT, 
				host TEXT, 
				uid INTEGER, 
				zpath TEXT, 
				bpath TEXT, 
				type TEXT, 
				passed_file INTEGER, 
				passed_dir INTEGER, 
				passed_link INTEGER,
				passed_total INTEGER,  
				nodescended INTEGER, 
				ignored INTEGER, 
				dupes INTEGER, 
				stats TEXT, 
				qlmanage TEXT, 
				sysvers TEXT, 
				diskutil TEXT, 
				disks TEXT, 
				df TEXT,
				df_device TEXT,
				df_volume TEXT,
				mdutil TEXT,
				profile TEXT, 
				status TEXT
				)");

$dbo->exec("CREATE TABLE family (
				pid TEXT, 
				fid TEXT, 
				children TEXT
				)");

$dbo->exec("CREATE TABLE files (
				pid TEXT, 
				fid TEXT, 
				Pathname TEXT,
				Path TEXT,
				Filename TEXT,
				Extension TEXT,
				Type TEXT,
				Size INTEGER,
				Inode INTEGER,
				Perms INTEGER,
				Owner TEXT,
				ATime INTEGER,
				MTime INTEGER,
				CTime INTEGER,
				LinkTarget TEXT,
				RealPath TEXT,
				stat TEXT,
				items INTEGER,
				newest INTEGER,
				gfi_type TEXT,
				gfi_attr TEXT,
				gfi_created TEXT,
				has_exif INTEGER,
				has_mediainfo INTEGER,
				has_hash INTEGER,
				thumb_filename TEXT,
				thumb_width INTEGER,
				thumb_height INTEGER,
				has_contents INTEGER,
				contents_filename TEXT
				)");

$stmt = $dbo->prepare("INSERT INTO _skim VALUES (:version, :opts, :host, :uid, :zpath, :bpath, :type, :passed_file, :passed_dir, :passed_link, :passed_total, :nodescended, :ignored, :dupes, :stats, :qlmanage, :sysvers, :diskutil, :disks, :df, :df_device, :df_volume, :mdutil, :profile, :status)");
$stmt->BindValue(":version",$version);
$stmt->BindValue(":opts",serialize($p));
$stmt->BindValue(":host",$host);
$stmt->BindValue(":uid",posix_getuid());
$stmt->BindValue(":zpath",$zpath);
$stmt->BindValue(":bpath",$bpath);
$stmt->BindValue(":type",$type);
$stmt->BindValue(":stats",$dstats);
$stmt->BindValue(":qlmanage",$qlmanage);
$stmt->BindValue(":sysvers",$sysvers);
$stmt->BindValue(":diskutil",$diskutil);
$stmt->BindValue(":disks",$disks);
$stmt->BindValue(":df",$df);
$stmt->BindValue(":df_device",$df_device);
$stmt->BindValue(":df_volume",$df_volume);
$stmt->BindValue(":mdutil",$mdutil);
$stmt->BindValue(":profile",$profile);
$stmt->BindValue(":status","aborted");
$stmt->execute();

// Iterator
//////////////////////////////////////////

$first_run = 1;
$passed_file = $passed_dir = $passed_link = $passed_total = $nodescended = $ignored = 0;
$files = new RecursiveIteratorIterator(
	new RecursiveCallbackFilterIterator(
		new RecursiveDirectoryIterator(
      		$zpath,
			RecursiveDirectoryIterator::SKIP_DOTS
			),
		function ($current, $key, $iterator) use ($p) {
			global $nodescended, $ignored, $passed_file, $passed_dir, $passed_link, $passed_total, $first_run;
			$clean = true;
			// identify ignore files
			if (is_array($p['ignore'])) {
				foreach ($p['ignore'] as $wildcard) {
					if (fnmatch($wildcard, $current->getFilename())) {
						$clean = false;
						if ($first_run) { $ignored++; }
						}
					}
				}
			// identify nodescend dirs
			if (is_array($p['nodescend'])) {
				foreach ($p['nodescend'] as $wildcard) {
					if (fnmatch($wildcard, $current->getPath())) {
						$clean = false;
						if ($first_run) { $nodescended++; }
						}
					}
				}
			//tally stats
			if ($clean && $first_run) {
				if ($current->getType() == "file") {
					$passed_file++;
					} elseif ($current->getType() == "dir") {
					$passed_dir++;
					} elseif ($current->getType() == "link") {
					$passed_link++;
					}
				$passed_total++;
				}
			return $clean;
			}
		),
	RecursiveIteratorIterator::SELF_FIRST,
	RecursiveIteratorIterator::CATCH_GET_CHILD
	);

// Tally
//////////////////////////////////////////

echo "Counting files...\n";

foreach ($files as $null) { }
$first_run = 0;

if (!$passed_total) {
	echo "Nothing was found, exiting";
	die;
	}

echo "Total files: ".$passed_total."\n";

// Pool DB
//////////////////////////////////////////

$dbp = new PDO("sqlite:".$bpath."/pool.sqlite3");
$dbp->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$dbp->exec("CREATE TABLE IF NOT EXISTS md5 (fid TEXT, hash TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS exiftool (fid TEXT, tags TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS mediainfo (fid TEXT, info TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS thumbs (fid TEXT, created INTEGER, relative_path TEXT, width INTEGER, height INTEGER, tool TEXT)");
$dbp->exec("CREATE TABLE IF NOT EXISTS contents (fid TEXT, created INTEGER, relative_path TEXT)");

// Prescan
//////////////////////////////////////////

$i = 0;
$family = array();
$fids = array();
$noread = array();

echo ProgressBar::start($passed_total,"Prescan");

foreach ($files as $splFileInfo) {

	$path = $splFileInfo->getPath();
	$pathname = $splFileInfo->getPathname();
	$shellpath = escapeshellarg($splFileInfo->getPathname());
	$realpath = $splFileInfo->getRealPath();

	$key = md5($pathname);
	$pkey = md5($path);

	if (array_key_exists($key, $family)) {
		echo "Duplicate key on ".$pathname."\n"; die;
		}
	
	$family[$key] = array();
	
	// Path-agnostic Unique File ID (to prevent redundant hashes and thumbs)

	if ($splFileInfo->getType() != "dir" && $splFileInfo->getType() != "link") {

		$fid = md5($splFileInfo->getSize().$splFileInfo->getMtime().$splFileInfo->getBasename());
		$dx[$fid][] = $pathname;
		$fx[] = array($fid, $pathname);
		$family[$key]['fid'] = $fid;
		
		}

	// Cache stat values in stx array

	if ($splFileInfo->getType() != "link") {
		$stx[$i] = array(	$splFileInfo->getATime(),
							$splFileInfo->getMTime(),
							$splFileInfo->getCTime()	);
		}
	
	if ($p['bypass_stat']) {
		$sty[$i] = "bypass";
		} else {
		$sty[$i] = chop(shell_exec("stat -x ".$shellpath." 2>&1"));
		}
	
	// Check file can be read
	
	if ($realpath && !is_readable($realpath)) {
		$noread[] = $realpath;
		}

	// Children
	
	//$family[$pkey]['children'][] = $key;
	$family[$pkey]['children'][] = $i+1;
	
	echo ProgressBar::next();
	$i++;
	
	}

echo ProgressBar::finish();

// Thow permissions error

if (count($noread)) {
	"Current user (".posix_getuid().") does not have read access to the following files:";
	foreach ($noread as $file) {
		echo $file."\n";
		}
	if ($p['readability']) {
		echo "Exiting...";
		die;
		}
	}

// Debug record of duplicate FIDs

$dupes = array_filter($dx, function($a) { return count($a) > 1; });
$dxo = var_export($dupes, true);

if (strlen($dxo)) {
	file_put_contents($bpath."/".$stamp."_dupes.txt",$dxo);
	$dupecount = count($dupes,COUNT_RECURSIVE) - count($dupes);
	echo floor(($dupecount/$passed_total)*100)." percent of files look like duplicates\n";
	}

// Write family to DB

$message = "Writing family to DB: ";
$message .= $passed_file." files, ";
$message .= $passed_dir." dirs, ";
$message .= $nodescended." bundles, ";
$message .= $passed_link." links, ";
$message .= $ignored." ignored, ";
$message .= ($dupecount ? $dupecount : 0)." dupes";

echo ProgressBar::start(count($family),$message);

foreach ($family as $key => $item) {

	$stmt = $dbo->prepare("INSERT INTO family VALUES (:pid, :fid, :children)");
	$stmt->BindValue(":pid",$key);
	if (@$item['fid']) {
		$stmt->BindValue(":fid",$item['fid']);
		}
	if (@$item['children'] && is_array(@$item['children'])) {
		$stmt->BindValue(":children",serialize($item['children']));
		}
	$stmt->execute();

	echo ProgressBar::next();

	}

echo ProgressBar::finish();

unset($dx, $dxo, $dupes);

// stats
$stmt = "UPDATE _skim SET ";
$stmt .= "passed_file=".$passed_file.", ";
$stmt .= "passed_dir=".$passed_dir.", ";
$stmt .= "passed_link=".$passed_link.", ";
$stmt .= "passed_total=".$passed_total.", ";
$stmt .= "nodescended=".$nodescended.", ";
$stmt .= "ignored=".$ignored.", ";
$stmt .= "dupes=".($dupecount ? $dupecount : 0);
$dbo->exec($stmt);

// Contents
//////////////////////////////////////////

if ($p['contents']) {

	echo "DO CONTENTS HERE\n";
	// make a dir in the bundle called contents (similar to db)
	// match files smaller than x and with file extension of txt etc
	// copy files to hash dirs in bundle (like db dir)
	
	}

// Thumbnails
//////////////////////////////////////////

if ($p['thumbs']) {

	echo ProgressBar::start(count($fx),"Generating thumbnails");
	
	foreach ($fx as $array) {
		
		$fid = $array[0];
		$pathname = $array[1];
		$ext = pathinfo($pathname,PATHINFO_EXTENSION);
		$tpath = $bpath."/thumbs/".substr($fid, 0, 2);
		if (!is_dir($tpath)) { mkdir($tpath); }
		$tfile = $tpath."/".$fid.".jpg";
		
		// HACK for ql-thumbnail bug
		$t_skip = array("emlx");
		if (count($t_skip) && in_array($ext, $t_skip)) {
			echo ProgressBar::next("Skipping ".shortlabel($pathname));
			continue;
			}
		
		// if no thumb file, then poll database
		if (file_exists($tfile)) {
			echo ProgressBar::next("Thumb file found for ".shortlabel($pathname));
			continue;
			} elseif ($dbp->query("SELECT EXISTS(SELECT 1 FROM thumbs WHERE fid='".$fid."')")->fetch()[0]) {
			echo ProgressBar::next("Thumb record found for ".shortlabel($pathname));
			continue;
			} else {
			echo ProgressBar::next("Generating thumb for ".shortlabel($pathname));
			}
			
		$stmt = $dbp->prepare("INSERT INTO thumbs VALUES (:fid, :created, :relative_path, :width, :height, :tool)");
		$stmt->BindValue(":fid",$fid);
		$stmt->BindValue(":created",time());
		
		$shellpath = escapeshellarg($pathname);
			
		// first try to make a thumb with external tools
		$cmd = null;
		if (in_array($ext, $t_files['sips'])) {
			//$cmd = $bin_tv." ".$shellpath." -o ".$tfile."[Q=90,optimize_coding] --size=".$p['thumb_size'];
			$cmd = "sips -s format jpeg -s formatOptions 80 --resampleHeightWidthMax ".$p['thumb_size']." ".$shellpath." --out ".$tfile;
			$stmt->BindValue(":tool","sips");
			} elseif (in_array($ext, $t_files['ffmpeg'])) {
			//$cmd = $bin_tf." -i ".$shellpath." -o ".$tfile." -s ".$p['thumb_size']." -c jpg -q 8.5";
			$cmd = $bin_ffmpeg." -ss $(( $(".$bin_mediainfo." --Inform='Video;%Duration%' ".$shellpath.") / 10000 )) -i ".$shellpath." -vframes 1 -filter:v scale='400:-1' -q:v 3 ".$tfile;
			$stmt->BindValue(":tool","ffmpeg");
			}
			
		if ($cmd) { shell_exec($cmd." 2>&1"); }
		
		// if those tools failed, try quicklook
		if (!@filesize($tfile)) {
			//$cmd = $bin_qlthumb." ".$shellpath." ".$tfile." public.jpeg-2000 ".$p['thumb_size']." ".$p['thumb_size']." .8";
			$cmd = $bin_qlthumb." ".$shellpath." ".$tfile." public.jpeg ".$p['thumb_size']." ".$p['thumb_size']." .8";
			shell_exec($cmd." 2>&1");
			$stmt->BindValue(":tool","quicklook");
			}
		
		// success, move thumb into the bundle
		if (file_exists($tfile) && @filesize($tfile)) {
			$stmt->BindValue(":relative_path",substr($tfile, strlen($bpath)));
			list($width, $height) = getimagesize($tfile);
			$stmt->BindValue(":width",$width);
			$stmt->BindValue(":height",$height);
			}
	
		$stmt->execute();
	
		}
	
	echo ProgressBar::finish();

	}

// External metadata
//////////////////////////////////////////

if ($p['meta']) {

	echo ProgressBar::start(count($fx),"Collecting external metadata...");

	foreach ($fx as $array) {

		$fid = $array[0];
		$pathname = $array[1];
		$shellpath = escapeshellarg($pathname);
		$ext = pathinfo($pathname,PATHINFO_EXTENSION);
		$found = 1;
	
		if (!in_array($ext, $e_files) && !in_array($ext, $m_files)) {
			echo ProgressBar::next("Not a media file: ".shortlabel($pathname));
			continue;
			}
	
		if (in_array($ext, $e_files)) {
			$check = $dbp->query("SELECT EXISTS(SELECT 1 FROM exiftool WHERE fid='".$fid."')")->fetch()[0];
			if (!$check) {
				$rawexif = eval("return ".`$bin_exiftool -php $shellpath`);
				$stmt = $dbp->prepare("INSERT INTO exiftool VALUES (:fid, :tags)");
				$stmt->BindValue(":fid",$fid);
				$stmt->BindValue(":tags",serialize($rawexif[0]));
				$stmt->execute();
				$found = 0;
				}
			}
	
		if (in_array($ext, $m_files)) {
			$check = $dbp->query("SELECT EXISTS(SELECT 1 FROM mediainfo WHERE fid='".$fid."')")->fetch()[0];
			if (!$check) {
				$stmt = $dbp->prepare("INSERT INTO mediainfo VALUES (:fid, :info)");
				$stmt->BindValue(":fid",$fid);
				//$stmt->BindValue(":info",serialize(parseMediaInfo(shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1"))));
				$stmt->BindValue(":info",shell_exec($bin_mediainfo." --Output=OLDXML ".$shellpath." 2>&1"));
				$stmt->execute();
				$found = 0;
				}
			}
			
		if ($found) {
			echo ProgressBar::next("Metadata found: ".shortlabel($pathname));
			} else {
			echo ProgressBar::next("Collecting metadata: ".shortlabel($pathname));
			}
		
		}

	echo ProgressBar::finish();
	
	}

// Hashes
//////////////////////////////////////////

if ($p['hash']) {
	
	if ($p['hash_limit']) {
		$message = "Generating hashes for files under ".$p['hash_limit']."GB";
		} else {
		$message = "Generating hashes for all files";
		}
	
	echo ProgressBar::start(count($fx),$message);

	foreach ($fx as $array) {
		$fid = $array[0];
		$pathname = $array[1];
		$size = filesize($pathname);
		$limit = $p['hash_limit']*1000000000;
		$check = $dbp->query("SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')")->fetch()[0];
		if ($check) {
			echo ProgressBar::next("Hash already exists: ".shortlabel($pathname));
			} elseif ($p['hash_limit'] && ($size > $limit)) {
			echo ProgressBar::next("Too big to hash: ".shortlabel($pathname)." (".human_filesize($size).")");
			} else {
			echo ProgressBar::next("Generating hash: ".shortlabel($pathname));
			$stmt = $dbp->prepare("INSERT INTO md5 VALUES (:fid, :hash)");
			$stmt->BindValue(":fid",$fid);
			$stmt->BindValue(":hash",md5_file($pathname));
			$stmt->execute();
			}
		}

	echo ProgressBar::finish();
	
	}

// Spotlight
//////////////////////////////////////////

$mb['i'] = array(	"PixelWidth",
					"PixelHeight",
					"Latitude",
					"Longitude",
					"DurationSeconds",
					"UseCount",
					"FSInvisible",
					"NumberOfPages",
					"PageHeight",
					"PageWidth"		);

$mb['t'] = array(	"Title",
					"ContentType",
					"Creator",
					"FSCreatorCode",
					"Kind",
					"FSTypeCode"	);

$mb['a'] = array(	"UserTags",
					"WhereFroms",
					"EncodingApplications"	);
				
$mb['d'] = array(	"DateAdded",
					"LastUsedDate",
					"ContentModificationDate",
					"ContentCreationDate"	);

$ibuild[] = ":pid, :spotlight";
$cbuild[] = "pid TEXT, spotlight TEXT";
foreach (array_merge($mb['i'],$mb['d']) as $item) {
	$cbuild[] = $item." INTEGER";
	$ibuild[] = ":".$item;
	}
foreach (array_merge($mb['t'],$mb['a']) as $item) {
	$cbuild[] = $item." TEXT";
	$ibuild[] = ":".$item;
	}

$p['spotlight'] = 1;

if ($p['spotlight']) {
	
	echo ProgressBar::start($passed_total,"Spotlight");
	
	$dbo->exec("CREATE TABLE mdls (".implode(",",$cbuild).")");
	
	foreach ($files as $splFileInfo) {
		
		$pid = md5($splFileInfo->getPathname());
		$shellpath = escapeshellarg($splFileInfo->getPathname());
		$mdls = shell_exec("mdls -plist - ".$shellpath." 2>&1");
		if (substr_count(@$mdls,"\n") < 2) { continue; }

		$parser = new plistParser();
		$spotlight = $parser->parseString($mdls);
		
		$stmt = $dbo->prepare("INSERT INTO mdls VALUES (".implode(",",$ibuild).")");

		foreach ($mb as $key => $list) {
			foreach ($list as $item) {

				if (@$spotlight["kMDItem".$item]) {
				$stmt->BindValue(":pid",$pid);
				$stmt->BindValue(":spotlight",$mdls);
				switch($key) {
						case "i":
						case "t":
						$stmt->BindValue(":".$item,$spotlight["kMDItem".$item]);
						break;
						case "a":
						$stmt->BindValue(":".$item,serialize($spotlight["kMDItem".$item]));
						break;
						case "d":
						$stmt->BindValue(":".$item,strtotime($spotlight["kMDItem".$item]));
						break;
						}
					}
				}
			}
			
		$stmt->execute();	
		echo ProgressBar::next();
		
		}					

	echo ProgressBar::finish();

	}

// Milk
//////////////////////////////////////////

// Aggregate values
$sb['a'] = array(	"t*Title" => array("e_Title","k_Title","m_Title"),
					"t*Dimensions" => array("k_PixelWidth.k_PixelHeight","e_PixelWidth.e_PixelHeight","m_PixelWidth.m_PixelHeight"),
					"i*Seconds" => array("k_DurationSeconds","e_Duration","m_Duration"),
					"d*DateTime" => array("e_DateTimeOriginal","m_EncodedDate","e_CreateDate","e_MediaCreateDate","k_ContentCreationDate"),
					"t*Origin" => array("e_CameraModelName","e_Producer","e_CreatorTool","e_WriterName","e_Software","e_Encoder","m_WritingApplication"),
					"t*GPS" => array("e_GPSPosition","k_Latitude.k_Longitude"),
					"t*Author" => array("e_Author","e_Artist","e_Creator","e_By-line")
					);

// Exiftool values
$sb['e'] = array(	"i*Orientation",
					"t*Compression",
					"t*ProfileDescription",
					"i*BitDepth",
					"t*LensType",
					"t*FocalLength",
					"t*Aperture",
					"t*LightSource",
					"t*WhiteBalance"	);

// Mediainfo values
$sb['m'] = array(	"t*VideoFormat",
					"t*AudioFormat",
					"i*Tracks",
					"t*Profile",
					"t*Bitrate"	);

// Build DB
unset($cbuild, $ibuild);
foreach (array_merge(array_keys($sb['a']),$sb['e'],$sb['m']) as $name) {
	$parts = explode("*",$name);
	$ibuild[] = ":".$parts[1];
	if ($parts[0] == "t") {
		$cbuild[] = $parts[1]." TEXT";
		} else {
		$cbuild[] = $parts[1]." INTEGER";
		}
	}

$dbo->exec("CREATE TABLE milk (".implode(",",$cbuild).")");
$stmt = $dbo->prepare("INSERT INTO milk VALUES (".implode(",",$ibuild).")");

$stmt->execute();

// Files
//////////////////////////////////////////

$j = 0;

echo ProgressBar::start($passed_total,"Skimming");

foreach ($files as $splFileInfo) {

	echo "\n";

	// DB

	$stmt = $dbo->prepare("INSERT INTO files VALUES (:pid, :fid, :Pathname, :Path, :Filename, :Extension, :Type, :Size, :Inode, :Perms, :Owner, :ATime, :CTime, :MTime, :LinkTarget, :RealPath, :stat, :items, :newest, :gfi_type, :gfi_attr, :gfi_created, :has_exif, :has_mediainfo, :has_hash, :thumb_filename, :thumb_width, :thumb_height, :has_contents, :contents_filename)");

	// Identify dir, file, link or bundle dir

	$type = $splFileInfo->getType();
	if ($type == "dir") {
		foreach ($p['bundles'] as $bundle) {
			$check = ".".$bundle;
			if (substr($splFileInfo->getFilename(), -(strlen($check)), strlen($check)) == $check) { $type = "bundle"; }
			}	
		}
	
	$stmt->BindValue(":Type",$type);
	
	// Path basics
	
	$pathname = $splFileInfo->getPathname();
	$path = $splFileInfo->getPath();
	$filename = $splFileInfo->getFilename();
	$extension = $splFileInfo->getExtension();
	$shellpath = escapeshellarg($pathname);

	$stmt->BindValue(":Pathname",$pathname);
	$stmt->BindValue(":Path",$path);
	$stmt->BindValue(":Filename",$filename);
	$stmt->BindValue(":Extension",$extension);

	//stat
	$stmt->BindValue(":stat",$sty[$j]);

	if ($type == "link") {
	
		$stmt->BindValue(":LinkTarget",$splFileInfo->getLinkTarget());
		$stmt->BindValue(":RealPath",$splFileInfo->getRealPath());
	
		} else {
		
		$stmt->BindValue(":Inode",$splFileInfo->getInode());
		$stmt->BindValue(":Perms",$splFileInfo->getPerms());
		$stmt->BindValue(":Owner",$splFileInfo->getOwner().":".$splFileInfo->getGroup());
		
		$stmt->BindValue(":ATime",$stx[$j][0]);
		$stmt->BindValue(":MTime",$stx[$j][1]);
		$stmt->BindValue(":CTime",$stx[$j][2]);
	
		}

	echo shortlabel($pathname,50);
	
	// ------------------------------------------------ //

	// Generate PID and FID

	$pid = md5($pathname);
	$stmt->BindValue(":pid",$pid);

	if ($type == "file") {
		$fid = md5($splFileInfo->getSize().$splFileInfo->getMtime().$splFileInfo->getBasename());
		$stmt->BindValue(":fid",$fid);
		}

	// Size

	if ($type == "dir" || $type == "bundle") {
		$size = trim(shell_exec("du -ks ".$shellpath." | cut -f1"))*1024;
		} elseif ($type == "file") {
		$size = $splFileInfo->getSize();
		} else {
		$size = null;
		}
	$stmt->BindValue(":Size",@$size);
	
	// ------------------------------------------------ //

	// Items
	
	if ($type == "dir" || $type == "bundle" ) {
		$items = chop(@shell_exec("find ".$shellpath." \( ! -regex '.*/\..*' \) | wc -l 2>&1"))-1;
		$stmt->BindValue(":items",@$items);
		}

	// ------------------------------------------------ //

	// Newest
	
	if ($type == "dir") {
		$newest = @filemtime(chop(shell_exec("find ".$shellpath." -type f -not -path '*/\.*' -print0 | xargs -0 stat -f \"%m %N\" | sort -rn 2>&1 | head -1 | cut -f2- -d\" \"")));
		$stmt->BindValue(":newest",@$newest);
		}

	// ------------------------------------------------ //

	// GetFileInfo
	
	$gfiparts = explode("\n", chop(shell_exec($bin_gfi." -P ".$shellpath." 2>&1")));
	if (is_array($gfiparts)) {
		foreach ($gfiparts as $line) {
			list($label, $value) = explode(": ", $line);
			$gfi[$label] = isset($value) ? trim($value,"\"") : null;
			}
		}
	
	$writegfitype = @$gfi['type'].":".@$gfi['creator'];
	if ($writegfitype == "\\0\\0\\0\\0:\\0\\0\\0\\0" || $writegfitype == ":") { $writegfitype = null; }
	
	$stmt->BindValue(":gfi_type",$writegfitype);
	$stmt->BindValue(":gfi_attr",@$gfi['attributes']);
	$stmt->BindValue(":gfi_created",strtotime($gfi['created']));
	
	// ------------------------------------------------ //

	// Pool
	
	if ($type == "file") {
	
		unset($fetch_exif, $fetch_media, $fetch_hash, $fetch_thumb, $yes_exif, $yes_media, $yes_hash);
		
		$fetch_exif = @unserialize($dbp->query("SELECT tags FROM exiftool WHERE fid='".$fid."'")->fetch()[0]);
		is_array($fetch_exif) ? $yes_exif = 1 : $yes_exif = 0;
		$stmt->BindValue(":has_exif",$yes_exif);
		
		$fetch_media = @unserialize($dbp->query("SELECT info FROM mediainfo WHERE fid='".$fid."'")->fetch()[0]);
		is_array($fetch_media) ? $yes_media = 1 : $yes_media = 0;
		$stmt->BindValue(":has_mediainfo",$yes_media);

		$yes_hash = $dbp->query("SELECT EXISTS(SELECT 1 FROM md5 WHERE fid='".$fid."')")->fetch()[0];
		$stmt->BindValue(":has_hash",$yes_hash);
	
		$fetch_thumb = $dbp->query("SELECT * FROM thumbs WHERE fid='".$fid."'")->fetch();
		if (@$fetch_thumb['relative_path']) {
			$stmt->BindValue(":thumb_filename",$fetch_thumb['relative_path']);
			$stmt->BindValue(":thumb_width",$fetch_thumb['width']);
			$stmt->BindValue(":thumb_height",$fetch_thumb['height']);
			} else {
			$stmt->BindValue(":thumb_filename",null);
			}

		}

	// ------------------------------------------------ //

	// Milk
	
	
	
	
	
	
	
	

	// ------------------------------------------------ //

	// Write to DB

	$stmt->execute();

	// Set fileatime back to original value
	
	//if ($type != "link" && is_writable($pathname) && $p['fixatimes']) {
	//	exec("touch -at `date -r ".$atime." +%Y%m%d%H%M.%S` ".$shellpath." 2>&1");
	//	}
	
	// Double check stat for file against pre-run value
	
	if ($p['verify_stat'] && $type != "link") {
		
		$pre_access = $pre_modify = $pre_change = null;
		$post_access = $post_modify = $post_change = null;
				
		foreach (explode("\n", $sty[$j]) as $line) {
			$check = substr($line, 0, 6);
			if ($check == "Access") { $pre_access = $line; }
			if ($check == "Modify") { $pre_modify = $line; }
			if ($check == "Change") { $pre_change = $line; }
			}	
	
		foreach (explode("\n", chop(shell_exec("stat -x ".$shellpath." 2>&1"))) as $line) {
			$check = substr($line, 0, 6);
			if ($check == "Access") { $post_access = $line; }
			if ($check == "Modify") { $post_modify = $line; }
			if ($check == "Change") { $post_change = $line; }
			}
		
		$message = array();
		if ($pre_access != $post_access) {
			$message[] = "ATIME";
			}
		if ($pre_modify != $post_modify) {
			$message[] = "MTIME";
			}
		if ($pre_change != $post_change) {
			$message[] = "CTIME";
			}
		
		if (count($message)) { echo "\nChanged: ".implode(", ", $message)."\n"; }

		}
	
	echo "\n";
	echo ProgressBar::next();
	$j++;

	}
	
echo ProgressBar::finish();

// Cleanup
//////////////////////////////////////////

echo "\n";

if (file_exists($error_log_file)) { echo file_get_contents($error_log_file); }

$seconds = floor($time = microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"]);
$dbo->exec("UPDATE _skim SET status='completed_in_".$seconds."'");

// rsync

if ($p['rsync_dest']) {
	echo "\nrsync...\n";
	$command = "rsync -avv -e ssh ".$bpath." ".$p['rsync_dest'];
	$count = trim(shell_exec("find ".escapeshellarg($bpath)." | wc -l"));
	echo ProgressBar::start($count,"rsync");
	$pipe = popen($command, "r");
	while(fgets($pipe, 2048)) { echo ProgressBar::next(); }
	pclose($pipe);
	echo ProgressBar::finish();
	}

echo "Finished in ".$seconds." seconds\n\n";

unset($dbo, $dbp, $files, $family, $fx);

?>